Make rendernode suitable for public consumption

For non-OpenGL APIs the primary (and likely the only) way to add custom
rendering into the Qt Quick scene is via the render node. Other
approaches,like the before/afterRendering signals,
QQuickFramebufferObject, remain OpenGL-only. (although QQuickFramebufferObject
may get a multi-API replacement based on QSGRenderNode at a later time)

Note that this is not a generic 3D content integration enabler. It targets
creating 2D and 2.5D Quick items with custom rendering via the graphics
API in use.

Make QSGRenderNode public, enhance the docs a bit and add a releaseResources().
Add a QSGRendererInterface with a query function in QQuickWindow and
QSGEngine. The scenegraph adaptation can then return a custom implementation of
the interface. This will be necessary to query API-specific values, f.ex. the
ID3D12Device and ID3D12CommandList when running with the d3d12 backend.

The interface allows querying the API and void* resources. Resources that
we know about in advance are enum-based to prevent the
QPlatformNativeInterface-like ugliness of string keys.

Support is there in the batch renderer already, fix this up according to
the new public API, and implement the corresponding bits for the D3D12
renderer.

For D3D12, fix also an issue with QSGNode destruction where graphics resources
in use were attempted to be final-released without a proper wait.

The semantics of changedStates() in QSGRenderNode is changed so that it can be
called at any time, including before render(). This is very useful since we can
implement some state restoring in a more efficient manner.

Added a new example as well. Documentation for QSGRenderNode is heavily
expanded.

Change-Id: I4c4a261c55791d0e38743a784bc4c05a53b3462d
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

10 files changed, 196 insertions, 21 deletions

diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12context.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12context.cpp
index 0bb342226b..d43dcd5997 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12context.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12context.cpp
@@ -97,4 +97,14 @@ QSurfaceFormat QSGD3D12Context::defaultSurfaceFormat() const
     return QSurfaceFormat::defaultFormat();
 }
 
+QSGRendererInterface *QSGD3D12Context::rendererInterface(QSGRenderContext *renderContext)
+{
+    QSGD3D12RenderContext *rc = static_cast<QSGD3D12RenderContext *>(renderContext);
+    if (!rc->engine()) {
+        qWarning("No D3D12 engine available yet (no render thread due to window not exposed?)");
+        return nullptr;
+    }
+    return rc->engine();
+}
+
 QT_END_NAMESPACE
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12context_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12context_p.h
index 2afe22e3af..c597ed90dd 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12context_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12context_p.h
@@ -69,6 +69,7 @@ public:
     QSGLayer *createLayer(QSGRenderContext *rc) override;
     QSize minimumFBOSize() const override;
     QSurfaceFormat defaultSurfaceFormat() const override;
+    QSGRendererInterface *rendererInterface(QSGRenderContext *renderContext) override;
 };
 
 QT_END_NAMESPACE
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
index afeeea760c..cbdf54a9ea 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
@@ -360,6 +360,16 @@ void QSGD3D12Engine::endLayer()
     d->endLayer();
 }
 
+void QSGD3D12Engine::invalidateCachedFrameState()
+{
+    d->invalidateCachedFrameState();
+}
+
+void QSGD3D12Engine::restoreFrameState(bool minimal)
+{
+    d->restoreFrameState(minimal);
+}
+
 void QSGD3D12Engine::finalizePipeline(const QSGD3D12PipelineState &pipelineState)
 {
     d->finalizePipeline(pipelineState);
@@ -495,6 +505,16 @@ void QSGD3D12Engine::activateRenderTargetAsTexture(uint id)
     d->activateRenderTargetAsTexture(id);
 }
 
+QSGRendererInterface::GraphicsAPI QSGD3D12Engine::graphicsAPI() const
+{
+    return Direct3D12;
+}
+
+void *QSGD3D12Engine::getResource(Resource resource) const
+{
+    return d->getResource(resource);
+}
+
 static inline quint32 alignedSize(quint32 size, quint32 byteAlign)
 {
     return (size + byteAlign - 1) & ~(byteAlign - 1);
@@ -1296,7 +1316,11 @@ void QSGD3D12EnginePrivate::beginDrawCalls()
 {
     frameCommandList->Reset(frameCommandAllocator[frameIndex % frameInFlightCount].Get(), nullptr);
     commandList = frameCommandList.Get();
+    invalidateCachedFrameState();
+}
 
+void QSGD3D12EnginePrivate::invalidateCachedFrameState()
+{
     tframeData.drawingMode = QSGGeometry::DrawingMode(-1);
     tframeData.currentIndexBuffer = 0;
     tframeData.drawCount = 0;
@@ -1305,6 +1329,18 @@ void QSGD3D12EnginePrivate::beginDrawCalls()
     tframeData.descHeapSet = false;
 }
 
+void QSGD3D12EnginePrivate::restoreFrameState(bool minimal)
+{
+    queueSetRenderTarget(currentRenderTarget);
+    if (!minimal) {
+        queueViewport(tframeData.viewport);
+        queueScissor(tframeData.scissor);
+        queueSetBlendFactor(tframeData.blendFactor);
+        queueSetStencilRef(tframeData.stencilRef);
+    }
+    finalizePipeline(tframeData.pipelineState);
+}
+
 void QSGD3D12EnginePrivate::beginFrameDraw()
 {
     if (windowSamples == 1)
@@ -1886,12 +1922,7 @@ void QSGD3D12EnginePrivate::queueDraw(const QSGD3D12Engine::DrawParams &params)
         // start a new one
         beginDrawCalls();
         // prepare for the upcoming drawcalls
-        queueSetRenderTarget(currentRenderTarget);
-        queueViewport(tframeData.viewport);
-        queueScissor(tframeData.scissor);
-        queueSetBlendFactor(tframeData.blendFactor);
-        queueSetStencilRef(tframeData.stencilRef);
-        finalizePipeline(tframeData.pipelineState);
+        restoreFrameState();
     }
 }
 
@@ -2737,4 +2768,19 @@ void QSGD3D12EnginePrivate::activateRenderTargetAsTexture(uint id)
     tframeData.activeTextures.append(TransientFrameData::ActiveTexture::ActiveTexture(TransientFrameData::ActiveTexture::TypeRenderTarget, id));
 }
 
+void *QSGD3D12EnginePrivate::getResource(QSGRendererInterface::Resource resource) const
+{
+    switch (resource) {
+    case QSGRendererInterface::Device:
+        return device;
+    case QSGRendererInterface::CommandQueue:
+        return commandQueue.Get();
+    case QSGRendererInterface::CommandList:
+        return commandList;
+    default:
+        break;
+    }
+    return nullptr;
+}
+
 QT_END_NAMESPACE
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
index 84bb5a554e..3b7dd7cdc6 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
@@ -55,6 +55,7 @@
 #include <QImage>
 #include <QVector4D>
 #include <qsggeometry.h>
+#include <qsgrendererinterface.h>
 #include <qt_windows.h>
 
 QT_BEGIN_NAMESPACE
@@ -260,7 +261,7 @@ inline uint qHash(const QSGD3D12PipelineState &key, uint seed = 0)
             + key.topologyType;
 }
 
-class QSGD3D12Engine
+class QSGD3D12Engine : public QSGRendererInterface
 {
 public:
     QSGD3D12Engine();
@@ -278,6 +279,8 @@ public:
     void endFrame();
     void beginLayer();
     void endLayer();
+    void invalidateCachedFrameState();
+    void restoreFrameState(bool minimal = false);
 
     uint genBuffer();
     void releaseBuffer(uint id);
@@ -344,6 +347,10 @@ public:
     void releaseRenderTarget(uint id);
     void activateRenderTargetAsTexture(uint id);
 
+    // QSGRendererInterface
+    GraphicsAPI graphicsAPI() const override;
+    void *getResource(Resource resource) const override;
+
 private:
     QSGD3D12EnginePrivate *d;
     Q_DISABLE_COPY(QSGD3D12Engine)
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
index 40d1fdb510..047036b52a 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
@@ -142,6 +142,8 @@ public:
     void endFrame();
     void beginLayer();
     void endLayer();
+    void invalidateCachedFrameState();
+    void restoreFrameState(bool minimal = false);
 
     uint genBuffer();
     void releaseBuffer(uint id);
@@ -176,6 +178,8 @@ public:
     void releaseRenderTarget(uint id);
     void activateRenderTargetAsTexture(uint id);
 
+    void *getResource(QSGRendererInterface::Resource resource) const;
+
     // the device is intentionally hidden here. all resources have to go
     // through the engine and, unlike with GL, cannot just be created in random
     // places due to the need for proper tracking, managing and releasing.
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12rendercontext.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12rendercontext.cpp
index af3e8e36ee..ab590b95c5 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12rendercontext.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12rendercontext.cpp
@@ -56,6 +56,14 @@ QSGD3D12RenderContext::QSGD3D12RenderContext(QSGContext *ctx)
 {
 }
 
+bool QSGD3D12RenderContext::isValid() const
+{
+    // The render thread sets an engine when it starts up and resets when it
+    // quits. The rc is initialized and functional between those two points,
+    // regardless of any calls to invalidate(). See setEngine().
+    return m_engine != nullptr;
+}
+
 void QSGD3D12RenderContext::invalidate()
 {
     if (Q_UNLIKELY(debug_render()))
@@ -98,4 +106,15 @@ void QSGD3D12RenderContext::renderNextFrame(QSGRenderer *renderer, uint fbo)
     static_cast<QSGD3D12Renderer *>(renderer)->renderScene(fbo);
 }
 
+void QSGD3D12RenderContext::setEngine(QSGD3D12Engine *engine)
+{
+    if (m_engine == engine)
+        return;
+
+    m_engine = engine;
+
+    if (m_engine)
+        emit initialized();
+}
+
 QT_END_NAMESPACE
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12rendercontext_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12rendercontext_p.h
index 1acb7e4205..e2cd6e3182 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12rendercontext_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12rendercontext_p.h
@@ -61,12 +61,13 @@ class QSGD3D12RenderContext : public QSGRenderContext
 {
 public:
     QSGD3D12RenderContext(QSGContext *ctx);
+    bool isValid() const override;
     void invalidate() override;
     void renderNextFrame(QSGRenderer *renderer, uint fbo) override;
     QSGTexture *createTexture(const QImage &image, uint flags) const override;
     QSGRenderer *createRenderer() override;
 
-    void setEngine(QSGD3D12Engine *engine) { m_engine = engine; }
+    void setEngine(QSGD3D12Engine *engine);
     QSGD3D12Engine *engine() { return m_engine; }
 
 private:
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12renderer.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12renderer.cpp
index 7f8b10f003..f76927c0bd 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12renderer.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12renderer.cpp
@@ -40,6 +40,7 @@
 #include "qsgd3d12renderer_p.h"
 #include "qsgd3d12rendercontext_p.h"
 #include <private/qsgnodeupdater_p.h>
+#include <private/qsgrendernode_p.h>
 
 #include "vs_stencilclip.hlslh"
 #include "ps_stencilclip.hlslh"
@@ -206,6 +207,11 @@ void QSGD3D12Renderer::buildRenderList(QSGNode *node, QSGClipNode *clip)
         gn->setClipList(clip);
         if (node->type() == QSGNode::ClipNodeType)
             clip = static_cast<QSGClipNode *>(node);
+    } else if (node->type() == QSGNode::RenderNodeType) {
+        QSGRenderNode *rn = static_cast<QSGRenderNode *>(node);
+        Element e;
+        e.node = rn;
+        m_renderList.add(e);
     }
 
     QSGNODE_TRAVERSE(node)
@@ -316,6 +322,7 @@ void QSGD3D12Renderer::render()
                 if (gn->inheritedOpacity() > 0.999f && ((gn->activeMaterial()->flags() & QSGMaterial::Blending) == 0))
                     m_opaqueElements.setBit(i);
             }
+            // QSGRenderNodes are always treated as non-opaque
         }
     }
 
@@ -432,15 +439,36 @@ void QSGD3D12Renderer::renderElements()
 
     // ...then the alpha ones
     for (int i = 0; i < m_renderList.size(); ++i) {
-        if (m_renderList.at(i).node->type() == QSGNode::GeometryNodeType && !m_opaqueElements.testBit(i))
+        if ((m_renderList.at(i).node->type() == QSGNode::GeometryNodeType && !m_opaqueElements.testBit(i))
+                || m_renderList.at(i).node->type() == QSGNode::RenderNodeType)
             renderElement(i);
     }
 }
 
+struct RenderNodeState : public QSGRenderNode::RenderState
+{
+    const QMatrix4x4 *projectionMatrix() const override { return m_projectionMatrix; }
+    QRect scissorRect() const { return m_scissorRect; }
+    bool scissorEnabled() const { return m_scissorEnabled; }
+    int stencilValue() const { return m_stencilValue; }
+    bool stencilEnabled() const { return m_stencilEnabled; }
+
+    const QMatrix4x4 *m_projectionMatrix;
+    QRect m_scissorRect;
+    bool m_scissorEnabled;
+    int m_stencilValue;
+    bool m_stencilEnabled;
+};
+
 void QSGD3D12Renderer::renderElement(int elementIndex)
 {
     Element &e = m_renderList.at(elementIndex);
-    Q_ASSERT(e.node->type() == QSGNode::GeometryNodeType);
+    Q_ASSERT(e.node->type() == QSGNode::GeometryNodeType || e.node->type() == QSGNode::RenderNodeType);
+
+    if (e.node->type() == QSGNode::RenderNodeType) {
+        renderRenderNode(static_cast<QSGRenderNode *>(e.node), elementIndex);
+        return;
+    }
 
     if (e.vboOffset < 0)
         return;
@@ -508,7 +536,7 @@ void QSGD3D12Renderer::renderElement(int elementIndex)
 
     m_lastMaterialType = m->type();
 
-    setupClipping(gn, elementIndex);
+    setupClipping(gn->clipList(), elementIndex);
 
     // ### Lines and points with sizes other than 1 have to be implemented in some other way. Just ignore for now.
     if (g->drawingMode() == QSGGeometry::DrawLineStrip || g->drawingMode() == QSGGeometry::DrawLines) {
@@ -575,16 +603,10 @@ void QSGD3D12Renderer::queueDrawCall(const QSGGeometry *g, const QSGD3D12Rendere
     m_engine->queueDraw(dp);
 }
 
-void QSGD3D12Renderer::setupClipping(const QSGGeometryNode *gn, int elementIndex)
+void QSGD3D12Renderer::setupClipping(const QSGClipNode *clip, int elementIndex)
 {
-    const QSGClipNode *clip = gn->clipList();
-
     const QRect devRect = deviceRect();
     QRect scissorRect;
-    enum ClipType {
-        ClipScissor = 0x1,
-        ClipStencil = 0x2
-    };
     int clipTypes = 0;
     quint32 stencilValue = 0;
 
@@ -647,9 +669,13 @@ void QSGD3D12Renderer::setupClipping(const QSGGeometryNode *gn, int elementIndex
     if (clipTypes & ClipStencil) {
         m_pipelineState.stencilEnable = true;
         m_engine->queueSetStencilRef(stencilValue);
+        m_currentStencilValue = stencilValue;
     } else {
         m_pipelineState.stencilEnable = false;
+        m_currentStencilValue = 0;
     }
+
+    m_currentClipTypes = clipTypes;
 }
 
 void QSGD3D12Renderer::setScissor(const QRect &r)
@@ -707,4 +733,52 @@ void QSGD3D12Renderer::renderStencilClip(const QSGClipNode *clip, int elementInd
     ++stencilValue;
 }
 
+void QSGD3D12Renderer::renderRenderNode(QSGRenderNode *node, int elementIndex)
+{
+    QSGRenderNodePrivate *rd = QSGRenderNodePrivate::get(node);
+    RenderNodeState state;
+
+    setupClipping(rd->m_clip_list, elementIndex);
+
+    QMatrix4x4 pm = projectionMatrix();
+    state.m_projectionMatrix = &pm;
+    state.m_scissorEnabled = m_currentClipTypes & ClipScissor;
+    state.m_stencilEnabled = m_currentClipTypes & ClipStencil;
+    state.m_scissorRect = m_activeScissorRect;
+    state.m_stencilValue = m_currentStencilValue;
+
+    // ### rendernodes do not have the QSGBasicGeometryNode infrastructure
+    // for storing combined matrices, opacity and such, but perhaps they should.
+    QSGNode *xform = node->parent();
+    QSGNode *root = rootNode();
+    QMatrix4x4 modelview;
+    while (xform != root) {
+        if (xform->type() == QSGNode::TransformNodeType) {
+            modelview *= static_cast<QSGTransformNode *>(xform)->combinedMatrix();
+            break;
+        }
+        xform = xform->parent();
+    }
+    rd->m_matrix = &modelview;
+
+    QSGNode *opacity = node->parent();
+    rd->m_opacity = 1.0;
+    while (opacity != rootNode()) {
+        if (opacity->type() == QSGNode::OpacityNodeType) {
+            rd->m_opacity = static_cast<QSGOpacityNode *>(opacity)->combinedOpacity();
+            break;
+        }
+        opacity = opacity->parent();
+    }
+
+    node->render(&state);
+
+    m_engine->invalidateCachedFrameState();
+    // For simplicity, reset viewport, scissor, blend factor, stencil ref when
+    // any of them got changed. This will likely be rare so skip these otherwise.
+    // Render target, pipeline state, draw call related stuff will be reset always.
+    const bool restoreMinimal = node->changedStates() == 0;
+    m_engine->restoreFrameState(restoreMinimal);
+}
+
 QT_END_NAMESPACE
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12renderer_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12renderer_p.h
index 0877c3699a..bed34df3a1 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12renderer_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12renderer_p.h
@@ -59,6 +59,8 @@
 
 QT_BEGIN_NAMESPACE
 
+class QSGRenderNode;
+
 class QSGD3D12Renderer : public QSGRenderer
 {
 public:
@@ -78,12 +80,13 @@ private:
     void renderElements();
     void renderElement(int elementIndex);
     void setInputLayout(const QSGGeometry *g, QSGD3D12PipelineState *pipelineState);
-    void setupClipping(const QSGGeometryNode *gn, int elementIndex);
+    void setupClipping(const QSGClipNode *clip, int elementIndex);
     void setScissor(const QRect &r);
     void renderStencilClip(const QSGClipNode *clip, int elementIndex, const QMatrix4x4 &m, quint32 &stencilValue);
+    void renderRenderNode(QSGRenderNode *node, int elementIndex);
 
     struct Element {
-        QSGBasicGeometryNode *node = nullptr;
+        QSGNode *node = nullptr;
         qint32 vboOffset = -1;
         qint32 iboOffset = -1;
         quint32 iboStride = 0;
@@ -121,6 +124,12 @@ private:
     bool m_projectionChangedDueToDeviceSize;
 
     uint m_renderTarget = 0;
+    quint32 m_currentStencilValue;
+    enum ClipType {
+        ClipScissor = 0x1,
+        ClipStencil = 0x2
+    };
+    int m_currentClipTypes;
 };
 
 QT_END_NAMESPACE
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
index 94bdf38819..2c05ce01c6 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
@@ -331,8 +331,12 @@ bool QSGD3D12RenderThread::event(QEvent *e)
                 qDebug("RT - WM_TryRelease - invalidating rc");
             if (wme->window) {
                 QQuickWindowPrivate *wd = QQuickWindowPrivate::get(wme->window);
-                if (wme->destroying)
+                if (wme->destroying) {
+                    // QSGNode destruction may release graphics resources in use so wait first.
+                    engine->waitGPU();
+                    // Bye bye nodes...
                     wd->cleanupNodesOnShutdown();
+                }
                 rc->invalidate();
                 QCoreApplication::processEvents();
                 QCoreApplication::sendPostedEvents(0, QEvent::DeferredDelete);