rhi: Make QSGRenderNode usable in Quick3Dv5.15.0-beta2

Cannot just rely on a render() function, need a point where the "prepare"
step can be performed, outside the main renderpass.

Also enables proper m_useDepthBuffer handling for the RHI code path.

Task-number: QTBUG-82797
Task-number: QTBUG-82793
Change-Id: I525228e53acefad9f2e6a33d446260a1521d8ae1
Reviewed-by: Eirik Aavitsland <eirik.aavitsland@qt.io>

3 files changed, 56 insertions, 41 deletions

diff --git a/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp b/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp
index f0c57ffd80..e94dd502fc 100644
--- a/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp
+++ b/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp
@@ -1031,6 +1031,7 @@ Renderer::Renderer(QSGDefaultRenderContext *ctx)
                 ? "static" : (m_bufferStrategy == GL_DYNAMIC_DRAW ? "dynamic" : "stream")));
     }
 
+    static const bool useDepth = qEnvironmentVariableIsEmpty("QSG_NO_DEPTH_BUFFER");
     if (!m_rhi) {
         // If rendering with an OpenGL Core profile context, we need to create a VAO
         // to hold our vertex specification state.
@@ -1038,9 +1039,9 @@ Renderer::Renderer(QSGDefaultRenderContext *ctx)
             m_vao = new QOpenGLVertexArrayObject(this);
             m_vao->create();
         }
-
-        bool useDepth = qEnvironmentVariableIsEmpty("QSG_NO_DEPTH_BUFFER");
         m_useDepthBuffer = useDepth && ctx->openglContext()->format().depthBufferSize() > 0;
+    } else {
+        m_useDepthBuffer = useDepth;
     }
 }
 
@@ -1401,12 +1402,11 @@ void Renderer::nodeWasRemoved(Node *node)
             m_elementsToDelete.add(e);
 
             if (m_renderNodeElements.isEmpty()) {
-                if (m_rhi) {
-                    m_useDepthBuffer = true;
-                } else {
-                    static bool useDepth = qEnvironmentVariableIsEmpty("QSG_NO_DEPTH_BUFFER");
+                static const bool useDepth = qEnvironmentVariableIsEmpty("QSG_NO_DEPTH_BUFFER");
+                if (m_rhi)
+                    m_useDepthBuffer = useDepth;
+                else
                     m_useDepthBuffer = useDepth && m_context->openglContext()->format().depthBufferSize() > 0;
-                }
             }
         }
     }
@@ -3717,7 +3717,7 @@ void Renderer::renderMergedBatch(PreparedRenderBatch *renderBatch) // split prep
             { batch->vbo.buf, quint32(draw.vertices) },
             { batch->vbo.buf, quint32(draw.zorders) }
         };
-        cb->setVertexInput(VERTEX_BUFFER_BINDING, 2, vbufBindings,
+        cb->setVertexInput(VERTEX_BUFFER_BINDING, m_useDepthBuffer ? 2 : 1, vbufBindings,
                            batch->ibo.buf, draw.indices,
                            m_uint32IndexForRhi ? QRhiCommandBuffer::IndexUInt32 : QRhiCommandBuffer::IndexUInt16);
         cb->drawIndexed(draw.indexCount);
@@ -4053,8 +4053,8 @@ void Renderer::renderBatches()
         m_pstate.viewportSet = false;
         m_pstate.scissorSet = false;
 
-        m_gstate.depthTest = true;
-        m_gstate.depthWrite = true;
+        m_gstate.depthTest = m_useDepthBuffer;
+        m_gstate.depthWrite = m_useDepthBuffer;
         m_gstate.depthFunc = QRhiGraphicsPipeline::Less;
         m_gstate.blending = false;
 
@@ -4086,8 +4086,8 @@ void Renderer::renderBatches()
         m_gstate.blending = true;
         // factors never change, always set for premultiplied alpha based blending
 
-        // depth test stays enabled but no need to write out depth from the
-        // transparent (back-to-front) pass
+        // depth test stays enabled (if m_useDepthBuffer, that is) but no need
+        // to write out depth from the transparent (back-to-front) pass
         m_gstate.depthWrite = false;
 
         QVarLengthArray<PreparedRenderBatch, 64> alphaRenderBatches;
@@ -4508,6 +4508,35 @@ bool Renderer::prepareRhiRenderNode(Batch *batch, PreparedRenderBatch *renderBat
 
     updateClipState(rd->m_clip_list, batch);
 
+    QSGNode *xform = e->renderNode->parent();
+    QMatrix4x4 matrix;
+    QSGNode *root = rootNode();
+    if (e->root) {
+        matrix = qsg_matrixForRoot(e->root);
+        root = e->root->sgNode;
+    }
+    while (xform != root) {
+        if (xform->type() == QSGNode::TransformNodeType) {
+            matrix = matrix * static_cast<QSGTransformNode *>(xform)->combinedMatrix();
+            break;
+        }
+        xform = xform->parent();
+    }
+    rd->m_matrix = &matrix;
+
+    QSGNode *opacity = e->renderNode->parent();
+    rd->m_opacity = 1.0;
+    while (opacity != rootNode()) {
+        if (opacity->type() == QSGNode::OpacityNodeType) {
+            rd->m_opacity = static_cast<QSGOpacityNode *>(opacity)->combinedOpacity();
+            break;
+        }
+        opacity = opacity->parent();
+    }
+
+    if (rd->m_prepareCallback)
+        rd->m_prepareCallback();
+
     renderBatch->batch = batch;
     renderBatch->sms = nullptr;
 
@@ -4536,38 +4565,15 @@ void Renderer::renderRhiRenderNode(const Batch *batch) // split prepare-render (
     state.m_scissorEnabled = batch->clipState.type & ClipState::ScissorClip;
     state.m_stencilEnabled = batch->clipState.type & ClipState::StencilClip;
 
-    QSGNode *xform = e->renderNode->parent();
-    QMatrix4x4 matrix;
-    QSGNode *root = rootNode();
-    if (e->root) {
-        matrix = qsg_matrixForRoot(e->root);
-        root = e->root->sgNode;
-    }
-    while (xform != root) {
-        if (xform->type() == QSGNode::TransformNodeType) {
-            matrix = matrix * static_cast<QSGTransformNode *>(xform)->combinedMatrix();
-            break;
-        }
-        xform = xform->parent();
-    }
-    rd->m_matrix = &matrix;
-
-    QSGNode *opacity = e->renderNode->parent();
-    rd->m_opacity = 1.0;
-    while (opacity != rootNode()) {
-        if (opacity->type() == QSGNode::OpacityNodeType) {
-            rd->m_opacity = static_cast<QSGOpacityNode *>(opacity)->combinedOpacity();
-            break;
-        }
-        opacity = opacity->parent();
-    }
-
     const QSGRenderNode::StateFlags changes = e->renderNode->changedStates();
 
     QRhiCommandBuffer *cb = commandBuffer();
-    cb->beginExternal();
+    const bool needsExternal = rd->m_needsExternalRendering;
+    if (needsExternal)
+        cb->beginExternal();
     e->renderNode->render(&state);
-    cb->endExternal();
+    if (needsExternal)
+        cb->endExternal();
 
     rd->m_matrix = nullptr;
     rd->m_clip_list = nullptr;
diff --git a/src/quick/scenegraph/coreapi/qsgrendernode.cpp b/src/quick/scenegraph/coreapi/qsgrendernode.cpp
index 63878954bf..45ad49475f 100644
--- a/src/quick/scenegraph/coreapi/qsgrendernode.cpp
+++ b/src/quick/scenegraph/coreapi/qsgrendernode.cpp
@@ -77,6 +77,8 @@ QSGRenderNodePrivate::QSGRenderNodePrivate()
     : m_matrix(nullptr)
     , m_clip_list(nullptr)
     , m_opacity(1)
+    , m_needsExternalRendering(true)
+    , m_prepareCallback(nullptr)
 {
 }
 
diff --git a/src/quick/scenegraph/coreapi/qsgrendernode_p.h b/src/quick/scenegraph/coreapi/qsgrendernode_p.h
index 5c42e55689..534d630f15 100644
--- a/src/quick/scenegraph/coreapi/qsgrendernode_p.h
+++ b/src/quick/scenegraph/coreapi/qsgrendernode_p.h
@@ -51,12 +51,14 @@
 // We mean it.
 //
 
+#include <QtQuick/private/qtquickglobal_p.h>
 #include <QtQuick/qsgnode.h>
 #include <QtQuick/qsgrendernode.h>
+#include <functional>
 
 QT_BEGIN_NAMESPACE
 
-class QSGRenderNodePrivate
+class Q_QUICK_PRIVATE_EXPORT QSGRenderNodePrivate
 {
 public:
     QSGRenderNodePrivate();
@@ -66,6 +68,11 @@ public:
     const QMatrix4x4 *m_matrix;
     const QSGClipNode *m_clip_list;
     qreal m_opacity;
+
+    // ### Qt 6: change this into a value for flags()
+    bool m_needsExternalRendering;
+    // ### Qt 6: change this into a virtual prepare() function
+    std::function<void()> m_prepareCallback;
 };
 
 QT_END_NAMESPACE