Do not create depth/stencil buffer when the env var says so

Both for the swapchain and more importantly, layers. The latter was
never implemented, not in Qt 5 with OpenGL either, and it becomes a
problem for resource-limited use cases because QSG_NO_DEPTH_BUFFER still
creates depth/stencil attachments for layers even though the 2D
rendering has no use for those then.

Clarify the QQuickGraphicsConfiguration docs as well. The story is
fairly convoluted, because the useDepthFor2D flag one can set from C++
is not 100% the same as the env.var. (and it really should not be)
The flag is about relying on depth testing in 2D (and so enabling the
potentially more efficient opaque batches), but it does not prevent adding
a View3D or other stuff that requires a depth buffer. The env.var on
the other hand does both: it (depending on the platform) disables
depth (and often stencil) buffers, thus using fewer resources, and also
triggers the depth-less 2D rendering path (alpha batches only). But that
is not always compatible with 3D then (like an offscren View3D will work,
other modes may not)

Change-Id: I5ac1ce154fe78a3ec8bd1a698c1c0b944ce8077e
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

5 files changed, 53 insertions, 22 deletions

diff --git a/src/quick/scenegraph/coreapi/qsgbatchrenderer_p.h b/src/quick/scenegraph/coreapi/qsgbatchrenderer_p.h
index cfb5384eed..54652df923 100644
--- a/src/quick/scenegraph/coreapi/qsgbatchrenderer_p.h
+++ b/src/quick/scenegraph/coreapi/qsgbatchrenderer_p.h
@@ -956,6 +956,12 @@ int Renderer::mergedIndexElemSize() const
     return m_uint32IndexForRhi ? sizeof(quint32) : sizeof(quint16);
 }
 
+// "use" here means that both depth test and write is wanted (the latter for
+// opaque batches only). Therefore neither RenderMode2DNoDepthBuffer nor
+// RenderMode3D must result in true. So while RenderMode3D requires a depth
+// buffer, this here must say false. In addition, m_forceNoDepthBuffer is a
+// dynamic override relevant with QSGRenderNode.
+//
 bool Renderer::useDepthBuffer() const
 {
     return !m_forceNoDepthBuffer && m_renderMode == QSGRendererInterface::RenderMode2D;
diff --git a/src/quick/scenegraph/qsgdefaultcontext.cpp b/src/quick/scenegraph/qsgdefaultcontext.cpp
index b36c0de6ef..37674eb19c 100644
--- a/src/quick/scenegraph/qsgdefaultcontext.cpp
+++ b/src/quick/scenegraph/qsgdefaultcontext.cpp
@@ -200,6 +200,8 @@ QSGLayer *QSGDefaultContext::createLayer(QSGRenderContext *renderContext)
 QSurfaceFormat QSGDefaultContext::defaultSurfaceFormat() const
 {
     QSurfaceFormat format = QSurfaceFormat::defaultFormat();
+    // These depend solely on the env.vars., not QQuickGraphicsConfiguration
+    // since that does not have a flag that maps 100% to QSG_NO_xx_BUFFER.
     static bool useDepth = qEnvironmentVariableIsEmpty("QSG_NO_DEPTH_BUFFER");
     static bool useStencil = qEnvironmentVariableIsEmpty("QSG_NO_STENCIL_BUFFER");
     static bool enableDebug = qEnvironmentVariableIsSet("QSG_OPENGL_DEBUG");
diff --git a/src/quick/scenegraph/qsgrenderloop.cpp b/src/quick/scenegraph/qsgrenderloop.cpp
index 28c9f3dac6..f7cbdd6292 100644
--- a/src/quick/scenegraph/qsgrenderloop.cpp
+++ b/src/quick/scenegraph/qsgrenderloop.cpp
@@ -533,12 +533,15 @@ void QSGGuiThreadRenderLoop::renderWindow(QQuickWindow *window)
             flags |= QRhiSwapChain::SurfaceHasPreMulAlpha;
 
         cd->swapchain = rhi->newSwapChain();
-        cd->depthStencilForSwapchain = rhi->newRenderBuffer(QRhiRenderBuffer::DepthStencil,
-                                                            QSize(),
-                                                            rhiSampleCount,
-                                                            QRhiRenderBuffer::UsedWithSwapChainOnly);
+        static bool depthBufferEnabled = qEnvironmentVariableIsEmpty("QSG_NO_DEPTH_BUFFER");
+        if (depthBufferEnabled) {
+            cd->depthStencilForSwapchain = rhi->newRenderBuffer(QRhiRenderBuffer::DepthStencil,
+                                                                QSize(),
+                                                                rhiSampleCount,
+                                                                QRhiRenderBuffer::UsedWithSwapChainOnly);
+            cd->swapchain->setDepthStencil(cd->depthStencilForSwapchain);
+        }
         cd->swapchain->setWindow(window);
-        cd->swapchain->setDepthStencil(cd->depthStencilForSwapchain);
         qCDebug(QSG_LOG_INFO, "MSAA sample count for the swapchain is %d. Alpha channel requested = %s",
                 rhiSampleCount, alpha ? "yes" : "no");
         cd->swapchain->setSampleCount(rhiSampleCount);
diff --git a/src/quick/scenegraph/qsgrhilayer.cpp b/src/quick/scenegraph/qsgrhilayer.cpp
index 4c9e82bafd..cef9737fd8 100644
--- a/src/quick/scenegraph/qsgrhilayer.cpp
+++ b/src/quick/scenegraph/qsgrhilayer.cpp
@@ -264,6 +264,15 @@ void QSGRhiLayer::grab()
         if (m_mipmap)
             textureFlags |= QRhiTexture::MipMapped | QRhiTexture::UsedWithGenerateMips;
 
+        // Not the same as m_context->useDepthBufferFor2D(), only the env.var
+        // is to be checked here. Consider a layer with a non-offscreen View3D
+        // in it. That still needs a depth buffer, even when the 2D content
+        // renders without relying on it (i.e. RenderMode2DNoDepthBuffer does
+        // not imply not having a depth/stencil attachment for the render
+        // target! The env.var serves as a hard switch, on the other hand, and
+        // that will likely break 3D for instance but that's fine)
+        static bool depthBufferEnabled = qEnvironmentVariableIsEmpty("QSG_NO_DEPTH_BUFFER");
+
         if (m_multisampling) {
             releaseResources();
             m_msaaColorBuffer = m_rhi->newRenderBuffer(QRhiRenderBuffer::Color, m_size, effectiveSamples);
@@ -279,17 +288,20 @@ void QSGRhiLayer::grab()
                 releaseResources();
                 return;
             }
-            m_ds = m_rhi->newRenderBuffer(QRhiRenderBuffer::DepthStencil, m_size, effectiveSamples);
-            if (!m_ds->create()) {
-                qWarning("Failed to build depth-stencil buffer for layer");
-                releaseResources();
-                return;
+            if (depthBufferEnabled) {
+                m_ds = m_rhi->newRenderBuffer(QRhiRenderBuffer::DepthStencil, m_size, effectiveSamples);
+                if (!m_ds->create()) {
+                    qWarning("Failed to build depth-stencil buffer for layer");
+                    releaseResources();
+                    return;
+                }
             }
             QRhiTextureRenderTargetDescription desc;
             QRhiColorAttachment color0(m_msaaColorBuffer);
             color0.setResolveTexture(m_texture);
             desc.setColorAttachments({ color0 });
-            desc.setDepthStencilBuffer(m_ds);
+            if (depthBufferEnabled)
+                desc.setDepthStencilBuffer(m_ds);
             m_rt = m_rhi->newTextureRenderTarget(desc);
             m_rtRp = m_rt->newCompatibleRenderPassDescriptor();
             if (!m_rtRp) {
@@ -311,11 +323,13 @@ void QSGRhiLayer::grab()
                 releaseResources();
                 return;
             }
-            m_ds = m_rhi->newRenderBuffer(QRhiRenderBuffer::DepthStencil, m_size);
-            if (!m_ds->create()) {
-                qWarning("Failed to build depth-stencil buffer for layer");
-                releaseResources();
-                return;
+            if (depthBufferEnabled) {
+                m_ds = m_rhi->newRenderBuffer(QRhiRenderBuffer::DepthStencil, m_size);
+                if (!m_ds->create()) {
+                    qWarning("Failed to build depth-stencil buffer for layer");
+                    releaseResources();
+                    return;
+                }
             }
             QRhiColorAttachment color0(m_texture);
             if (m_recursive) {
@@ -329,7 +343,10 @@ void QSGRhiLayer::grab()
                 }
                 color0.setTexture(m_secondaryTexture);
             }
-            m_rt = m_rhi->newTextureRenderTarget({ color0, m_ds });
+            if (depthBufferEnabled)
+                m_rt = m_rhi->newTextureRenderTarget({ color0, m_ds });
+            else
+                m_rt = m_rhi->newTextureRenderTarget({ color0 });
             m_rtRp = m_rt->newCompatibleRenderPassDescriptor();
             if (!m_rtRp) {
                 qWarning("Failed to build render pass descriptor for layer");
diff --git a/src/quick/scenegraph/qsgthreadedrenderloop.cpp b/src/quick/scenegraph/qsgthreadedrenderloop.cpp
index d01ff15cb3..61ec505ca1 100644
--- a/src/quick/scenegraph/qsgthreadedrenderloop.cpp
+++ b/src/quick/scenegraph/qsgthreadedrenderloop.cpp
@@ -932,12 +932,15 @@ void QSGRenderThread::ensureRhi()
         if (alpha)
             flags |= QRhiSwapChain::SurfaceHasPreMulAlpha;
         cd->swapchain = rhi->newSwapChain();
-        cd->depthStencilForSwapchain = rhi->newRenderBuffer(QRhiRenderBuffer::DepthStencil,
-                                                            QSize(),
-                                                            rhiSampleCount,
-                                                            QRhiRenderBuffer::UsedWithSwapChainOnly);
+        static bool depthBufferEnabled = qEnvironmentVariableIsEmpty("QSG_NO_DEPTH_BUFFER");
+        if (depthBufferEnabled) {
+            cd->depthStencilForSwapchain = rhi->newRenderBuffer(QRhiRenderBuffer::DepthStencil,
+                                                                QSize(),
+                                                                rhiSampleCount,
+                                                                QRhiRenderBuffer::UsedWithSwapChainOnly);
+            cd->swapchain->setDepthStencil(cd->depthStencilForSwapchain);
+        }
         cd->swapchain->setWindow(window);
-        cd->swapchain->setDepthStencil(cd->depthStencilForSwapchain);
         qCDebug(QSG_LOG_INFO, "MSAA sample count for the swapchain is %d. Alpha channel requested = %s.",
                 rhiSampleCount, alpha ? "yes" : "no");
         cd->swapchain->setSampleCount(rhiSampleCount);