Avoid stale renderpass references in ShaderManager's pipeline list

A similar problem popped up for Quick3D before, and it seems the same
applies to Qt Quick layers as well: these can get released before the
(per-rendercontext) resource caches, thus leaving invalid references in
the cache keys.

Solve this by simply notifying the renderer to clean up the relevant
cache entries. Also utilize a very handy QRhi convenience, which allows
us to schedule and defer the destruction of a QRhiResource to endFrame()
(i.e. until after the current frame is submitted) with a single function
call.

Change-Id: I6db4914555f9504a2599e6ce1b0dc03b2b4b74de
Reviewed-by: Christian Strømme <christian.stromme@qt.io>

4 files changed, 25 insertions, 2 deletions

diff --git a/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp b/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp
index c18f4f6a2e..ed790fdc5c 100644
--- a/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp
+++ b/src/quick/scenegraph/coreapi/qsgbatchrenderer.cpp
@@ -4615,6 +4615,22 @@ bool Renderer::hasCustomRenderModeWithContinuousUpdate() const
     return m_visualizer->mode() == Visualizer::VisualizeOverdraw;
 }
 
+void Renderer::invalidatePipelineCacheDependency(QRhiRenderPassDescriptor *rpDesc)
+{
+    if (!rpDesc)
+        return;
+
+    for (auto it = m_shaderManager->pipelineCache.begin(); it != m_shaderManager->pipelineCache.end(); ) {
+        if (it.key().compatibleRenderPassDescriptor == rpDesc) {
+            QRhiGraphicsPipeline *ps = it.value();
+            it = m_shaderManager->pipelineCache.erase(it);
+            ps->releaseAndDestroyLater(); // QRhi takes care of it in endFrame()
+        } else {
+            ++it;
+        }
+    }
+}
+
 bool operator==(const GraphicsState &a, const GraphicsState &b) Q_DECL_NOTHROW
 {
     return a.depthTest == b.depthTest
diff --git a/src/quick/scenegraph/coreapi/qsgbatchrenderer_p.h b/src/quick/scenegraph/coreapi/qsgbatchrenderer_p.h
index da58b98a72..3db93bee41 100644
--- a/src/quick/scenegraph/coreapi/qsgbatchrenderer_p.h
+++ b/src/quick/scenegraph/coreapi/qsgbatchrenderer_p.h
@@ -847,6 +847,8 @@ private:
     void setCustomRenderMode(const QByteArray &mode) override;
     bool hasCustomRenderModeWithContinuousUpdate() const override;
 
+    void invalidatePipelineCacheDependency(QRhiRenderPassDescriptor *rpDesc) override;
+
     QSGDefaultRenderContext *m_context;
     QSet<Node *> m_taggedRoots;
     QDataBuffer<Element *> m_opaqueRenderList;
diff --git a/src/quick/scenegraph/coreapi/qsgrenderer_p.h b/src/quick/scenegraph/coreapi/qsgrenderer_p.h
index b25ecb0ae6..3f2154e3c1 100644
--- a/src/quick/scenegraph/coreapi/qsgrenderer_p.h
+++ b/src/quick/scenegraph/coreapi/qsgrenderer_p.h
@@ -98,6 +98,7 @@ public:
     virtual void setCustomRenderMode(const QByteArray &) { }
     virtual bool hasCustomRenderModeWithContinuousUpdate() const { return false; }
     virtual void releaseCachedResources() { }
+    virtual void invalidatePipelineCacheDependency(QRhiRenderPassDescriptor *) { }
 
     void clearChangedFlag() { m_changed_emitted = false; }
 
diff --git a/src/quick/scenegraph/qsgrhilayer.cpp b/src/quick/scenegraph/qsgrhilayer.cpp
index 5772ff3a3d..003ffaf5ea 100644
--- a/src/quick/scenegraph/qsgrhilayer.cpp
+++ b/src/quick/scenegraph/qsgrhilayer.cpp
@@ -225,8 +225,12 @@ void QSGRhiLayer::releaseResources()
     delete m_rt;
     m_rt = nullptr;
 
-    delete m_rtRp;
-    m_rtRp = nullptr;
+    if (m_rtRp) {
+        if (m_renderer)
+            m_renderer->invalidatePipelineCacheDependency(m_rtRp);
+        delete m_rtRp;
+        m_rtRp = nullptr;
+    }
 
     delete m_ds;
     m_ds = nullptr;