Fix flashing and crashing of scene2d

Prevent simultanious usage of the GLTexture being rendered to by scene2d.

Task-number: QTBUG-68511
Task-number: QT3DS-1792
Change-Id: I88de12cba68ef7af2c7afb1f6e9d6143028efc86
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

4 files changed, 60 insertions, 3 deletions

diff --git a/src/render/backend/resourceaccessor.cpp b/src/render/backend/resourceaccessor.cpp
index 7558eb4ad..8ca11e7a7 100644
--- a/src/render/backend/resourceaccessor.cpp
+++ b/src/render/backend/resourceaccessor.cpp
@@ -85,9 +85,10 @@ bool ResourceAccessor::accessResource(ResourceType type, Qt3DCore::QNodeId nodeI
         if (glTex->isDirty())
             return false;
 
+        glTex->setExternalRenderingEnabled(true);
         QOpenGLTexture **glTextureHandle = reinterpret_cast<QOpenGLTexture **>(handle);
         *glTextureHandle = glTex->getOrCreateGLTexture();
-        *lock = glTex->textureLock();
+        *lock = glTex->externalRenderingLock();
         return true;
     }
 
diff --git a/src/render/renderers/opengl/graphicshelpers/submissioncontext.cpp b/src/render/renderers/opengl/graphicshelpers/submissioncontext.cpp
index 4ec929b6e..ecaa12d8b 100644
--- a/src/render/renderers/opengl/graphicshelpers/submissioncontext.cpp
+++ b/src/render/renderers/opengl/graphicshelpers/submissioncontext.cpp
@@ -93,6 +93,40 @@ QT_BEGIN_NAMESPACE
 namespace Qt3DRender {
 namespace Render {
 
+
+class TextureExtRendererLocker
+{
+public:
+    static void lock(GLTexture *tex)
+    {
+        if (!tex->isExternalRenderingEnabled())
+            return;
+        if (s_lockHash.keys().contains(tex)) {
+            ++s_lockHash[tex];
+        } else {
+            tex->externalRenderingLock()->lock();
+            s_lockHash[tex] = 1;
+        }
+    }
+    static void unlock(GLTexture *tex)
+    {
+        if (!tex->isExternalRenderingEnabled())
+            return;
+        if (!s_lockHash.keys().contains(tex))
+            return;
+
+        --s_lockHash[tex];
+        if (s_lockHash[tex] == 0) {
+            s_lockHash.remove(tex);
+            tex->externalRenderingLock()->unlock();
+        }
+    }
+private:
+    static QHash<GLTexture*, int> s_lockHash;
+};
+
+QHash<GLTexture*, int> TextureExtRendererLocker::s_lockHash = QHash<GLTexture*, int>();
+
 static QHash<unsigned int, SubmissionContext*> static_contexts;
 
 namespace {
@@ -475,6 +509,9 @@ void SubmissionContext::endDrawing(bool swapBuffers)
     if (m_ownCurrent)
         m_gl->doneCurrent();
     decayTextureScores();
+    for (int i = 0; i < m_activeTextures.size(); ++i)
+        if (m_activeTextures[i].texture)
+            TextureExtRendererLocker::unlock(m_activeTextures[i].texture);
 }
 
 void SubmissionContext::activateRenderTarget(Qt3DCore::QNodeId renderTargetNodeId, const AttachmentPack &attachments, GLuint defaultFboId)
@@ -827,6 +864,8 @@ void SubmissionContext::bindFrameBufferAttachmentHelper(GLuint fboId, const Atta
         if (!m_glHelper->frameBufferNeedsRenderBuffer(attachment)) {
             QOpenGLTexture *glTex = rTex ? rTex->getOrCreateGLTexture() : nullptr;
             if (glTex != nullptr) {
+                // The texture can not be rendered simultaniously by another renderer
+                Q_ASSERT(!rTex->isExternalRenderingEnabled());
                 if (fboSize.isEmpty())
                     fboSize = QSize(glTex->width(), glTex->height());
                 else
@@ -892,7 +931,10 @@ int SubmissionContext::activateTexture(TextureScope scope, GLTexture *tex, int o
         if (glTex == nullptr)
             return -1;
         glTex->bind(onUnit);
+        if (m_activeTextures[onUnit].texture)
+            TextureExtRendererLocker::unlock(m_activeTextures[onUnit].texture);
         m_activeTextures[onUnit].texture = tex;
+        TextureExtRendererLocker::lock(tex);
     }
 
 #if defined(QT3D_RENDER_ASPECT_OPENGL_DEBUG)
diff --git a/src/render/renderers/opengl/textures/gltexture.cpp b/src/render/renderers/opengl/textures/gltexture.cpp
index c98e95759..4286a69b6 100644
--- a/src/render/renderers/opengl/textures/gltexture.cpp
+++ b/src/render/renderers/opengl/textures/gltexture.cpp
@@ -72,6 +72,7 @@ GLTexture::GLTexture(TextureDataManager *texDataMgr,
     , m_textureDataManager(texDataMgr)
     , m_textureImageDataManager(texImgDataMgr)
     , m_dataFunctor(texGen)
+    , m_externalRendering(false)
 {
     // make sure texture generator is executed
     // this is needed when Texture have the TargetAutomatic
diff --git a/src/render/renderers/opengl/textures/gltexture_p.h b/src/render/renderers/opengl/textures/gltexture_p.h
index cde0a6973..5f4a20d30 100644
--- a/src/render/renderers/opengl/textures/gltexture_p.h
+++ b/src/render/renderers/opengl/textures/gltexture_p.h
@@ -166,9 +166,19 @@ public:
         return m_dirtyFlags.load() != 0;
     }
 
-    QMutex *textureLock()
+    QMutex *externalRenderingLock()
     {
-        return &m_textureMutex;
+        return &m_externalRenderingMutex;
+    }
+
+    void setExternalRenderingEnabled(bool enable)
+    {
+        m_externalRendering = enable;
+    }
+
+    bool isExternalRenderingEnabled() const
+    {
+        return m_externalRendering;
     }
 
 protected:
@@ -218,6 +228,7 @@ private:
     bool m_unique;
     QAtomicInt m_dirtyFlags;
     QMutex m_textureMutex;
+    QMutex m_externalRenderingMutex;
     QOpenGLTexture *m_gl;
     RenderBuffer *m_renderBuffer;
 
@@ -235,6 +246,8 @@ private:
     // cache actual image data generated by the functors
     QTextureDataPtr m_textureData;
     QVector<QTextureImageDataPtr> m_imageData;
+
+    bool m_externalRendering;
 };
 
 } // namespace Render