Fix subpresentations in custom shaders

Custom shaders don't have presentationIds separately. The property value
has to be interpreted as a presentationId. The main layer also has to be
set dirty if custom shader subpresentation changes so that it animates
even without other changes in the scene.

Task-number: QT3DS-4083
Change-Id: I0fef13a8c7d99a74c524e459bb5490ff48158e26
Reviewed-by: Mahmoud Badri <mahmoud.badri@qt.io>
Reviewed-by: Tomi Korpipää <tomi.korpipaa@qt.io>

4 files changed, 29 insertions, 13 deletions

diff --git a/src/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp b/src/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
index 593687b..5f54b4c 100644
--- a/src/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
+++ b/src/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
@@ -1310,10 +1310,15 @@ struct SMaterialSystem : public ICustomMaterialSystem
                                 image->m_ImagePath = *theStrPtr;
                                 image->m_Flags.SetDirty(true);
                             } else {
-                                if (image->m_OffscreenRendererId.IsValid()) {
-                                    SetSubpresentation(inShader, inPropertyName,
-                                                       image->m_TextureData.m_Texture,
-                                                       &inDefinition);
+                                IOffscreenRenderManager &offscreenRenderer(
+                                    m_Context->GetOffscreenRenderManager());
+                                if (offscreenRenderer.HasOffscreenRenderer(*theStrPtr)) {
+                                    SOffscreenRenderResult result
+                                            = offscreenRenderer.GetRenderedItem(*theStrPtr);
+                                    if (result.m_Texture) {
+                                        SetSubpresentation(inShader, inPropertyName,
+                                                           result.m_Texture, &inDefinition);
+                                    }
                                 } else {
                                     SetTexture(inShader, inPropertyName,
                                                image->m_TextureData.m_Texture, &inDefinition,
@@ -1925,12 +1930,13 @@ struct SMaterialSystem : public ICustomMaterialSystem
                            applier);
     }
 
-    void renderSubpresentations(SCustomMaterial &inMaterial) override
+    bool renderSubpresentations(SCustomMaterial &inMaterial) override
     {
         SMaterialClass *theClass = GetMaterialClass(inMaterial.m_ClassName);
         if (!theClass)
-            return;
+            return false;
 
+        bool wasDirty = false;
         NVConstDataRef<SPropertyDefinition> theDefs = theClass->m_Class->GetProperties();
         for (QT3DSU32 idx = 0, end = theDefs.size(); idx < end; ++idx) {
             const SPropertyDefinition &theDefinition(theDefs[idx]);
@@ -1943,11 +1949,16 @@ struct SMaterialSystem : public ICustomMaterialSystem
                     m_Context->GetOffscreenRenderManager());
 
                 if (theStrPtr->IsValid()) {
-                    if (theOffscreenRenderer.HasOffscreenRenderer(*theStrPtr))
-                        theOffscreenRenderer.GetRenderedItem(*theStrPtr);
+                    if (theOffscreenRenderer.HasOffscreenRenderer(*theStrPtr)) {
+                        SOffscreenRenderResult result
+                                = theOffscreenRenderer.GetRenderedItem(*theStrPtr);
+                        if (result.m_HasChangedSinceLastFrame)
+                            wasDirty = true;
+                    }
                 }
             }
         }
+        return wasDirty;
     }
 
     void clearCaches() override
diff --git a/src/runtimerender/Qt3DSRenderCustomMaterialSystem.h b/src/runtimerender/Qt3DSRenderCustomMaterialSystem.h
index 0f87533..9a46554 100644
--- a/src/runtimerender/Qt3DSRenderCustomMaterialSystem.h
+++ b/src/runtimerender/Qt3DSRenderCustomMaterialSystem.h
@@ -127,7 +127,7 @@ namespace render {
         // apply property values
         virtual void ApplyShaderPropertyValues(const SCustomMaterial &inMaterial,
                                                NVRenderShaderProgram &inProgram) = 0;
-        virtual void renderSubpresentations(SCustomMaterial &inMaterial) = 0;
+        virtual bool renderSubpresentations(SCustomMaterial &inMaterial) = 0;
         virtual void clearCaches() = 0;
         virtual void setRequiresCompilation(const CRegisteredString &name, bool value) = 0;
         virtual bool requiresCompilation(const CRegisteredString &name) const = 0;
diff --git a/src/runtimerender/Qt3DSRenderEffectSystem.cpp b/src/runtimerender/Qt3DSRenderEffectSystem.cpp
index b10345c..faaf4d2 100644
--- a/src/runtimerender/Qt3DSRenderEffectSystem.cpp
+++ b/src/runtimerender/Qt3DSRenderEffectSystem.cpp
@@ -1082,9 +1082,10 @@ struct SEffectSystem : public IEffectSystem
                                 image->m_ImagePath = *theStrPtr;
                                 image->m_Flags.SetDirty(true);
                             } else {
-                                if (image->m_OffscreenRendererId.IsValid()) {
-                                    IOffscreenRenderManager &theOffscreenRenderer(
-                                                            m_Context->GetOffscreenRenderManager());
+                                IOffscreenRenderManager &theOffscreenRenderer(
+                                                        m_Context->GetOffscreenRenderManager());
+                                if (image->m_OffscreenRendererId.IsValid()
+                                        || theOffscreenRenderer.HasOffscreenRenderer(*theStrPtr)) {
                                     SOffscreenRenderResult theResult
                                             = theOffscreenRenderer.GetRenderedItem(*theStrPtr);
                                     needsAlphaMultiply = false;
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
index eb098b4..26b0a3c 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
@@ -859,7 +859,10 @@ namespace render {
 
         // If the custom material uses subpresentations, those have to be rendered before
         // the custom material itself
-        m_Renderer.GetQt3DSContext().GetCustomMaterialSystem().renderSubpresentations(inMaterial);
+        if (m_Renderer.GetQt3DSContext().GetCustomMaterialSystem()
+                .renderSubpresentations(inMaterial)) {
+            retval.m_Dirty = true;
+        }
 
         // set wireframe mode
         m_Renderer.DefaultMaterialShaderKeyProperties().m_WireframeMode.SetValue(
@@ -1046,6 +1049,7 @@ namespace render {
                                                        subsetOpacity, subsetDirty));
                     SShaderDefaultMaterialKey theGeneratedKey = theMaterialPrepResult.m_MaterialKey;
                     subsetOpacity = theMaterialPrepResult.m_Opacity;
+                    subsetDirty |= theMaterialPrepResult.m_Dirty;
                     SRenderableImage *firstImage(theMaterialPrepResult.m_FirstImage);
                     renderableFlags = theMaterialPrepResult.m_RenderableFlags;