Allow subpresentations as textures for custom materials and effects

Effects already supported taking the subpresentation as a texture in
the runtime. That behavior is now copied by the custom materials.

This also fixes a runtime bug where the subpresentation was rendered
only if a default material already had the subpresentation as a texture.

Now custom material and effect properties are iterated prior rendering
If they contain subpresentations, those are added the to the render task
list.

Task-number: QT3DS-3165
Change-Id: I7bddec2fece442065d517320d0e381eba5f1b159
Reviewed-by: Mahmoud Badri <mahmoud.badri@qt.io>
Reviewed-by: Miikka Heikkinen <miikka.heikkinen@qt.io>

6 files changed, 105 insertions, 21 deletions

diff --git a/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp b/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
index 3841dd07..a723c6e0 100644
--- a/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
+++ b/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
@@ -53,6 +53,7 @@
 #include "rendererimpl/Qt3DSRendererImplLayerRenderData.h"
 #include "Qt3DSRenderCustomMaterialShaderGenerator.h"
 #include "Qt3DSRenderModel.h"
+#include "Qt3DSOffscreenRenderKey.h"
 
 using namespace qt3ds::render;
 using namespace qt3ds::render::dynamic;
@@ -1023,6 +1024,17 @@ struct SMaterialSystem : public ICustomMaterialSystem
         theTextureEntry->Set(inPropDec);
     }
 
+    void SetSubpresentation(NVRenderShaderProgram &inShader, CRegisteredString inPropName,
+                            NVRenderTexture2D *inTexture,
+                            const SPropertyDefinition *inPropDec)
+    {
+        SPropertyDefinition propDef = *inPropDec;
+        propDef.m_MinFilterOp = NVRenderTextureMinifyingOp::Linear;
+        propDef.m_MagFilterOp = NVRenderTextureMagnifyingOp::Linear;
+        SCustomMaterialTextureData::CreateTextureEntry(inShader, inTexture, inPropName, false)
+                .Set(&propDef);
+    }
+
     void SetPropertyEnumNames(CRegisteredString inName, CRegisteredString inPropName,
                                       NVConstDataRef<CRegisteredString> inNames) override
     {
@@ -1248,15 +1260,27 @@ struct SMaterialSystem : public ICustomMaterialSystem
                                  == sizeof(NVRenderTexture2DPtr)>::valid_expression();
                     CRegisteredString *theStrPtr = reinterpret_cast<CRegisteredString *>(inDataPtr);
                     IBufferManager &theBufferManager(m_Context->GetBufferManager());
-                    NVRenderTexture2D *theTexture = NULL;
+                    IOffscreenRenderManager &theOffscreenRenderer(
+                        m_Context->GetOffscreenRenderManager());
+                    NVRenderTexture2D *theTexture = nullptr;
 
                     if (theStrPtr->IsValid()) {
-                        SImageTextureData theTextureData =
-                            theBufferManager.LoadRenderImage(*theStrPtr);
-                        if (theTextureData.m_Texture) {
+                        if (theOffscreenRenderer.HasOffscreenRenderer(*theStrPtr)) {
+                            SOffscreenRenderResult theResult
+                                    = theOffscreenRenderer.GetRenderedItem(*theStrPtr);
+                            theTexture = theResult.m_Texture;
+                            if (theTexture) {
+                                SetSubpresentation(inShader, inPropertyName, theTexture,
+                                                   &inDefinition);
+                            }
+                        } else {
+                            SImageTextureData theTextureData
+                                    = theBufferManager.LoadRenderImage(*theStrPtr);
                             theTexture = theTextureData.m_Texture;
-                            SetTexture(inShader, inPropertyName, theTexture, &inDefinition,
-                                       TextureNeedsMips(&inDefinition, theTexture));
+                            if (theTexture) {
+                                SetTexture(inShader, inPropertyName, theTexture, &inDefinition,
+                                           TextureNeedsMips(&inDefinition, theTexture));
+                            }
                         }
                     }
                 } else {
@@ -1617,6 +1641,15 @@ struct SMaterialSystem : public ICustomMaterialSystem
                     NVRenderFrameBuffer *inFrameBuffer, bool inRenderTargetNeedsClear,
                     NVRenderInputAssembler &inAssembler, QT3DSU32 inCount, QT3DSU32 inOffset)
     {
+        ICustomMaterialShaderGenerator &theMaterialGenerator(
+            m_Context->GetCustomMaterialShaderGenerator());
+
+        theMaterialGenerator.SetMaterialProperties(
+            *inShader.m_Shader, inRenderContext.m_Material, QT3DSVec2(1.0, 1.0),
+            inRenderContext.m_ModelViewProjection, inRenderContext.m_NormalMatrix,
+            inRenderContext.m_ModelMatrix, inRenderContext.m_FirstImage, inRenderContext.m_Opacity,
+            GetLayerGlobalRenderProperties(inRenderContext));
+
         NVRenderContext &theContext(m_Context->GetRenderContext());
         theContext.SetRenderTarget(inFrameBuffer);
 
@@ -1628,15 +1661,6 @@ struct SMaterialSystem : public ICustomMaterialSystem
             theContext.Clear(qt3ds::render::NVRenderClearValues::Color);
         }
 
-        ICustomMaterialShaderGenerator &theMaterialGenerator(
-            m_Context->GetCustomMaterialShaderGenerator());
-
-        theMaterialGenerator.SetMaterialProperties(
-            *inShader.m_Shader, inRenderContext.m_Material, QT3DSVec2(1.0, 1.0),
-            inRenderContext.m_ModelViewProjection, inRenderContext.m_NormalMatrix,
-            inRenderContext.m_ModelMatrix, inRenderContext.m_FirstImage, inRenderContext.m_Opacity,
-            GetLayerGlobalRenderProperties(inRenderContext));
-
         // I think the prim type should always be fetched from the
         // current mesh subset setup because there you get the actual draw mode
         // for this frame
@@ -1809,6 +1833,31 @@ struct SMaterialSystem : public ICustomMaterialSystem
                            applier);
     }
 
+    void renderSubpresentations(SCustomMaterial &inMaterial) override
+    {
+        SMaterialClass *theClass = GetMaterialClass(inMaterial.m_ClassName);
+        if (!theClass)
+            return;
+
+        NVConstDataRef<SPropertyDefinition> theDefs = theClass->m_Class->GetProperties();
+        for (QT3DSU32 idx = 0, end = theDefs.size(); idx < end; ++idx) {
+            const SPropertyDefinition &theDefinition(theDefs[idx]);
+            if (theDefinition.m_DataType == NVRenderShaderDataTypes::NVRenderTexture2DPtr) {
+                QT3DSU8 *dataPtr = inMaterial.GetDataSectionBegin() + theDefinition.m_Offset;
+                StaticAssert<sizeof(CRegisteredString)
+                             == sizeof(NVRenderTexture2DPtr)>::valid_expression();
+                CRegisteredString *theStrPtr = reinterpret_cast<CRegisteredString *>(dataPtr);
+                IOffscreenRenderManager &theOffscreenRenderer(
+                    m_Context->GetOffscreenRenderManager());
+
+                if (theStrPtr->IsValid()) {
+                    if (theOffscreenRenderer.HasOffscreenRenderer(*theStrPtr))
+                        theOffscreenRenderer.GetRenderedItem(*theStrPtr);
+                }
+            }
+        }
+    }
+
     virtual void PrepareTextureForRender(SMaterialClass &inClass, SCustomMaterial &inMaterial)
     {
         NVConstDataRef<SPropertyDefinition> thePropDefs = inClass.m_Class->GetProperties();
diff --git a/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.h b/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.h
index cb2a5f7d..dbe390bf 100644
--- a/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.h
+++ b/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.h
@@ -125,6 +125,7 @@ namespace render {
         // apply property values
         virtual void ApplyShaderPropertyValues(const SCustomMaterial &inMaterial,
                                                NVRenderShaderProgram &inProgram) = 0;
+        virtual void renderSubpresentations(SCustomMaterial &inMaterial) = 0;
         // Called by the uiccontext so this system can clear any per-frame render information.
         virtual void EndFrame() = 0;
     };
diff --git a/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.cpp b/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.cpp
index d86c100d..b906241c 100644
--- a/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.cpp
+++ b/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.cpp
@@ -1047,16 +1047,16 @@ struct SEffectSystem : public IEffectSystem
                     IOffscreenRenderManager &theOffscreenRenderer(
                         m_Context->GetOffscreenRenderManager());
                     bool needsAlphaMultiply = true;
-                    NVRenderTexture2D *theTexture = NULL;
+                    NVRenderTexture2D *theTexture = nullptr;
                     if (theStrPtr->IsValid()) {
                         if (theOffscreenRenderer.HasOffscreenRenderer(*theStrPtr)) {
-                            SOffscreenRenderResult theResult =
-                                theOffscreenRenderer.GetRenderedItem(*theStrPtr);
+                            SOffscreenRenderResult theResult
+                                    = theOffscreenRenderer.GetRenderedItem(*theStrPtr);
                             needsAlphaMultiply = false;
                             theTexture = theResult.m_Texture;
                         } else {
-                            SImageTextureData theTextureData =
-                                theBufferManager.LoadRenderImage(*theStrPtr);
+                            SImageTextureData theTextureData
+                                    = theBufferManager.LoadRenderImage(*theStrPtr);
                             needsAlphaMultiply = true;
                             theTexture = theTextureData.m_Texture;
                         }
@@ -1838,6 +1838,31 @@ struct SEffectSystem : public IEffectSystem
     {
         return *m_ResourceManager;
     }
+
+    void renderSubpresentations(SEffect &inEffect) override
+    {
+        SEffectClass *theClass = GetEffectClass(inEffect.m_ClassName);
+        if (!theClass)
+            return;
+
+        NVConstDataRef<SPropertyDefinition> theDefs = theClass->m_DynamicClass->GetProperties();
+        for (QT3DSU32 idx = 0, end = theDefs.size(); idx < end; ++idx) {
+            const SPropertyDefinition &theDefinition(theDefs[idx]);
+            if (theDefinition.m_DataType == NVRenderShaderDataTypes::NVRenderTexture2DPtr) {
+                QT3DSU8 *dataPtr = inEffect.GetDataSectionBegin() + theDefinition.m_Offset;
+                StaticAssert<sizeof(CRegisteredString)
+                             == sizeof(NVRenderTexture2DPtr)>::valid_expression();
+                CRegisteredString *theStrPtr = reinterpret_cast<CRegisteredString *>(dataPtr);
+                IOffscreenRenderManager &theOffscreenRenderer(
+                    m_Context->GetOffscreenRenderManager());
+
+                if (theStrPtr->IsValid()) {
+                    if (theOffscreenRenderer.HasOffscreenRenderer(*theStrPtr))
+                        theOffscreenRenderer.GetRenderedItem(*theStrPtr);
+                }
+            }
+        }
+    }
 };
 }
 
diff --git a/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.h b/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.h
index 119564da..879addc9 100644
--- a/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.h
+++ b/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.h
@@ -201,6 +201,8 @@ namespace render {
         // enabling blending when rendering to the target
         virtual bool RenderEffect(SEffectRenderArgument inRenderArgument, QT3DSMat44 &inMVP,
                                   bool inEnableBlendWhenRenderToTarget) = 0;
+
+        virtual void renderSubpresentations(SEffect &inEffect) = 0;
     };
 }
 }
diff --git a/src/Runtime/Source/runtimerender/Qt3DSRenderRenderList.h b/src/Runtime/Source/runtimerender/Qt3DSRenderRenderList.h
index dce9a95a..c498c7ee 100644
--- a/src/Runtime/Source/runtimerender/Qt3DSRenderRenderList.h
+++ b/src/Runtime/Source/runtimerender/Qt3DSRenderRenderList.h
@@ -118,4 +118,4 @@ namespace render {
 }
 }
 
-#endif
\ No newline at end of file
+#endif
diff --git a/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp b/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
index e1086304..a6e6b14f 100644
--- a/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
+++ b/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
@@ -795,6 +795,10 @@ namespace render {
         retval.m_Opacity = inOpacity;
         QT3DSF32 &subsetOpacity(retval.m_Opacity);
 
+        // If the custom material uses subpresentations, those have to be rendered before
+        // the custom material itself
+        m_Renderer.GetQt3DSContext().GetCustomMaterialSystem().renderSubpresentations(inMaterial);
+
         // set wireframe mode
         m_Renderer.DefaultMaterialShaderKeyProperties().m_WireframeMode.SetValue(
             theGeneratedKey, m_Renderer.GetQt3DSContext().GetWireframeMode());
@@ -1170,6 +1174,9 @@ namespace render {
                     theEffect->m_Flags.SetDirty(false);
                 }
                 if (theEffect->m_Flags.IsActive()) {
+                    // If the effect uses subpresentations, those have to be rendered before
+                    // the effect itself
+                    theEffectSystem.renderSubpresentations(*theEffect);
                     theLastEffect = theEffect;
                     if (hasOffscreenRenderer == false
                         && theEffectSystem.DoesEffectRequireDepthTexture(theEffect->m_ClassName))