Allow subpresentations as textures for custom materials and effects

Effects already supported taking the subpresentation as a texture in
the runtime. That behavior is now copied by the custom materials.

This also fixes a runtime bug where the subpresentation was rendered
only if a default material already had the subpresentation as a texture.

Now custom material and effect properties are iterated prior rendering
If they contain subpresentations, those are added the to the render task
list.

Task-number: QT3DS-3165
Change-Id: I7bddec2fece442065d517320d0e381eba5f1b159
Reviewed-by: Mahmoud Badri <mahmoud.badri@qt.io>
Reviewed-by: Miikka Heikkinen <miikka.heikkinen@qt.io>

12 files changed, 168 insertions, 34 deletions

diff --git a/src/Authoring/Studio/Application/StudioApp.cpp b/src/Authoring/Studio/Application/StudioApp.cpp
index 0d82d4cd..23dd2d73 100644
--- a/src/Authoring/Studio/Application/StudioApp.cpp
+++ b/src/Authoring/Studio/Application/StudioApp.cpp
@@ -1969,9 +1969,9 @@ void CStudioApp::OnPresentationModifiedExternally()
     }
 }
 
-// Get the renderable id for a file path.
+// Converts a renderable path to the format used in the SubPresentationRecord struct
 // filePath can be absolute or relative to either presentation or project
-QString CStudioApp::getRenderableId(const QString &filePath) const
+QString CStudioApp::getRenderablePath(const QString &filePath) const
 {
     QString renderablePath;
     QDir projectDir(m_core->getProjectFile().getProjectPath());
@@ -1991,6 +1991,38 @@ QString CStudioApp::getRenderableId(const QString &filePath) const
         }
         renderablePath = checkFile.mid(index);
     }
+    return renderablePath;
+}
+
+// Get the presentation id, returns an empty string for qml streams
+// filePath can be absolute or relative to either presentation or project
+QString CStudioApp::getPresentationId(const QString &filePath) const
+{
+    QString renderablePath = getRenderablePath(filePath);
+    for (SubPresentationRecord r : qAsConst(m_subpresentations)) {
+        if (r.m_type == QLatin1String("presentation") && r.m_argsOrSrc == renderablePath)
+            return r.m_id;
+    }
+    return {};
+}
+
+// Get the qml stream id, returns an empty string for presentations
+// filePath can be absolute or relative to either presentation or project
+QString CStudioApp::getQmlId(const QString &filePath) const
+{
+    QString renderablePath = getRenderablePath(filePath);
+    for (SubPresentationRecord r : qAsConst(m_subpresentations)) {
+        if (r.m_type == QLatin1String("presentation-qml") && r.m_argsOrSrc == renderablePath)
+            return r.m_id;
+    }
+    return {};
+}
+
+// Get the renderable id for a file path.
+// filePath can be absolute or relative to either presentation or project
+QString CStudioApp::getRenderableId(const QString &filePath) const
+{
+    QString renderablePath = getRenderablePath(filePath);
     for (SubPresentationRecord r : qAsConst(m_subpresentations)) {
         if (r.m_argsOrSrc == renderablePath)
             return r.m_id;
diff --git a/src/Authoring/Studio/Application/StudioApp.h b/src/Authoring/Studio/Application/StudioApp.h
index 04e9f57d..86603af0 100644
--- a/src/Authoring/Studio/Application/StudioApp.h
+++ b/src/Authoring/Studio/Application/StudioApp.h
@@ -131,6 +131,7 @@ protected:
 private:
     void playbackPreviewStart();
     void playbackPreviewEnd();
+    QString getRenderablePath(const QString &filePath) const;
 
     bool m_welcomeShownThisSession;
     // are we are launching welcome screen again due to user canceling file dialog?
@@ -255,6 +256,8 @@ public:
     QVector<SubPresentationRecord> m_subpresentations;
     QMap<QString, CDataInputDialogItem *> m_dataInputDialogItems;
 
+    QString getPresentationId(const QString &filePath) const;
+    QString getQmlId(const QString &filePath) const;
     QString getRenderableId(const QString &filePath) const;
     QString getRenderableAbsolutePath(const QString &renderableId) const;
     QSize getRenderableSize(const QString &renderableId);
diff --git a/src/Authoring/Studio/Palettes/Inspector/ImageChooserModel.cpp b/src/Authoring/Studio/Palettes/Inspector/ImageChooserModel.cpp
index dbb720fb..dd7ed605 100644
--- a/src/Authoring/Studio/Palettes/Inspector/ImageChooserModel.cpp
+++ b/src/Authoring/Studio/Palettes/Inspector/ImageChooserModel.cpp
@@ -31,9 +31,9 @@
 #include "ProjectFile.h"
 #include "Core.h"
 
-ImageChooserModel::ImageChooserModel(bool showRenderables, QObject *parent)
+ImageChooserModel::ImageChooserModel(bool showQmls, QObject *parent)
     : ChooserModelBase(parent)
-    , m_showRenderables(showRenderables)
+    , m_showQmls(showQmls)
 {
     connect(&g_StudioApp.GetCore()->getProjectFile(), &ProjectFile::presentationIdChanged,
             this, &ImageChooserModel::handlePresentationIdChange);
@@ -46,7 +46,8 @@ ImageChooserModel::~ImageChooserModel()
 bool ImageChooserModel::isVisible(const QString &path) const
 {
     return getIconType(path) == OBJTYPE_IMAGE
-            || (m_showRenderables && !g_StudioApp.getRenderableId(path).isEmpty());
+            || !g_StudioApp.getPresentationId(path).isEmpty()
+            || (m_showQmls && !g_StudioApp.getQmlId(path).isEmpty());
 }
 
 const QVector<ChooserModelBase::FixedItem> ImageChooserModel::getFixedItems() const
diff --git a/src/Authoring/Studio/Palettes/Inspector/ImageChooserModel.h b/src/Authoring/Studio/Palettes/Inspector/ImageChooserModel.h
index 7a1c6b5a..ceb34b29 100644
--- a/src/Authoring/Studio/Palettes/Inspector/ImageChooserModel.h
+++ b/src/Authoring/Studio/Palettes/Inspector/ImageChooserModel.h
@@ -36,7 +36,7 @@ class ImageChooserModel : public ChooserModelBase
     Q_OBJECT
 
 public:
-    explicit ImageChooserModel(bool showRenderables, QObject *parent = nullptr);
+    explicit ImageChooserModel(bool showQmls, QObject *parent = nullptr);
     virtual ~ImageChooserModel();
 
 private:
@@ -45,7 +45,7 @@ private:
     QString specialDisplayName(const TreeItem &item) const override;
     void handlePresentationIdChange(const QString &path, const QString &id);
 
-    bool m_showRenderables = false;
+    bool m_showQmls = false;
 };
 
 #endif // IMAGECHOOSERMODEL_H
diff --git a/src/Authoring/Studio/Palettes/Inspector/InspectorControlView.cpp b/src/Authoring/Studio/Palettes/Inspector/InspectorControlView.cpp
index 66f8c868..89e26a69 100644
--- a/src/Authoring/Studio/Palettes/Inspector/InspectorControlView.cpp
+++ b/src/Authoring/Studio/Palettes/Inspector/InspectorControlView.cpp
@@ -686,8 +686,13 @@ QObject *InspectorControlView::showTextureChooser(int handle, int instance, cons
         m_textureChooserView = new TextureChooserView(this);
         connect(m_textureChooserView, &TextureChooserView::textureSelected, this,
                 [this] (int handle, int instance, const QString &fileName) {
-            if (m_textureChooserView->currentDataModelPath() != fileName)
-                setPropertyValueFromFilename(instance, handle, fileName);
+            if (m_textureChooserView->currentDataModelPath() != fileName) {
+                QString renderableId = g_StudioApp.getRenderableId(fileName);
+                if (renderableId.isEmpty())
+                    setPropertyValueFromFilename(instance, handle, fileName);
+                else
+                    m_inspectorControlModel->setPropertyValue(instance, handle, renderableId);
+            }
         });
     }
 
diff --git a/src/Authoring/Studio/Palettes/Inspector/TextureChooserView.cpp b/src/Authoring/Studio/Palettes/Inspector/TextureChooserView.cpp
index a4c2614a..59e8bfac 100644
--- a/src/Authoring/Studio/Palettes/Inspector/TextureChooserView.cpp
+++ b/src/Authoring/Studio/Palettes/Inspector/TextureChooserView.cpp
@@ -92,14 +92,23 @@ QString TextureChooserView::currentDataModelPath() const
     qt3dsdm::SValue value;
     propertySystem->GetInstancePropertyValue(m_instance, m_handle, value);
 
-    const QString currentValue = qt3dsdm::get<QString>(value);
+    const QString path = qt3dsdm::get<QString>(value);
+
     // An empty value can sometimes be represented by a relative path either to project root or the
     // presentation file, such as"./" or "../", so let's just consider all directory paths as empty
-    if (currentValue.isEmpty() || QFileInfo(currentValue).isDir()) {
+    if (path.isEmpty() || QFileInfo(path).isDir()) {
         cleanPath = ChooserModelBase::noneString();
     } else {
-        cleanPath = QDir::cleanPath(QDir(doc->GetDocumentDirectory().toQString())
-                                    .filePath(currentValue));
+        // If path is renderable id, we need to resolve the actual path
+        const QString renderablePath = g_StudioApp.getRenderableAbsolutePath(path);
+
+        if (renderablePath.isEmpty())
+            cleanPath = path;
+        else
+            cleanPath = renderablePath;
+
+        cleanPath = QDir::cleanPath(
+                    QDir(doc->GetDocumentDirectory().toQString()).filePath(cleanPath));
     }
     return cleanPath;
 }
diff --git a/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp b/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
index 3841dd07..a723c6e0 100644
--- a/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
+++ b/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
@@ -53,6 +53,7 @@
 #include "rendererimpl/Qt3DSRendererImplLayerRenderData.h"
 #include "Qt3DSRenderCustomMaterialShaderGenerator.h"
 #include "Qt3DSRenderModel.h"
+#include "Qt3DSOffscreenRenderKey.h"
 
 using namespace qt3ds::render;
 using namespace qt3ds::render::dynamic;
@@ -1023,6 +1024,17 @@ struct SMaterialSystem : public ICustomMaterialSystem
         theTextureEntry->Set(inPropDec);
     }
 
+    void SetSubpresentation(NVRenderShaderProgram &inShader, CRegisteredString inPropName,
+                            NVRenderTexture2D *inTexture,
+                            const SPropertyDefinition *inPropDec)
+    {
+        SPropertyDefinition propDef = *inPropDec;
+        propDef.m_MinFilterOp = NVRenderTextureMinifyingOp::Linear;
+        propDef.m_MagFilterOp = NVRenderTextureMagnifyingOp::Linear;
+        SCustomMaterialTextureData::CreateTextureEntry(inShader, inTexture, inPropName, false)
+                .Set(&propDef);
+    }
+
     void SetPropertyEnumNames(CRegisteredString inName, CRegisteredString inPropName,
                                       NVConstDataRef<CRegisteredString> inNames) override
     {
@@ -1248,15 +1260,27 @@ struct SMaterialSystem : public ICustomMaterialSystem
                                  == sizeof(NVRenderTexture2DPtr)>::valid_expression();
                     CRegisteredString *theStrPtr = reinterpret_cast<CRegisteredString *>(inDataPtr);
                     IBufferManager &theBufferManager(m_Context->GetBufferManager());
-                    NVRenderTexture2D *theTexture = NULL;
+                    IOffscreenRenderManager &theOffscreenRenderer(
+                        m_Context->GetOffscreenRenderManager());
+                    NVRenderTexture2D *theTexture = nullptr;
 
                     if (theStrPtr->IsValid()) {
-                        SImageTextureData theTextureData =
-                            theBufferManager.LoadRenderImage(*theStrPtr);
-                        if (theTextureData.m_Texture) {
+                        if (theOffscreenRenderer.HasOffscreenRenderer(*theStrPtr)) {
+                            SOffscreenRenderResult theResult
+                                    = theOffscreenRenderer.GetRenderedItem(*theStrPtr);
+                            theTexture = theResult.m_Texture;
+                            if (theTexture) {
+                                SetSubpresentation(inShader, inPropertyName, theTexture,
+                                                   &inDefinition);
+                            }
+                        } else {
+                            SImageTextureData theTextureData
+                                    = theBufferManager.LoadRenderImage(*theStrPtr);
                             theTexture = theTextureData.m_Texture;
-                            SetTexture(inShader, inPropertyName, theTexture, &inDefinition,
-                                       TextureNeedsMips(&inDefinition, theTexture));
+                            if (theTexture) {
+                                SetTexture(inShader, inPropertyName, theTexture, &inDefinition,
+                                           TextureNeedsMips(&inDefinition, theTexture));
+                            }
                         }
                     }
                 } else {
@@ -1617,6 +1641,15 @@ struct SMaterialSystem : public ICustomMaterialSystem
                     NVRenderFrameBuffer *inFrameBuffer, bool inRenderTargetNeedsClear,
                     NVRenderInputAssembler &inAssembler, QT3DSU32 inCount, QT3DSU32 inOffset)
     {
+        ICustomMaterialShaderGenerator &theMaterialGenerator(
+            m_Context->GetCustomMaterialShaderGenerator());
+
+        theMaterialGenerator.SetMaterialProperties(
+            *inShader.m_Shader, inRenderContext.m_Material, QT3DSVec2(1.0, 1.0),
+            inRenderContext.m_ModelViewProjection, inRenderContext.m_NormalMatrix,
+            inRenderContext.m_ModelMatrix, inRenderContext.m_FirstImage, inRenderContext.m_Opacity,
+            GetLayerGlobalRenderProperties(inRenderContext));
+
         NVRenderContext &theContext(m_Context->GetRenderContext());
         theContext.SetRenderTarget(inFrameBuffer);
 
@@ -1628,15 +1661,6 @@ struct SMaterialSystem : public ICustomMaterialSystem
             theContext.Clear(qt3ds::render::NVRenderClearValues::Color);
         }
 
-        ICustomMaterialShaderGenerator &theMaterialGenerator(
-            m_Context->GetCustomMaterialShaderGenerator());
-
-        theMaterialGenerator.SetMaterialProperties(
-            *inShader.m_Shader, inRenderContext.m_Material, QT3DSVec2(1.0, 1.0),
-            inRenderContext.m_ModelViewProjection, inRenderContext.m_NormalMatrix,
-            inRenderContext.m_ModelMatrix, inRenderContext.m_FirstImage, inRenderContext.m_Opacity,
-            GetLayerGlobalRenderProperties(inRenderContext));
-
         // I think the prim type should always be fetched from the
         // current mesh subset setup because there you get the actual draw mode
         // for this frame
@@ -1809,6 +1833,31 @@ struct SMaterialSystem : public ICustomMaterialSystem
                            applier);
     }
 
+    void renderSubpresentations(SCustomMaterial &inMaterial) override
+    {
+        SMaterialClass *theClass = GetMaterialClass(inMaterial.m_ClassName);
+        if (!theClass)
+            return;
+
+        NVConstDataRef<SPropertyDefinition> theDefs = theClass->m_Class->GetProperties();
+        for (QT3DSU32 idx = 0, end = theDefs.size(); idx < end; ++idx) {
+            const SPropertyDefinition &theDefinition(theDefs[idx]);
+            if (theDefinition.m_DataType == NVRenderShaderDataTypes::NVRenderTexture2DPtr) {
+                QT3DSU8 *dataPtr = inMaterial.GetDataSectionBegin() + theDefinition.m_Offset;
+                StaticAssert<sizeof(CRegisteredString)
+                             == sizeof(NVRenderTexture2DPtr)>::valid_expression();
+                CRegisteredString *theStrPtr = reinterpret_cast<CRegisteredString *>(dataPtr);
+                IOffscreenRenderManager &theOffscreenRenderer(
+                    m_Context->GetOffscreenRenderManager());
+
+                if (theStrPtr->IsValid()) {
+                    if (theOffscreenRenderer.HasOffscreenRenderer(*theStrPtr))
+                        theOffscreenRenderer.GetRenderedItem(*theStrPtr);
+                }
+            }
+        }
+    }
+
     virtual void PrepareTextureForRender(SMaterialClass &inClass, SCustomMaterial &inMaterial)
     {
         NVConstDataRef<SPropertyDefinition> thePropDefs = inClass.m_Class->GetProperties();
diff --git a/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.h b/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.h
index cb2a5f7d..dbe390bf 100644
--- a/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.h
+++ b/src/Runtime/Source/runtimerender/Qt3DSRenderCustomMaterialSystem.h
@@ -125,6 +125,7 @@ namespace render {
         // apply property values
         virtual void ApplyShaderPropertyValues(const SCustomMaterial &inMaterial,
                                                NVRenderShaderProgram &inProgram) = 0;
+        virtual void renderSubpresentations(SCustomMaterial &inMaterial) = 0;
         // Called by the uiccontext so this system can clear any per-frame render information.
         virtual void EndFrame() = 0;
     };
diff --git a/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.cpp b/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.cpp
index d86c100d..b906241c 100644
--- a/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.cpp
+++ b/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.cpp
@@ -1047,16 +1047,16 @@ struct SEffectSystem : public IEffectSystem
                     IOffscreenRenderManager &theOffscreenRenderer(
                         m_Context->GetOffscreenRenderManager());
                     bool needsAlphaMultiply = true;
-                    NVRenderTexture2D *theTexture = NULL;
+                    NVRenderTexture2D *theTexture = nullptr;
                     if (theStrPtr->IsValid()) {
                         if (theOffscreenRenderer.HasOffscreenRenderer(*theStrPtr)) {
-                            SOffscreenRenderResult theResult =
-                                theOffscreenRenderer.GetRenderedItem(*theStrPtr);
+                            SOffscreenRenderResult theResult
+                                    = theOffscreenRenderer.GetRenderedItem(*theStrPtr);
                             needsAlphaMultiply = false;
                             theTexture = theResult.m_Texture;
                         } else {
-                            SImageTextureData theTextureData =
-                                theBufferManager.LoadRenderImage(*theStrPtr);
+                            SImageTextureData theTextureData
+                                    = theBufferManager.LoadRenderImage(*theStrPtr);
                             needsAlphaMultiply = true;
                             theTexture = theTextureData.m_Texture;
                         }
@@ -1838,6 +1838,31 @@ struct SEffectSystem : public IEffectSystem
     {
         return *m_ResourceManager;
     }
+
+    void renderSubpresentations(SEffect &inEffect) override
+    {
+        SEffectClass *theClass = GetEffectClass(inEffect.m_ClassName);
+        if (!theClass)
+            return;
+
+        NVConstDataRef<SPropertyDefinition> theDefs = theClass->m_DynamicClass->GetProperties();
+        for (QT3DSU32 idx = 0, end = theDefs.size(); idx < end; ++idx) {
+            const SPropertyDefinition &theDefinition(theDefs[idx]);
+            if (theDefinition.m_DataType == NVRenderShaderDataTypes::NVRenderTexture2DPtr) {
+                QT3DSU8 *dataPtr = inEffect.GetDataSectionBegin() + theDefinition.m_Offset;
+                StaticAssert<sizeof(CRegisteredString)
+                             == sizeof(NVRenderTexture2DPtr)>::valid_expression();
+                CRegisteredString *theStrPtr = reinterpret_cast<CRegisteredString *>(dataPtr);
+                IOffscreenRenderManager &theOffscreenRenderer(
+                    m_Context->GetOffscreenRenderManager());
+
+                if (theStrPtr->IsValid()) {
+                    if (theOffscreenRenderer.HasOffscreenRenderer(*theStrPtr))
+                        theOffscreenRenderer.GetRenderedItem(*theStrPtr);
+                }
+            }
+        }
+    }
 };
 }
 
diff --git a/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.h b/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.h
index 119564da..879addc9 100644
--- a/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.h
+++ b/src/Runtime/Source/runtimerender/Qt3DSRenderEffectSystem.h
@@ -201,6 +201,8 @@ namespace render {
         // enabling blending when rendering to the target
         virtual bool RenderEffect(SEffectRenderArgument inRenderArgument, QT3DSMat44 &inMVP,
                                   bool inEnableBlendWhenRenderToTarget) = 0;
+
+        virtual void renderSubpresentations(SEffect &inEffect) = 0;
     };
 }
 }
diff --git a/src/Runtime/Source/runtimerender/Qt3DSRenderRenderList.h b/src/Runtime/Source/runtimerender/Qt3DSRenderRenderList.h
index dce9a95a..c498c7ee 100644
--- a/src/Runtime/Source/runtimerender/Qt3DSRenderRenderList.h
+++ b/src/Runtime/Source/runtimerender/Qt3DSRenderRenderList.h
@@ -118,4 +118,4 @@ namespace render {
 }
 }
 
-#endif
\ No newline at end of file
+#endif
diff --git a/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp b/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
index e1086304..a6e6b14f 100644
--- a/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
+++ b/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
@@ -795,6 +795,10 @@ namespace render {
         retval.m_Opacity = inOpacity;
         QT3DSF32 &subsetOpacity(retval.m_Opacity);
 
+        // If the custom material uses subpresentations, those have to be rendered before
+        // the custom material itself
+        m_Renderer.GetQt3DSContext().GetCustomMaterialSystem().renderSubpresentations(inMaterial);
+
         // set wireframe mode
         m_Renderer.DefaultMaterialShaderKeyProperties().m_WireframeMode.SetValue(
             theGeneratedKey, m_Renderer.GetQt3DSContext().GetWireframeMode());
@@ -1170,6 +1174,9 @@ namespace render {
                     theEffect->m_Flags.SetDirty(false);
                 }
                 if (theEffect->m_Flags.IsActive()) {
+                    // If the effect uses subpresentations, those have to be rendered before
+                    // the effect itself
+                    theEffectSystem.renderSubpresentations(*theEffect);
                     theLastEffect = theEffect;
                     if (hasOffscreenRenderer == false
                         && theEffectSystem.DoesEffectRequireDepthTexture(theEffect->m_ClassName))