Fix progressive antialiasing

There were two problems with the antialiasing. The default materials
inside container were never cleared dirty. The subpresentation rendering
didn't correctly set the dirty flags when the same subpresentation is
rendered multiple times.

Task-number: QT3DS-1796
Change-Id: Ice1942ec4b919df009c59aa3ffdfc8efd430db1e
Reviewed-by: Miikka Heikkinen <miikka.heikkinen@qt.io>
Reviewed-by: Mahmoud Badri <mahmoud.badri@qt.io>
Reviewed-by: Jari Karppinen <jari.karppinen@qt.io>
Reviewed-by: Pasi Keränen <pasi.keranen@qt.io>

8 files changed, 56 insertions, 30 deletions

diff --git a/src/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp b/src/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
index aaf799f..a6fb9ee 100644
--- a/src/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
+++ b/src/runtimerender/Qt3DSRenderCustomMaterialSystem.cpp
@@ -1975,7 +1975,7 @@ struct SMaterialSystem : public ICustomMaterialSystem
     // TODO - return more information, specifically about transparency (object is transparent,
     // object is completely transparent
     bool PrepareForRender(const SModel & /*inModel*/, const SRenderSubset & /*inSubset*/,
-                                  SCustomMaterial &inMaterial, bool clearMaterialDirtyFlags) override
+                          SCustomMaterial &inMaterial) override
     {
         SMaterialClass *theMaterialClass = GetMaterialClass(inMaterial.m_ClassName);
         if (theMaterialClass == NULL) {
@@ -1990,8 +1990,6 @@ struct SMaterialSystem : public ICustomMaterialSystem
         inMaterial.m_hasVolumetricDF = false;
 
         bool wasDirty = inMaterial.IsDirty() || theMaterialClass->m_AlwaysDirty;
-        if (clearMaterialDirtyFlags)
-            inMaterial.UpdateDirtyForFrame();
 
         return wasDirty;
     }
diff --git a/src/runtimerender/Qt3DSRenderCustomMaterialSystem.h b/src/runtimerender/Qt3DSRenderCustomMaterialSystem.h
index 6fe522f..478d2e3 100644
--- a/src/runtimerender/Qt3DSRenderCustomMaterialSystem.h
+++ b/src/runtimerender/Qt3DSRenderCustomMaterialSystem.h
@@ -111,7 +111,7 @@ namespace render {
         // Returns true if the material is dirty and thus will produce a different render result
         // than previously.  This effects things like progressive AA.
         virtual bool PrepareForRender(const SModel &inModel, const SRenderSubset &inSubset,
-                                      SCustomMaterial &inMaterial, bool inClearDirty) = 0;
+                                      SCustomMaterial &inMaterial) = 0;
 
         virtual bool RenderDepthPrepass(const QT3DSMat44 &inMVP, const SCustomMaterial &inMaterial,
                                         const SRenderSubset &inSubset) = 0;
diff --git a/src/runtimerender/graphobjects/Qt3DSRenderScene.cpp b/src/runtimerender/graphobjects/Qt3DSRenderScene.cpp
index 7917bd7..7049035 100644
--- a/src/runtimerender/graphobjects/Qt3DSRenderScene.cpp
+++ b/src/runtimerender/graphobjects/Qt3DSRenderScene.cpp
@@ -37,8 +37,8 @@ using namespace qt3ds::render;
 
 SScene::SScene()
     : SGraphObject(GraphObjectTypes::Scene)
-    , m_Presentation(NULL)
-    , m_FirstChild(NULL)
+    , m_Presentation(nullptr)
+    , m_FirstChild(nullptr)
     , m_ClearColor(0.0f)
     , m_UseClearColor(true)
     , m_Dirty(true)
@@ -47,7 +47,7 @@ SScene::SScene()
 
 void SScene::AddChild(SLayer &inLayer)
 {
-    if (m_FirstChild == NULL)
+    if (m_FirstChild == nullptr)
         m_FirstChild = &inLayer;
     else
         GetLastChild()->m_NextSibling = &inLayer;
@@ -71,6 +71,7 @@ bool SScene::PrepareForRender(const QT3DSVec2 &inViewportDimensions, IQt3DSRende
     // We need to iterate through the layers in reverse order and ask them to render.
     bool wasDirty = m_Dirty;
     m_Dirty = false;
+
     if (m_FirstChild) {
         wasDirty |=
             inContext.GetRenderer().PrepareLayerForRender(*m_FirstChild, inViewportDimensions,
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp b/src/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp
index c40ff55..10f62c3 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp
@@ -236,6 +236,20 @@ namespace render {
 
             if (theRenderData) {
                 theRenderData->PrepareForRender();
+                if (id) {
+                    if (m_initialPrepareData.contains(theLayer)) {
+                        // Copy dirty state from the initial since the graph is
+                        // not dirty for subsequent calls
+                        auto &flags = theRenderData->m_LayerPrepResult->m_Flags;
+                        const auto &initialFlags
+                                = m_initialPrepareData[theLayer]->m_LayerPrepResult->m_Flags;
+                        flags.SetWasDirty(flags.WasDirty() || initialFlags.WasDirty());
+                        flags.SetLayerDataDirty(flags.WasLayerDataDirty()
+                                                || initialFlags.WasLayerDataDirty());
+                    } else {
+                        m_initialPrepareData.insert(theLayer, theRenderData);
+                    }
+                }
                 retval = retval || theRenderData->m_LayerPrepResult->m_Flags.WasDirty();
             } else {
                 QT3DS_ASSERT(false);
@@ -495,6 +509,14 @@ namespace render {
         for (QT3DSU32 idx = 0, end = m_LastFrameLayers.size(); idx < end; ++idx)
             m_LastFrameLayers[idx]->ResetForFrame();
         m_LastFrameLayers.clear();
+        m_initialPrepareData.clear();
+        for (auto *obj : qAsConst(m_materialClearDirty)) {
+            if (obj->m_Type == GraphObjectTypes::DefaultMaterial)
+                static_cast<SDefaultMaterial *>(obj)->m_Dirty.UpdateDirtyForFrame();
+            else if (obj->m_Type == GraphObjectTypes::CustomMaterial)
+                static_cast<SCustomMaterial *>(obj)->UpdateDirtyForFrame();
+        }
+        m_materialClearDirty.clear();
         m_BeginFrameViewport = m_qt3dsContext.GetRenderList().GetViewport();
     }
     void Qt3DSRendererImpl::EndFrame()
@@ -1141,6 +1163,11 @@ namespace render {
             inImage.m_TextureData.m_Texture->GenerateMipmaps();
     }
 
+    void Qt3DSRendererImpl::addMaterialDirtyClear(SGraphObject *obj)
+    {
+        m_materialClearDirty.insert(obj);
+    }
+
     bool NodeContainsBoneRoot(SNode &childNode, QT3DSI32 rootID)
     {
         for (SNode *childChild = childNode.m_FirstChild; childChild != NULL;
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImpl.h b/src/runtimerender/rendererimpl/Qt3DSRendererImpl.h
index 906afd6..6311a2d 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImpl.h
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImpl.h
@@ -292,6 +292,9 @@ namespace render {
         bool m_LayerGPuProfilingEnabled;
         SShaderDefaultMaterialKeyProperties m_DefaultMaterialShaderKeyProperties;
 
+        QHash<SLayer *, SLayerRenderData *> m_initialPrepareData;
+        QSet<SGraphObject *> m_materialClearDirty;
+
     public:
         Qt3DSRendererImpl(IQt3DSRenderContext &ctx);
         virtual ~Qt3DSRendererImpl();
@@ -402,6 +405,7 @@ namespace render {
         void BeginLayerRender(SLayerRenderData &inLayer);
         void EndLayerRender();
         void PrepareImageForIbl(SImage &inImage);
+        void addMaterialDirtyClear(SGraphObject *obj);
 
         NVRenderShaderProgram *CompileShader(CRegisteredString inName, const char8_t *inVert,
                                              const char8_t *inFrame);
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
index 43934b1..69c79ff 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
@@ -1281,9 +1281,8 @@ namespace render {
         // If the user has disabled all layer caching this has the side effect of disabling the
         // progressive AA algorithm.
         if (thePrepResult.m_Flags.WasLayerDataDirty()
-            || thePrepResult.m_Flags.WasDirty()
-            || m_Renderer.IsLayerCachingEnabled() == false
-            || thePrepResult.m_Flags.ShouldRenderToTexture()) {
+                || thePrepResult.m_Flags.WasDirty()
+                || m_Renderer.IsLayerCachingEnabled() == false) {
             m_ProgressiveAAPassIndex = 0;
             m_NonDirtyTemporalAAPassIndex = 0;
             needsRender = true;
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
index 5d1e4ed..6dcd4c9 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
@@ -501,10 +501,8 @@ namespace render {
 
             if (theMaterial != NULL && theMaterial->m_Type == GraphObjectTypes::DefaultMaterial) {
                 SDefaultMaterial *theDefaultMaterial = static_cast<SDefaultMaterial *>(theMaterial);
-                // Don't clear dirty flags if the material was referenced.
-                bool clearMaterialFlags = theMaterial == inPath.m_Material;
                 SDefaultMaterialPreparationResult prepResult(PrepareDefaultMaterialForRender(
-                    *theDefaultMaterial, theFlags, subsetOpacity, clearMaterialFlags));
+                    *theDefaultMaterial, theFlags, subsetOpacity));
 
                 theFlags = prepResult.m_RenderableFlags;
                 if (inPath.m_PathType == PathTypes::Geometry) {
@@ -546,7 +544,8 @@ namespace render {
                 // Don't clear dirty flags if the material was referenced.
                 // bool clearMaterialFlags = theMaterial == inPath.m_Material;
                 SDefaultMaterialPreparationResult prepResult(
-                    PrepareCustomMaterialForRender(*theCustomMaterial, theFlags, subsetOpacity));
+                    PrepareCustomMaterialForRender(*theCustomMaterial, theFlags, subsetOpacity,
+                                                   retval));
 
                 theFlags = prepResult.m_RenderableFlags;
                 if (inPath.m_PathType == PathTypes::Geometry) {
@@ -661,8 +660,7 @@ namespace render {
     }
 
     SDefaultMaterialPreparationResult SLayerRenderPreparationData::PrepareDefaultMaterialForRender(
-        SDefaultMaterial &inMaterial, SRenderableObjectFlags &inExistingFlags, QT3DSF32 inOpacity,
-        bool inClearDirtyFlags)
+        SDefaultMaterial &inMaterial, SRenderableObjectFlags &inExistingFlags, QT3DSF32 inOpacity)
     {
         SDefaultMaterial *theMaterial = &inMaterial;
         SDefaultMaterialPreparationResult retval(GenerateLightingKey(theMaterial->m_Lighting));
@@ -676,8 +674,6 @@ namespace render {
             renderableFlags |= RenderPreparationResultFlagValues::Dirty;
         }
         subsetOpacity *= theMaterial->m_Opacity;
-        if (inClearDirtyFlags)
-            theMaterial->m_Dirty.UpdateDirtyForFrame();
 
         SRenderableImage *firstImage = NULL;
 
@@ -781,11 +777,14 @@ namespace render {
         retval.m_FirstImage = firstImage;
         if (retval.m_RenderableFlags.IsDirty())
             retval.m_Dirty = true;
+        if (retval.m_Dirty)
+            m_Renderer.addMaterialDirtyClear(&inMaterial);
         return retval;
     }
 
     SDefaultMaterialPreparationResult SLayerRenderPreparationData::PrepareCustomMaterialForRender(
-        SCustomMaterial &inMaterial, SRenderableObjectFlags &inExistingFlags, QT3DSF32 inOpacity)
+        SCustomMaterial &inMaterial, SRenderableObjectFlags &inExistingFlags, QT3DSF32 inOpacity,
+        bool alreadyDirty)
     {
         SDefaultMaterialPreparationResult retval(GenerateLightingKey(
             DefaultMaterialLighting::FragmentLighting)); // always fragment lighting
@@ -837,6 +836,8 @@ namespace render {
 #undef CHECK_IMAGE_AND_PREPARE
 
         retval.m_FirstImage = firstImage;
+        if (retval.m_Dirty || alreadyDirty)
+            m_Renderer.addMaterialDirtyClear(&inMaterial);
         return retval;
     }
 
@@ -927,10 +928,6 @@ namespace render {
                         subsetDirty | (theSubset.m_WireframeMode != inModel.m_WireframeMode);
                     inModel.m_WireframeMode = false;
                 }
-                // Only clear flags on the materials in this direct hierarchy.  Do not clear them of
-                // this
-                // references materials in another hierarchy.
-                bool clearMaterialDirtyFlags = theMaterialObject == theSourceMaterialObject;
 
                 if (theMaterialObject == NULL)
                     continue;
@@ -939,8 +936,8 @@ namespace render {
                     SDefaultMaterial &theMaterial(
                         static_cast<SDefaultMaterial &>(*theMaterialObject));
                     SDefaultMaterialPreparationResult theMaterialPrepResult(
-                        PrepareDefaultMaterialForRender(theMaterial, renderableFlags, subsetOpacity,
-                                                        clearMaterialDirtyFlags));
+                        PrepareDefaultMaterialForRender(theMaterial, renderableFlags,
+                                                        subsetOpacity));
                     SShaderDefaultMaterialKey theGeneratedKey = theMaterialPrepResult.m_MaterialKey;
                     subsetOpacity = theMaterialPrepResult.m_Opacity;
                     SRenderableImage *firstImage(theMaterialPrepResult.m_FirstImage);
@@ -969,11 +966,11 @@ namespace render {
                     ICustomMaterialSystem &theMaterialSystem(
                         qt3dsContext.GetCustomMaterialSystem());
                     subsetDirty |= theMaterialSystem.PrepareForRender(
-                        theModelContext.m_Model, theSubset, theMaterial, clearMaterialDirtyFlags);
+                        theModelContext.m_Model, theSubset, theMaterial);
 
                     SDefaultMaterialPreparationResult theMaterialPrepResult(
                         PrepareCustomMaterialForRender(theMaterial, renderableFlags,
-                                                       subsetOpacity));
+                                                       subsetOpacity, subsetDirty));
                     SShaderDefaultMaterialKey theGeneratedKey = theMaterialPrepResult.m_MaterialKey;
                     subsetOpacity = theMaterialPrepResult.m_Opacity;
                     SRenderableImage *firstImage(theMaterialPrepResult.m_FirstImage);
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.h b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.h
index 5b8d6e1..4a4415f 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.h
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.h
@@ -313,12 +313,12 @@ namespace render {
 
         SDefaultMaterialPreparationResult
         PrepareDefaultMaterialForRender(SDefaultMaterial &inMaterial,
-                                        SRenderableObjectFlags &inExistingFlags, QT3DSF32 inOpacity,
-                                        bool inClearMaterialFlags);
+                                        SRenderableObjectFlags &inExistingFlags, QT3DSF32 inOpacity);
 
         SDefaultMaterialPreparationResult
         PrepareCustomMaterialForRender(SCustomMaterial &inMaterial,
-                                       SRenderableObjectFlags &inExistingFlags, QT3DSF32 inOpacity);
+                                       SRenderableObjectFlags &inExistingFlags, QT3DSF32 inOpacity,
+                                       bool alreadyDirty);
 
         bool PrepareModelForRender(SModel &inModel, const QT3DSMat44 &inViewProjection,
                                    const Option<SClippingFrustum> &inClipFrustum,