RenderStates: fix override of nested RenderStates

When merging states, make sure we don't add several states of the same types
with possibly different values. We should only add states with type we don't
already have.

Since the FG traversal is done from leaf to root, we know that the states we
already contain should override any state that may have been specified higher
up in the FG branch.

Change-Id: I9bd1eadd37e8addf740a4b85b2318f9be269fedb
Task-number: QTBUG-76766
Reviewed-by: Sean Harmer <sean.harmer@kdab.com>

7 files changed, 33 insertions, 14 deletions

diff --git a/src/render/renderers/opengl/jobs/renderviewjobutils.cpp b/src/render/renderers/opengl/jobs/renderviewjobutils.cpp
index acb38ffb9..49e85e1d3 100644
--- a/src/render/renderers/opengl/jobs/renderviewjobutils.cpp
+++ b/src/render/renderers/opengl/jobs/renderviewjobutils.cpp
@@ -193,8 +193,10 @@ void setRenderViewConfigFromFrameGraphLeafNode(RenderView *rv, const FrameGraphN
                     rv->setStateSet(stateSet);
                 }
 
+                // Add states from new stateSet we might be missing
+                // but don' t override existing states (lower StateSetNode always has priority)
                 if (rStateSet->hasRenderStates())
-                    addToRenderStateSet(stateSet, rStateSet->renderStates(), manager->renderStateManager());
+                    addUniqueStatesToRenderStateSet(stateSet, rStateSet->renderStates(), manager->renderStateManager());
                 break;
             }
 
@@ -417,14 +419,16 @@ void parametersFromMaterialEffectTechnique(ParameterInfoList *infoList,
     parametersFromParametersProvider(infoList, manager, technique);
 }
 
-void addToRenderStateSet(RenderStateSet *stateSet,
-                         const QVector<Qt3DCore::QNodeId> stateIds,
-                         RenderStateManager *manager)
+// Only add states with types we don't already have
+void addUniqueStatesToRenderStateSet(RenderStateSet *stateSet,
+                                     const QVector<Qt3DCore::QNodeId> stateIds,
+                                     RenderStateManager *manager)
 {
     for (const Qt3DCore::QNodeId &stateId : stateIds) {
         RenderStateNode *node = manager->lookupResource(stateId);
-        if (node->isEnabled())
+        if (node->isEnabled() && !stateSet->hasStateOfType(node->type())) {
             stateSet->addState(node->impl());
+        }
     }
 }
 
diff --git a/src/render/renderers/opengl/jobs/renderviewjobutils_p.h b/src/render/renderers/opengl/jobs/renderviewjobutils_p.h
index 90c4a53cc..bd2e12534 100644
--- a/src/render/renderers/opengl/jobs/renderviewjobutils_p.h
+++ b/src/render/renderers/opengl/jobs/renderviewjobutils_p.h
@@ -150,9 +150,9 @@ void parametersFromParametersProvider(ParameterInfoList *infoList,
 Q_AUTOTEST_EXPORT ParameterInfoList::const_iterator findParamInfo(ParameterInfoList *infoList,
                                                                   const int nameId);
 
-Q_AUTOTEST_EXPORT void addToRenderStateSet(RenderStateSet *stateSet,
-                                           const QVector<Qt3DCore::QNodeId> stateIds,
-                                           RenderStateManager *manager);
+Q_AUTOTEST_EXPORT void addUniqueStatesToRenderStateSet(RenderStateSet *stateSet,
+                                                       const QVector<Qt3DCore::QNodeId> stateIds,
+                                                       RenderStateManager *manager);
 
 typedef QHash<int, QVariant> UniformBlockValueBuilderHash;
 
diff --git a/src/render/renderers/opengl/renderer/renderview.cpp b/src/render/renderers/opengl/renderer/renderview.cpp
index be9968d06..2c96939a7 100644
--- a/src/render/renderers/opengl/renderer/renderview.cpp
+++ b/src/render/renderers/opengl/renderer/renderview.cpp
@@ -603,10 +603,7 @@ QVector<RenderCommand *> RenderView::buildDrawRenderCommands(const QVector<Entit
                 RenderPass *pass = passData.pass;
                 if (pass->hasRenderStates()) {
                     command->m_stateSet = new RenderStateSet();
-                    addToRenderStateSet(command->m_stateSet, pass->renderStates(), m_manager->renderStateManager());
-
-                    // Merge per pass stateset with global stateset
-                    // so that the local stateset only overrides
+                    addUniqueStatesToRenderStateSet(command->m_stateSet, pass->renderStates(), m_manager->renderStateManager());
                     if (m_stateSet != nullptr)
                         command->m_stateSet->merge(m_stateSet);
                     command->m_changeCost = m_renderer->defaultRenderState()->changeCost(command->m_stateSet);
diff --git a/src/render/renderers/opengl/renderstates/renderstateset.cpp b/src/render/renderers/opengl/renderstates/renderstateset.cpp
index f7fc279a1..b14695c77 100644
--- a/src/render/renderers/opengl/renderstates/renderstateset.cpp
+++ b/src/render/renderers/opengl/renderstates/renderstateset.cpp
@@ -103,9 +103,24 @@ StateMaskSet RenderStateSet::stateMask() const
     return m_stateMask;
 }
 
+// This modifies our state to add states from others
+// if we don't already contain a state with that type set
 void RenderStateSet::merge(RenderStateSet *other)
 {
     m_stateMask |= other->stateMask();
+    const QVector<StateVariant> otherStates = other->states();
+
+    // We only add states which are new (different type)
+    for (const StateVariant &otherState : otherStates) {
+        const bool hasFoundStateOfSameType = hasStateOfType(otherState.type);
+        if (!hasFoundStateOfSameType)
+            m_states.push_back(otherState);
+    }
+}
+
+bool RenderStateSet::hasStateOfType(StateMask type) const
+{
+    return (type & stateMask());
 }
 
 bool RenderStateSet::contains(const StateVariant &ds) const
diff --git a/src/render/renderers/opengl/renderstates/renderstateset_p.h b/src/render/renderers/opengl/renderstates/renderstateset_p.h
index 09b58b859..29be4d2f1 100644
--- a/src/render/renderers/opengl/renderstates/renderstateset_p.h
+++ b/src/render/renderers/opengl/renderstates/renderstateset_p.h
@@ -93,6 +93,9 @@ public:
 
     QVector<StateVariant> states() const { return m_states; }
 
+    bool hasStateOfType(StateMask type) const;
+
+
     /**
      * @brief contains - check if this set contains a matching piece of state
      * @param ds
diff --git a/src/render/renderstates/genericstate_p.h b/src/render/renderstates/genericstate_p.h
index 69c3dee15..b07487d65 100644
--- a/src/render/renderstates/genericstate_p.h
+++ b/src/render/renderstates/genericstate_p.h
@@ -96,7 +96,7 @@ public:
     bool equalTo(const RenderStateImpl &renderState) const override
     {
         const GenericState *other = static_cast<const GenericState*>(&renderState);
-        return (other != NULL && other->m_values == m_values);
+        return (other != nullptr && other->m_values == m_values);
     }
 
     StateMask mask() const override
diff --git a/src/render/renderstates/renderstatenode_p.h b/src/render/renderstates/renderstatenode_p.h
index 886bb0c95..277b8a7c8 100644
--- a/src/render/renderstates/renderstatenode_p.h
+++ b/src/render/renderstates/renderstatenode_p.h
@@ -65,7 +65,7 @@ public:
 
     void sceneChangeEvent(const Qt3DCore::QSceneChangePtr &e) override;
 
-    StateMaskSet mask() const { return m_impl.type; }
+    StateMask type() const { return m_impl.type; }
     StateVariant impl() const { return m_impl; }
 
 protected: