Fix rendering problems with ordered group and alphatest objects

Change the logic of the ordered group rendering. Previously all objects in
ordered group were drawn in one go as opaque objects and alpha-test
objects were all rendered in transparent pass.

- Change the logic so that ordered property only affects transparent
  objects when depth test is on and all if it is off.
- Allow adding alpha-test objects to both opaque and transparent object
  lists and only draw the respective parts in the each pass.
- Change transparent object rendering into one pass.

Task-number: QT3DS-4155
Change-Id: I53fd6acfc0a887e01996306b5eef0c14c97e18ad
Reviewed-by: Tomi Korpipää <tomi.korpipaa@qt.io>

2 files changed, 82 insertions, 80 deletions

diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
index eb06cb0..3c0f541 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
@@ -1025,44 +1025,55 @@ void SLayerRenderData::renderTransparentObjectsPass(
             for (QT3DSU32 idx = 0, end = theTransparentObjects.size(); idx < end; ++idx) {
                 SRenderableObject &theObject(*theTransparentObjects[idx]);
                 if (!(theObject.m_RenderableFlags.IsCompletelyTransparent())) {
+                    if (theObject.m_RenderableFlags.isOrderedGroup()) {
+                        renderOrderedGroup(theObject, inRenderFn, inEnableBlending,
+                                           inEnableDepthWrite, inEnableTransparentDepthWrite,
+                                           indexLight, inCamera, theFB);
+                    } else {
 #ifdef ADVANCED_BLEND_SW_FALLBACK
-                    // SW fallback for advanced blend modes.
-                    // Renders transparent objects to a separate FBO and blends them in shader
-                    // with the opaque items and background.
-                    DefaultMaterialBlendMode::Enum blendMode
-                            = DefaultMaterialBlendMode::Enum::Normal;
-                    if (theObject.m_RenderableFlags.IsDefaultMaterialMeshSubset())
-                        blendMode = static_cast<SSubsetRenderable &>(theObject).getBlendingMode();
-                    bool useBlendFallback
-                            = (blendMode == DefaultMaterialBlendMode::Overlay
-                               || blendMode == DefaultMaterialBlendMode::ColorBurn
-                               || blendMode == DefaultMaterialBlendMode::ColorDodge)
-                            && !theRenderContext.IsAdvancedBlendHwSupported()
-                            && !theRenderContext.IsAdvancedBlendHwSupportedKHR()
-                            && m_LayerPrepassDepthTexture;
-                    if (useBlendFallback)
-                        SetupDrawFB(true);
+                        // SW fallback for advanced blend modes.
+                        // Renders transparent objects to a separate FBO and blends them in shader
+                        // with the opaque items and background.
+                        DefaultMaterialBlendMode::Enum blendMode
+                                = DefaultMaterialBlendMode::Enum::Normal;
+                        if (theObject.m_RenderableFlags.IsDefaultMaterialMeshSubset())
+                            blendMode = static_cast<SSubsetRenderable &>(theObject).getBlendingMode();
+                        bool useBlendFallback
+                                = (blendMode == DefaultMaterialBlendMode::Overlay
+                                   || blendMode == DefaultMaterialBlendMode::ColorBurn
+                                   || blendMode == DefaultMaterialBlendMode::ColorDodge)
+                                && !theRenderContext.IsAdvancedBlendHwSupported()
+                                && !theRenderContext.IsAdvancedBlendHwSupportedKHR()
+                                && m_LayerPrepassDepthTexture;
+                        if (useBlendFallback)
+                            SetupDrawFB(true);
 #endif
-                    SScopedLightsListScope lightsScope(m_Lights, m_LightDirections,
-                                                       m_SourceLightDirections,
-                                                       theObject.m_ScopedLights);
-                    SetShaderFeature(m_CGLightingFeatureName, m_Lights.empty() == false);
+                        SScopedLightsListScope lightsScope(m_Lights, m_LightDirections,
+                                                           m_SourceLightDirections,
+                                                           theObject.m_ScopedLights);
+                        SetShaderFeature(m_CGLightingFeatureName, m_Lights.empty() == false);
+
+                        if (theObject.m_RenderableFlags.hasAlphaTest())
+                            m_Renderer.setAlphaTest(true, -1.0f, 1.0f);
+                        else
+                            m_Renderer.setAlphaTest(false, 1.0f, 1.0f);
 
-                    inRenderFn(*this, theObject, theCameraProps, GetShaderFeatureSet(),
-                               indexLight, inCamera);
+                        inRenderFn(*this, theObject, theCameraProps, GetShaderFeatureSet(),
+                                   indexLight, inCamera);
 #ifdef ADVANCED_BLEND_SW_FALLBACK
-                    // SW fallback for advanced blend modes.
-                    // Continue blending after transparent objects have been rendered to a FBO
-                    if (useBlendFallback) {
-                        BlendAdvancedToFB(blendMode, true, theFB);
-                        // restore blending status
-                        theRenderContext.SetBlendingEnabled(inEnableBlending);
-                        // restore depth test status
-                        theRenderContext.SetDepthTestEnabled(
-                                    m_Layer.m_Flags.IsLayerEnableDepthTest());
-                        theRenderContext.SetDepthWriteEnabled(inEnableTransparentDepthWrite);
-                    }
+                        // SW fallback for advanced blend modes.
+                        // Continue blending after transparent objects have been rendered to a FBO
+                        if (useBlendFallback) {
+                            BlendAdvancedToFB(blendMode, true, theFB);
+                            // restore blending status
+                            theRenderContext.SetBlendingEnabled(inEnableBlending);
+                            // restore depth test status
+                            theRenderContext.SetDepthTestEnabled(
+                                        m_Layer.m_Flags.IsLayerEnableDepthTest());
+                            theRenderContext.SetDepthWriteEnabled(inEnableTransparentDepthWrite);
+                        }
 #endif
+                    }
                 }
             }
         }
@@ -1102,6 +1113,12 @@ void SLayerRenderData::renderTransparentObjectsPass(
                                                            m_SourceLightDirections,
                                                            theObject.m_ScopedLights);
                         SetShaderFeature(m_CGLightingFeatureName, m_Lights.empty() == false);
+
+                        if (theObject.m_RenderableFlags.hasAlphaTest())
+                            m_Renderer.setAlphaTest(true, -1.0f, 1.0f);
+                        else
+                            m_Renderer.setAlphaTest(false, 1.0f, 1.0f);
+
                         inRenderFn(*this, theObject, theCameraProps, GetShaderFeatureSet(),
                                    indexLight, inCamera);
 #ifdef ADVANCED_BLEND_SW_FALLBACK
@@ -1142,31 +1159,19 @@ void SLayerRenderData::RunRenderPass(TRenderRenderableFunction inRenderFn,
         for (QT3DSU32 idx = 0, end = theOpaqueObjects.size(); idx < end; ++idx) {
             SRenderableObject &theObject(*theOpaqueObjects[idx]);
 
-            if (theObject.m_RenderableFlags.isOrderedGroup()) {
-                renderOrderedGroup(theObject, inRenderFn, inEnableBlending, inEnableDepthWrite,
-                                   inEnableTransparentDepthWrite, indexLight, inCamera, theFB);
-            } else {
-                theRenderContext.SetBlendingEnabled(false);
-                theRenderContext.SetDepthWriteEnabled(opaqueDepthWrite);
-                SetShaderFeature(m_CGLightingFeatureName, m_Lights.empty() == false);
-                SScopedLightsListScope lightsScope(m_Lights, m_LightDirections, m_SourceLightDirections,
-                                                   theObject.m_ScopedLights);
-                inRenderFn(*this, theObject, theCameraProps, GetShaderFeatureSet(), indexLight,
-                           inCamera);
-            }
-        }
-    }
-    {
-        QT3DS_PERF_SCOPED_TIMER(m_Renderer.GetQt3DSContext().GetPerfTimer(),
-                                "LayerRenderData: Render transparent pass1")
-        NVDataRef<SRenderableObject *> theTransparentObjects = GetTransparentRenderableObjects();
-        // Also draw opaque parts of transparent objects
-        m_Renderer.setAlphaTest(true, 1.0f, -1.0f + (1.0f / 255.0f));
-        for (QT3DSU32 idx = 0, end = theTransparentObjects.size(); idx < end; ++idx) {
-            SRenderableObject &theObject(*theTransparentObjects[idx]);
+            QT3DS_ASSERT(!theObject.m_RenderableFlags.isOrderedGroup());
+
+            theRenderContext.SetBlendingEnabled(false);
+            theRenderContext.SetDepthWriteEnabled(opaqueDepthWrite);
+            SetShaderFeature(m_CGLightingFeatureName, m_Lights.empty() == false);
             SScopedLightsListScope lightsScope(m_Lights, m_LightDirections, m_SourceLightDirections,
                                                theObject.m_ScopedLights);
-            SetShaderFeature(m_CGLightingFeatureName, m_Lights.empty() == false);
+
+            if (theObject.m_RenderableFlags.hasAlphaTest())
+                m_Renderer.setAlphaTest(true, 1.0f, -1.0f + (1.0f / 255.0f));
+            else
+                m_Renderer.setAlphaTest(false, 1.0f, 1.0f);
+
             inRenderFn(*this, theObject, theCameraProps, GetShaderFeatureSet(), indexLight,
                        inCamera);
         }
@@ -1174,20 +1179,7 @@ void SLayerRenderData::RunRenderPass(TRenderRenderableFunction inRenderFn,
 
     {
         QT3DS_PERF_SCOPED_TIMER(m_Renderer.GetQt3DSContext().GetPerfTimer(),
-                                "LayerRenderData: Render transparent pass2")
-        m_Renderer.setAlphaTest(true, -1.0f, 1.0f);
-        // transparent parts of transparent objects
-        // does not render objects without alpha test enabled so
-        // we need another pass without alpha test
-        renderTransparentObjectsPass(inRenderFn, inEnableBlending, inEnableDepthWrite,
-                                     inEnableTransparentDepthWrite, indexLight, inCamera, theFB);
-    }
-
-    {
-        QT3DS_PERF_SCOPED_TIMER(m_Renderer.GetQt3DSContext().GetPerfTimer(),
-                                "LayerRenderData: Render transparent pass3")
-        m_Renderer.setAlphaTest(false, 1.0, 1.0);
-        // transparent objects without alpha test
+                                "LayerRenderData: Render transparent pass")
         renderTransparentObjectsPass(inRenderFn, inEnableBlending, inEnableDepthWrite,
                                      inEnableTransparentDepthWrite, indexLight, inCamera, theFB);
     }
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
index a8b44f7..fe7c1e5 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderPreparationData.cpp
@@ -267,12 +267,6 @@ namespace render {
             QT3DSVec3 theCameraDirection(GetCameraDirection());
             QT3DSVec3 theCameraPosition = m_Camera->GetGlobalPos();
             m_RenderedOpaqueObjects.assign(m_OpaqueObjects.begin(), m_OpaqueObjects.end());
-            m_RenderedOpaqueObjects.insert(m_RenderedOpaqueObjects.end(), m_GroupObjects.begin(),
-                                           m_GroupObjects.end());
-
-            // Set position for group objects
-            for (int i = 0; i < m_GroupObjects.size(); i++)
-                static_cast<SOrderedGroupRenderable *>(m_GroupObjects[i])->update();
 
             // Setup the object's sorting information
             for (QT3DSU32 idx = 0, end = m_RenderedOpaqueObjects.size(); idx < end; ++idx) {
@@ -297,6 +291,12 @@ namespace render {
 
         m_RenderedTransparentObjects.assign(m_TransparentObjects.begin(),
                                             m_TransparentObjects.end());
+        m_RenderedTransparentObjects.insert(m_RenderedTransparentObjects.end(), m_GroupObjects.begin(),
+                                       m_GroupObjects.end());
+
+        // Set position for group objects
+        for (int i = 0; i < m_GroupObjects.size(); i++)
+            static_cast<SOrderedGroupRenderable *>(m_GroupObjects[i])->update();
 
         if (m_Layer.m_Flags.IsLayerEnableDepthTest() == false) {
             m_RenderedTransparentObjects.insert(m_RenderedTransparentObjects.end(),
@@ -1083,12 +1083,22 @@ namespace render {
                     theRenderableObject->m_ScopedLights = inScopedLights;
                     // set tessellation
                     theRenderableObject->m_TessellationMode = inModel.m_TessellationMode;
-
-                    if (group) {
+                    bool transparentObject = theRenderableObject->m_RenderableFlags.HasTransparency()
+                            || theRenderableObject->m_RenderableFlags.HasRefraction();
+                    bool alphaTestObject = theRenderableObject->m_RenderableFlags.hasAlphaTest();
+
+                    // Only put transparent object into ordered group if depth test is on
+                    // Put object into both lists if it is alpha-test object
+                    // object must have transparency on if it is alpha-test object
+                    if (group && (!m_Layer.m_Flags.IsLayerEnableDepthTest() || transparentObject)) {
                         group->m_renderables.push_back(theRenderableObject);
+                        if (alphaTestObject)
+                            m_OpaqueObjects.push_back(theRenderableObject);
                     } else {
-                        if (theRenderableObject->m_RenderableFlags.HasTransparency()
-                            || theRenderableObject->m_RenderableFlags.HasRefraction()) {
+                        if (transparentObject && alphaTestObject) {
+                            m_TransparentObjects.push_back(theRenderableObject);
+                            m_OpaqueObjects.push_back(theRenderableObject);
+                        } else if (transparentObject) {
                             m_TransparentObjects.push_back(theRenderableObject);
                         } else {
                             m_OpaqueObjects.push_back(theRenderableObject);