Fix subpresentation transparent clear

Always include alpha channel for subpresentations and properly handle
clear color setup. It needs to setup differently when we render to texture
vs viewport. When rendering to viewport, if the clear color contains
alpha, we need to blend the clear color to the target instead of clearing
the target. Also we need to blend the scene color to the matte color, when
matte and scene color are enabled and the scene color contains alpha.

Task-number: QT3DS-4008
Change-Id: Ic2b3bac8ea586cbd205421c8efd21c1e3e49abd4
Reviewed-by: Miikka Heikkinen <miikka.heikkinen@qt.io>
Reviewed-by: Tomi Korpipää <tomi.korpipaa@qt.io>

11 files changed, 115 insertions, 34 deletions

diff --git a/src/runtimerender/Qt3DSRenderContextCore.cpp b/src/runtimerender/Qt3DSRenderContextCore.cpp
index 596ad37..50042d1 100644
--- a/src/runtimerender/Qt3DSRenderContextCore.cpp
+++ b/src/runtimerender/Qt3DSRenderContextCore.cpp
@@ -249,7 +249,7 @@ struct SRenderContext : public IQt3DSRenderContext
     StereoViews::Enum m_StereoView;
     double m_StereoEyeSeparation;
     bool m_WireframeMode;
-    bool m_IsInSubPresentation;
+    bool m_subPresentationRenderInLayer;
     Option<QT3DSVec4> m_SceneColor;
     Option<QT3DSVec4> m_MatteColor;
     bool m_matteEnabled;
@@ -290,7 +290,7 @@ struct SRenderContext : public IQt3DSRenderContext
         , m_StereoView(StereoViews::Mono)
         , m_StereoEyeSeparation(0.4)
         , m_WireframeMode(false)
-        , m_IsInSubPresentation(false)
+        , m_subPresentationRenderInLayer(false)
         , m_matteEnabled(false)
         , m_Rotation(RenderRotationValues::NoRotation)
         , m_ContextRenderTarget(NULL)
@@ -417,8 +417,14 @@ struct SRenderContext : public IQt3DSRenderContext
     bool IsAuthoringMode() override { return m_AuthoringMode; }
     void SetAuthoringMode(bool inMode) override { m_AuthoringMode = inMode; }
 
-    bool IsInSubPresentation() override { return m_IsInSubPresentation; }
-    void SetInSubPresentation(bool inValue) override { m_IsInSubPresentation = inValue; }
+    bool isSubPresentationRenderInLayer() override
+    {
+        return m_subPresentationRenderInLayer;
+    }
+    void setSubPresentationRenderInLayer(bool inValue) override
+    {
+        m_subPresentationRenderInLayer = inValue;
+    }
 
     ITextRenderer *GetTextRenderer() override { return m_TextRenderer; }
 
@@ -762,13 +768,19 @@ struct SRenderContext : public IQt3DSRenderContext
                     m_RotationTexture = NULL;
                     m_RotationDepthBuffer = NULL;
                 }
-                if (m_SceneColor.hasValue() && m_SceneColor.getValue().w != 0.0f) {
+                if (m_SceneColor.hasValue() && m_SceneColor.getValue().w > 0.0f) {
                     QT3DSVec4 theClearColor = m_SceneColor;
                     if (theClearColor.w < 1.0f) {
                         theClearColor.x *= theClearColor.w;
                         theClearColor.y *= theClearColor.w;
                         theClearColor.z *= theClearColor.w;
                     }
+                    if (m_MatteColor.hasValue() && m_matteEnabled && theClearColor.w < 1.0f) {
+                        theClearColor.x += m_MatteColor->x * (1.0f - theClearColor.w);
+                        theClearColor.y += m_MatteColor->y * (1.0f - theClearColor.w);
+                        theClearColor.z += m_MatteColor->z * (1.0f - theClearColor.w);
+                        theClearColor.w += m_MatteColor->w * (1.0f - theClearColor.w);
+                    }
                     m_RenderContext->SetClearColor(theClearColor);
                     m_RenderContext->Clear(qt3ds::render::NVRenderClearValues::Color);
                 }
diff --git a/src/runtimerender/Qt3DSRenderContextCore.h b/src/runtimerender/Qt3DSRenderContextCore.h
index 9a26876..670becc 100644
--- a/src/runtimerender/Qt3DSRenderContextCore.h
+++ b/src/runtimerender/Qt3DSRenderContextCore.h
@@ -164,8 +164,8 @@ namespace render {
         virtual ITextTextureAtlas *GetTextureAtlas() = 0;
 
         // Sub presentations change the rendering somewhat.
-        virtual bool IsInSubPresentation() = 0;
-        virtual void SetInSubPresentation(bool inValue) = 0;
+        virtual bool isSubPresentationRenderInLayer() = 0;
+        virtual void setSubPresentationRenderInLayer(bool inValue) = 0;
         virtual void SetSceneColor(Option<QT3DSVec4> inSceneColor) = 0;
         virtual void SetMatteColor(Option<QT3DSVec4> inMatteColor) = 0;
         virtual void setMatteEnabled(bool enable) = 0;
diff --git a/src/runtimerender/Qt3DSRenderSubPresentationHelper.h b/src/runtimerender/Qt3DSRenderSubPresentationHelper.h
index d77c7a7..5ae71b8 100644
--- a/src/runtimerender/Qt3DSRenderSubPresentationHelper.h
+++ b/src/runtimerender/Qt3DSRenderSubPresentationHelper.h
@@ -37,11 +37,6 @@
 namespace qt3ds {
 namespace render {
 
-    // Small helper object to setup the state needed to render a sub presentation
-    // correctly.  Sub presentations may have transparency, and if they do then
-    // then need to be rendered with pre multiple alpha disabled.  If they don't,
-    // then they need to be rendered with pre-multiply alpha enabled (and have the alpha channel
-    // set to 1
     struct SSubPresentationHelper
     {
         IQt3DSRenderContext &m_RenderContext;
@@ -53,17 +48,17 @@ namespace render {
                                const QSize &inPresDimensions)
             : m_RenderContext(inContext)
             , m_PreviousPresentationDimensions(inContext.GetCurrentPresentationDimensions())
-            , m_WasInSubPresentation(inContext.IsInSubPresentation())
+            , m_WasInSubPresentation(inContext.isSubPresentationRenderInLayer())
         {
-            m_RenderContext.SetInSubPresentation(true);
+            m_RenderContext.setSubPresentationRenderInLayer(true);
             m_RenderContext.SetPresentationDimensions(inPresDimensions);
         }
         ~SSubPresentationHelper()
         {
-            m_RenderContext.SetInSubPresentation(m_WasInSubPresentation);
+            m_RenderContext.setSubPresentationRenderInLayer(m_WasInSubPresentation);
             m_RenderContext.SetPresentationDimensions(m_PreviousPresentationDimensions);
         }
     };
 }
 }
-#endif
\ No newline at end of file
+#endif
diff --git a/src/runtimerender/Qt3DSRenderSubpresentation.cpp b/src/runtimerender/Qt3DSRenderSubpresentation.cpp
index 0b9d42d..64f08be 100644
--- a/src/runtimerender/Qt3DSRenderSubpresentation.cpp
+++ b/src/runtimerender/Qt3DSRenderSubpresentation.cpp
@@ -56,10 +56,7 @@ namespace render {
     SOffscreenRendererEnvironment
         CSubPresentationRenderer::GetDesiredEnvironment(QT3DSVec2 /*inPresScale*/)
     {
-        // If we aren't using a clear color, then we are expected to blend with the background
-        bool hasTransparency = m_Presentation.m_Scene->m_UseClearColor ? false : true;
-        NVRenderTextureFormats::Enum format =
-            hasTransparency ? NVRenderTextureFormats::RGBA8 : NVRenderTextureFormats::RGB8;
+        NVRenderTextureFormats::Enum format = NVRenderTextureFormats::RGBA8;
         return SOffscreenRendererEnvironment((QT3DSU32)(m_Presentation.m_PresentationDimensions.x),
                                              (QT3DSU32)(m_Presentation.m_PresentationDimensions.y),
                                              format, OffscreenRendererDepthValues::Depth16, false,
@@ -71,12 +68,12 @@ namespace render {
                                           QT3DSVec2 /*inPresScale*/,
                                           const SRenderInstanceId instanceId)
     {
-        bool hasTransparency = m_Presentation.m_Scene->m_UseClearColor ? false : true;
         NVRenderRect theViewportSize(m_RenderContext.GetRenderList().GetViewport());
         bool wasDirty = m_Presentation.m_Scene->PrepareForRender(
             QT3DSVec2((QT3DSF32)theViewportSize.m_Width, (QT3DSF32)theViewportSize.m_Height),
             m_RenderContext, instanceId);
-        return SOffscreenRenderFlags(hasTransparency, wasDirty);
+        // Always transparent
+        return SOffscreenRenderFlags(true, wasDirty);
     }
 
     // Returns true if the rendered result image has transparency, or false
@@ -86,9 +83,6 @@ namespace render {
                                           SScene::RenderClearCommand inClearColorBuffer,
                                           const SRenderInstanceId instanceId)
     {
-        SSubPresentationHelper theHelper(
-            m_RenderContext,
-            QSize((QT3DSU32)inEnvironment.m_Width, (QT3DSU32)inEnvironment.m_Height));
         NVRenderRect theViewportSize(inRenderContext.GetViewport());
         m_Presentation.m_Scene->Render(
             QT3DSVec2((QT3DSF32)theViewportSize.m_Width, (QT3DSF32)theViewportSize.m_Height),
diff --git a/src/runtimerender/Qt3DSRenderer.h b/src/runtimerender/Qt3DSRenderer.h
index 392fb62..2e6e6aa 100644
--- a/src/runtimerender/Qt3DSRenderer.h
+++ b/src/runtimerender/Qt3DSRenderer.h
@@ -123,6 +123,8 @@ namespace render {
         virtual void RenderQuad(const QT3DSVec2 inDimensions, const QT3DSMat44 &inMVP,
                                 NVRenderTexture2D &inQuadTexture) = 0;
 
+        virtual void FillQuad(const QT3DSVec4 &color) = 0;
+
         // This point rendering works uisng indirect array drawing
         // This means you need to setup a GPU buffer
         // which contains the drawing information
diff --git a/src/runtimerender/graphobjects/Qt3DSRenderScene.cpp b/src/runtimerender/graphobjects/Qt3DSRenderScene.cpp
index a38d904..2dadc26 100644
--- a/src/runtimerender/graphobjects/Qt3DSRenderScene.cpp
+++ b/src/runtimerender/graphobjects/Qt3DSRenderScene.cpp
@@ -85,25 +85,27 @@ void SScene::Render(const QT3DSVec2 &inViewportDimensions, IQt3DSRenderContext &
 {
     if ((inClearColorBuffer == SScene::ClearIsOptional && m_UseClearColor)
         || inClearColorBuffer == SScene::AlwaysClear) {
-        QT3DSF32 clearColorAlpha
-                = inContext.IsInSubPresentation() && !m_UseClearColor ? 0.0f : 1.0f;
-        QT3DSVec4 clearColor(0.0f, 0.0f, 0.0f, clearColorAlpha);
+        QT3DSVec4 clearColor(0.0f, 0.0f, 0.0f, 0.0f);
         if (m_UseClearColor) {
             clearColor.x = m_ClearColor.x;
             clearColor.y = m_ClearColor.y;
             clearColor.z = m_ClearColor.z;
             clearColor.w = m_ClearColor.w;
-            if (m_ClearColor.w < 1.0) {
+            if (m_ClearColor.w < 1.0f) {
                 clearColor.x *= m_ClearColor.w;
                 clearColor.y *= m_ClearColor.w;
                 clearColor.z *= m_ClearColor.w;
             }
         }
-        // Maybe clear and reset to previous clear color after we leave.
-        qt3ds::render::NVRenderContextScopedProperty<QT3DSVec4> __clearColor(
-            inContext.GetRenderContext(), &NVRenderContext::GetClearColor,
-            &NVRenderContext::SetClearColor, clearColor);
-        inContext.GetRenderContext().Clear(qt3ds::render::NVRenderClearValues::Color);
+        if (inContext.isSubPresentationRenderInLayer() && clearColor.w < 1.0f) {
+            inContext.GetRenderer().FillQuad(clearColor);
+        } else {
+            // Maybe clear and reset to previous clear color after we leave.
+            qt3ds::render::NVRenderContextScopedProperty<QT3DSVec4> __clearColor(
+                inContext.GetRenderContext(), &NVRenderContext::GetClearColor,
+                &NVRenderContext::SetClearColor, clearColor);
+            inContext.GetRenderContext().Clear(qt3ds::render::NVRenderClearValues::Color);
+        }
     }
     if (m_FirstChild) {
         inContext.GetRenderer().RenderLayer(*m_FirstChild, inViewportDimensions, m_UseClearColor,
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp b/src/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp
index 99d5c45..a0ea8b1 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp
@@ -1131,6 +1131,26 @@ namespace render {
         }
     }
 
+    void Qt3DSRendererImpl::FillQuad(const QT3DSVec4 &color)
+    {
+        m_Context->SetCullingEnabled(false);
+        SFillRectShader *theShader = GetFillRectShader();
+        NVRenderContext &theContext(*m_Context);
+        theContext.SetActiveShader(&theShader->m_Shader);
+        NVRenderBlendEquationArgument equ(NVRenderBlendEquation::Add, NVRenderBlendEquation::Add);
+        NVRenderBlendFunctionArgument func(NVRenderSrcBlendFunc::One,
+                                           NVRenderDstBlendFunc::OneMinusSrcAlpha,
+                                           NVRenderSrcBlendFunc::One,
+                                           NVRenderDstBlendFunc::OneMinusSrcAlpha);
+        theContext.SetBlendEquation(equ);
+        theContext.SetBlendFunction(func);
+        theContext.SetBlendingEnabled(true);
+        theShader->m_color.Set(color);
+        GenerateXYQuad();
+        m_Context->SetInputAssembler(m_QuadInputAssembler);
+        m_Context->Draw(NVRenderDrawMode::Triangles, m_QuadIndexBuffer->GetNumIndices(), 0);
+    }
+
     void Qt3DSRendererImpl::RenderQuad(const QT3DSVec2 inDimensions, const QT3DSMat44 &inMVP,
                                       NVRenderTexture2D &inQuadTexture)
     {
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImpl.h b/src/runtimerender/rendererimpl/Qt3DSRendererImpl.h
index 32c48d3..792d71e 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImpl.h
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImpl.h
@@ -211,6 +211,7 @@ namespace render {
         NVScopedRefCounted<NVRenderAttribLayout> m_PointAttribLayout;
 
         Option<NVScopedRefCounted<SLayerSceneShader>> m_SceneLayerShader;
+        Option<NVScopedRefCounted<SFillRectShader>> m_fillRectShader;
         Option<NVScopedRefCounted<SLayerProgAABlendShader>> m_LayerProgAAShader;
 
         TShaderMap m_Shaders;
@@ -414,6 +415,7 @@ namespace render {
         void RenderQuad(const QT3DSVec2 inDimensions, const QT3DSMat44 &inMVP,
                                 NVRenderTexture2D &inQuadTexture) override;
         void RenderQuad() override;
+        void FillQuad(const QT3DSVec4 &color) override;
 
         void RenderPointsIndirect() override;
 
@@ -481,6 +483,7 @@ namespace render {
         STextRenderHelper GetTextWidgetShader();
         STextRenderHelper GetOnscreenTextShader();
         SLayerSceneShader *GetSceneLayerShader();
+        SFillRectShader *GetFillRectShader();
         NVRenderShaderProgram *GetTextAtlasEntryShader();
         void GenerateXYQuad();
         void GenerateXYQuadStrip();
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
index 24a55af..5eebd82 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderData.cpp
@@ -59,6 +59,7 @@
 #include "Qt3DSRenderTextTextureAtlas.h"
 #include "Qt3DSRenderRenderList.h"
 #include "Qt3DSRendererUtil.h"
+#include "Qt3DSRenderSubPresentationHelper.h"
 
 #ifdef WIN32
 #pragma warning(disable : 4355)
@@ -1232,6 +1233,9 @@ namespace render {
     {
         if (m_LayerPrepResult->IsLayerVisible()) {
             if (GetOffscreenRenderer()) {
+                auto rect = m_Renderer.GetQt3DSContext().GetRenderList().GetViewport();
+                SSubPresentationHelper helper(m_Renderer.GetQt3DSContext(),
+                                              QSize(rect.m_Width, rect.m_Height));
                 if (m_Layer.m_Background == LayerBackground::Color) {
                     m_LastFrameOffscreenRenderer->RenderWithClear(
                         CreateOffscreenRenderEnvironment(), m_Renderer.GetContext(),
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImplShaders.cpp b/src/runtimerender/rendererimpl/Qt3DSRendererImplShaders.cpp
index d69e46e..9949b62 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImplShaders.cpp
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImplShaders.cpp
@@ -2445,6 +2445,36 @@ namespace render {
         return m_SceneLayerShader.getValue();
     }
 
+    SFillRectShader *Qt3DSRendererImpl::GetFillRectShader()
+    {
+        if (m_fillRectShader.hasValue())
+            return m_fillRectShader.getValue();
+
+        GetProgramGenerator().BeginProgram();
+
+        IShaderStageGenerator &vertexGenerator(
+            *GetProgramGenerator().GetStage(ShaderGeneratorStages::Vertex));
+        IShaderStageGenerator &fragmentGenerator(
+            *GetProgramGenerator().GetStage(ShaderGeneratorStages::Fragment));
+
+        vertexGenerator.AddIncoming("attr_pos", "vec3");
+        vertexGenerator.Append("void main() {");
+        vertexGenerator.Append("\tgl_Position = vec4(attr_pos, 1.0);");
+        vertexGenerator.Append("}");
+
+        fragmentGenerator.AddUniform("color", "vec4");
+        fragmentGenerator.Append("void main() {");
+        fragmentGenerator.Append("\tfragOutput = color;\n");
+        fragmentGenerator.Append("}");
+        NVRenderShaderProgram *theShader = GetProgramGenerator().CompileGeneratedShader(
+            "fill rect shader", SShaderCacheProgramFlags(), TShaderFeatureSet());
+        NVScopedRefCounted<SFillRectShader> retval;
+        if (theShader)
+            retval = QT3DS_NEW(m_Context->GetAllocator(), SFillRectShader)(*theShader);
+        m_fillRectShader = retval;
+        return m_fillRectShader.getValue();
+    }
+
     SLayerProgAABlendShader *Qt3DSRendererImpl::GetLayerProgAABlendShader()
     {
         if (m_LayerProgAAShader.hasValue())
diff --git a/src/runtimerender/rendererimpl/Qt3DSRendererImplShaders.h b/src/runtimerender/rendererimpl/Qt3DSRendererImplShaders.h
index 0d77121..50606dd 100644
--- a/src/runtimerender/rendererimpl/Qt3DSRendererImplShaders.h
+++ b/src/runtimerender/rendererimpl/Qt3DSRendererImplShaders.h
@@ -360,6 +360,25 @@ namespace render {
         QT3DS_IMPLEMENT_REF_COUNT_ADDREF_RELEASE(m_Shader.GetRenderContext().GetAllocator())
     };
 
+    struct SFillRectShader
+    {
+        NVRenderShaderProgram &m_Shader;
+
+        NVRenderCachedShaderProperty<QT3DSVec4> m_color;
+        volatile QT3DSI32 mRefCount;
+
+        SFillRectShader(NVRenderShaderProgram &inShader)
+            : m_Shader(inShader)
+            , m_color("color", inShader)
+            , mRefCount(0)
+        {
+            m_Shader.addRef();
+        }
+        ~SFillRectShader() { m_Shader.release(); }
+
+        QT3DS_IMPLEMENT_REF_COUNT_ADDREF_RELEASE(m_Shader.GetRenderContext().GetAllocator())
+    };
+
     struct SShadowmapPreblurShader
     {
         NVRenderShaderProgram &m_Shader;