Fix OpenGL runtime picking with scaled camera

When camera scale property has been modified, picking scale needs
to be inverted or it scales into wrong direction. This affects
both runtime viewer and editor in scene camera view.

Task-number: QT3DS-3393
Change-Id: Ib643af22861b81fb0fc9e78ebfd2d724e765eff5
Reviewed-by: Tomi Korpipää <tomi.korpipaa@qt.io>
Reviewed-by: Miikka Heikkinen <miikka.heikkinen@qt.io>
Reviewed-by: Antti Määttä <antti.maatta@qt.io>

9 files changed, 44 insertions, 9 deletions

diff --git a/src/Authoring/Studio/Render/IStudioRenderer.h b/src/Authoring/Studio/Render/IStudioRenderer.h
index 3b3068c3..fc543a89 100644
--- a/src/Authoring/Studio/Render/IStudioRenderer.h
+++ b/src/Authoring/Studio/Render/IStudioRenderer.h
@@ -57,6 +57,7 @@ public:
 
     virtual void SetViewRect(const QRect &inRect) = 0;
     virtual void setFullSizePreview(bool enabled) = 0;
+    virtual void setIsSceneCameraView(bool sceneCameraView) = 0;
 
     virtual void GetEditCameraList(QStringList &outCameras) = 0;
     virtual void SetPolygonFillModeEnabled(bool inEnableFillMode) = 0;
diff --git a/src/Authoring/Studio/Render/StudioRenderer.cpp b/src/Authoring/Studio/Render/StudioRenderer.cpp
index ee03a614..dde2d64c 100644
--- a/src/Authoring/Studio/Render/StudioRenderer.cpp
+++ b/src/Authoring/Studio/Render/StudioRenderer.cpp
@@ -377,6 +377,11 @@ struct SRendererImpl : public IStudioRenderer,
         m_fullSizePreview = enabled;
     }
 
+    void setIsSceneCameraView(bool sceneCameraView) override
+    {
+        m_RenderContext->GetRenderContext().setIsSceneCameraView(sceneCameraView);
+    }
+
     // Request that this object renders.  May be ignored if a transaction
     // is ongoing so we don't get multiple rendering per transaction.
     void RequestRender() override
diff --git a/src/Authoring/Studio/UI/PlayerContainerWnd.cpp b/src/Authoring/Studio/UI/PlayerContainerWnd.cpp
index 68cbf78d..e077cf9e 100644
--- a/src/Authoring/Studio/UI/PlayerContainerWnd.cpp
+++ b/src/Authoring/Studio/UI/PlayerContainerWnd.cpp
@@ -245,6 +245,10 @@ void CPlayerContainerWnd::SetViewMode(EViewMode inViewMode)
 {
     m_ViewMode = inViewMode;
     m_SceneView->recheckSizingMode();
+
+    // When viewmode changes, update scene camera view mode status
+    // into renderer context
+    g_StudioApp.getRenderer().setIsSceneCameraView(IsDeploymentView());
 }
 
 //==============================================================================
diff --git a/src/Runtime/Source/render/Qt3DSRenderContext.h b/src/Runtime/Source/render/Qt3DSRenderContext.h
index e5b51341..dfaf1bb9 100644
--- a/src/Runtime/Source/render/Qt3DSRenderContext.h
+++ b/src/Runtime/Source/render/Qt3DSRenderContext.h
@@ -152,9 +152,11 @@ namespace render {
         virtual bool IsAdvancedBlendHwSupportedKHR() const = 0;
         virtual bool IsStandardDerivativesSupported() const = 0;
         virtual bool IsTextureLodSupported() const = 0;
+        virtual bool isSceneCameraView() const = 0;
 
         virtual void SetDefaultRenderTarget(QT3DSU64 targetID) = 0;
         virtual void SetDefaultDepthBufferBitCount(QT3DSI32 depthBits) = 0;
+        virtual void setIsSceneCameraView(bool sceneCameraView) = 0;
 
         virtual NVRenderDepthStencilState *
         CreateDepthStencilState(bool enableDepth, bool depthMask, NVRenderBoolOp::Enum depthFunc,
@@ -481,6 +483,7 @@ namespace render {
                                             ///never render to a window directly (GL only)
         QT3DSI32 m_DephBits; ///< this is the depth bits count of the default window render target
         QT3DSI32 m_StencilBits; ///< this is the stencil bits count of the default window render target
+        bool m_isSceneCameraView = true; ///< true in viewer and in editor when in scene camera view mode
 
     protected:
         volatile QT3DSI32 mRefCount;
@@ -777,6 +780,11 @@ namespace render {
             return GetRenderBackendCap(NVRenderBackend::NVRenderBackendCaps::TextureLod);
         }
 
+        bool isSceneCameraView() const override
+        {
+            return m_isSceneCameraView;
+        }
+
         void SetDefaultRenderTarget(QT3DSU64 targetID) override
         {
             m_DefaultOffscreenRenderTarget =
@@ -785,6 +793,10 @@ namespace render {
 
         void SetDefaultDepthBufferBitCount(QT3DSI32 depthBits) override { m_DephBits = depthBits; }
 
+        void setIsSceneCameraView(bool sceneCameraView) override {
+            m_isSceneCameraView = sceneCameraView;
+        }
+
         virtual NVRenderDepthStencilState *
         CreateDepthStencilState(bool enableDepth, bool depthMask, NVRenderBoolOp::Enum depthFunc,
                                 bool enableStencil,
diff --git a/src/Runtime/Source/runtimerender/graphobjects/Qt3DSRenderCamera.cpp b/src/Runtime/Source/runtimerender/graphobjects/Qt3DSRenderCamera.cpp
index 062a36c5..b9f9c200 100644
--- a/src/Runtime/Source/runtimerender/graphobjects/Qt3DSRenderCamera.cpp
+++ b/src/Runtime/Source/runtimerender/graphobjects/Qt3DSRenderCamera.cpp
@@ -414,7 +414,7 @@ void SCamera::SetupOrthographicCameraForOffscreenRender(NVRenderTexture2D &inTex
 }
 
 SRay SCamera::Unproject(const QT3DSVec2 &inViewportRelativeCoords, const NVRenderRectF &inViewport,
-                        const QT3DSVec2 &inDesignDimensions) const
+                        const QT3DSVec2 &inDesignDimensions, bool sceneCameraView) const
 {
     SRay theRay;
     QT3DSMat44 tempVal(QT3DSMat44::createIdentity());
@@ -448,8 +448,18 @@ SRay SCamera::Unproject(const QT3DSVec2 &inViewportRelativeCoords, const NVRende
     }
 
     outOrigin = m_GlobalTransform.transform(outOrigin);
-    QT3DSMat33 theNormalMatrix;
-    CalculateNormalMatrix(theNormalMatrix);
+
+    // CalculateNormalMatrix(), but 4x4 matrix to have scale() method
+    QT3DSMat44 theNormalMatrix = m_GlobalTransform.getInverse().getTranspose();
+    if (sceneCameraView) {
+        // When in scene camera view mode, camera scale needs to be inverted.
+        // See QT3DS-3393.
+        const float scaleX = m_GlobalTransform[0][0] / theNormalMatrix[0][0];
+        const float scaleY = m_GlobalTransform[1][1] / theNormalMatrix[1][1];
+        const float scaleZ = m_GlobalTransform[2][2] / theNormalMatrix[2][2];
+        QT3DSVec4 scaleVector(scaleX, scaleY, scaleZ, 1.0);
+        theNormalMatrix.scale(scaleVector);
+    }
 
     outDir = theNormalMatrix.transform(outDir);
     outDir.normalize();
diff --git a/src/Runtime/Source/runtimerender/graphobjects/Qt3DSRenderCamera.h b/src/Runtime/Source/runtimerender/graphobjects/Qt3DSRenderCamera.h
index d6d8ce82..be8d53e8 100644
--- a/src/Runtime/Source/runtimerender/graphobjects/Qt3DSRenderCamera.h
+++ b/src/Runtime/Source/runtimerender/graphobjects/Qt3DSRenderCamera.h
@@ -161,7 +161,7 @@ namespace render {
         // Unproject a point (x,y) in viewport relative coordinates meaning
         // left, bottom is 0,0 and values are increasing right,up respectively.
         SRay Unproject(const QT3DSVec2 &inLayerRelativeMouseCoords, const NVRenderRectF &inViewport,
-                       const QT3DSVec2 &inDesignDimensions) const;
+                       const QT3DSVec2 &inDesignDimensions, bool sceneCameraView = false) const;
 
         // Unproject a given coordinate to a 3d position that lies on the same camera
         // plane as inGlobalPos.
diff --git a/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp b/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp
index b0af9884..5b07b25a 100644
--- a/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp
+++ b/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImpl.cpp
@@ -1310,9 +1310,10 @@ namespace render {
         bool wasRenderToTarget(inLayerRenderData.m_Layer.m_Flags.IsLayerRenderToTarget());
         if (wasRenderToTarget && inLayerRenderData.m_Camera != NULL) {
             Option<SRay> theHitRay;
-            if (inLayerRenderData.m_LayerPrepResult.hasValue())
+            if (inLayerRenderData.m_LayerPrepResult.hasValue()) {
                 theHitRay = inLayerRenderData.m_LayerPrepResult->GetPickRay(
-                    inPresCoords, inViewportDimensions, false);
+                    inPresCoords, inViewportDimensions, false, m_Context->isSceneCameraView());
+            }
             if (inLayerRenderData.m_LastFrameOffscreenRenderer.mPtr == NULL) {
                 if (theHitRay.hasValue()) {
                     // Scale the mouse coords to change them into the camera's numerical space.
diff --git a/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderHelper.cpp b/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderHelper.cpp
index 8a050db0..07ee5132 100644
--- a/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderHelper.cpp
+++ b/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderHelper.cpp
@@ -238,7 +238,8 @@ Option<QT3DSVec2> SLayerRenderHelper::GetLayerMouseCoords(const QT3DSVec2 &inMou
 
 Option<SRay> SLayerRenderHelper::GetPickRay(const QT3DSVec2 &inMouseCoords,
                                             const QT3DSVec2 &inWindowDimensions,
-                                            bool inForceIntersect) const
+                                            bool inForceIntersect,
+                                            bool sceneCameraView) const
 {
     if (m_Camera == NULL)
         return Empty();
@@ -248,7 +249,8 @@ Option<SRay> SLayerRenderHelper::GetPickRay(const QT3DSVec2 &inMouseCoords,
         // The cameras projection is different if we are onscreen vs. offscreen.
         // When offscreen, we need to move the mouse coordinates into a local space
         // to the layer.
-        return m_Camera->Unproject(*theCoords, m_Viewport, m_PresentationDesignDimensions);
+        return m_Camera->Unproject(*theCoords, m_Viewport, m_PresentationDesignDimensions,
+                                   sceneCameraView);
     }
     return Empty();
 }
diff --git a/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderHelper.h b/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderHelper.h
index 8ee4ff73..616ebe1a 100644
--- a/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderHelper.h
+++ b/src/Runtime/Source/runtimerender/rendererimpl/Qt3DSRendererImplLayerRenderHelper.h
@@ -100,7 +100,7 @@ namespace render {
                                            bool inForceIntersect) const;
 
         Option<SRay> GetPickRay(const QT3DSVec2 &inMouseCoords, const QT3DSVec2 &inWindowDimensions,
-                                bool inForceIntersect) const;
+                                bool inForceIntersect, bool sceneCameraView = false) const;
 
         // Checks the various viewports and determines if the layer is visible or not.
         bool IsLayerVisible() const;