Fix issues due to changes in camera view matrix

Picking relied on the transform matrix of the camera being the same
as the view matrix. This has changed but picking and lod code was not
update to use the new convention.

Change-Id: I3250d3409d15e78a5c0b56eb3b1e74bce1e22843
Reviewed-by: Sean Harmer <sean.harmer@kdab.com>

5 files changed, 18 insertions, 13 deletions

diff --git a/src/render/backend/cameralens.cpp b/src/render/backend/cameralens.cpp
index 3d0e7fdaf..9870dbf63 100644
--- a/src/render/backend/cameralens.cpp
+++ b/src/render/backend/cameralens.cpp
@@ -68,6 +68,18 @@ void CameraLens::cleanup()
     QBackendNode::setEnabled(false);
 }
 
+QMatrix4x4 CameraLens::viewMatrix(const QMatrix4x4 &worldTransform)
+{
+    const QVector4D position = worldTransform * QVector4D(0.0f, 0.0f, 0.0f, 1.0f);
+    // OpenGL convention is looking down -Z
+    const QVector4D viewDirection = worldTransform * QVector4D(0.0f, 0.0f, -1.0f, 0.0f);
+    const QVector4D upVector = worldTransform * QVector4D(0.0f, 1.0f, 0.0f, 0.0f);
+
+    QMatrix4x4 m;
+    m.lookAt(position.toVector3D(), (position + viewDirection).toVector3D(), upVector.toVector3D());
+    return m;
+}
+
 void CameraLens::initializeFromPeer(const Qt3DCore::QNodeCreatedChangeBasePtr &change)
 {
     const auto typedChange = qSharedPointerCast<Qt3DCore::QNodeCreatedChange<QCameraLensData>>(change);
diff --git a/src/render/backend/cameralens_p.h b/src/render/backend/cameralens_p.h
index 72282a88b..790d0ed64 100644
--- a/src/render/backend/cameralens_p.h
+++ b/src/render/backend/cameralens_p.h
@@ -63,13 +63,15 @@ namespace Render {
 
 class CameraManager;
 
-class CameraLens : public BackendNode
+class QT3DRENDERSHARED_PRIVATE_EXPORT CameraLens : public BackendNode
 {
 public:
     CameraLens();
     ~CameraLens();
     void cleanup();
 
+    QMatrix4x4 viewMatrix(const QMatrix4x4 &worldTransform);
+
     void setProjection(const QMatrix4x4 &projection);
     inline QMatrix4x4 projection() const { return m_projection; }
 
diff --git a/src/render/backend/renderview.cpp b/src/render/backend/renderview.cpp
index 4ee92a54a..27bc0cb0d 100644
--- a/src/render/backend/renderview.cpp
+++ b/src/render/backend/renderview.cpp
@@ -717,16 +717,7 @@ void RenderView::updateMatrices()
 {
     if (m_data.m_renderCameraNode && m_data.m_renderCameraLens && m_data.m_renderCameraLens->isEnabled()) {
         const QMatrix4x4 cameraWorld = *(m_data.m_renderCameraNode->worldTransform());
-
-        const QVector4D position = cameraWorld * QVector4D(0.0f, 0.0f, 0.0f, 1.0f);
-        // OpenGL convention is looking down -Z
-        const QVector4D viewDirection = cameraWorld * QVector4D(0.0f, 0.0f, -1.0f, 0.0f);
-        const QVector4D upVector = cameraWorld * QVector4D(0.0f, 1.0f, 0.0f, 0.0f);
-
-        QMatrix4x4 m;
-        m.lookAt(position.toVector3D(), (position + viewDirection).toVector3D(), upVector.toVector3D());
-
-        setViewMatrix(m);
+        setViewMatrix(m_data.m_renderCameraLens->viewMatrix(cameraWorld));
 
         setViewProjectionMatrix(m_data.m_renderCameraLens->projection() * viewMatrix());
         //To get the eyePosition of the camera, we need to use the inverse of the
diff --git a/src/render/jobs/pickboundingvolumejob.cpp b/src/render/jobs/pickboundingvolumejob.cpp
index 616ee0ca8..0e751a6e0 100644
--- a/src/render/jobs/pickboundingvolumejob.cpp
+++ b/src/render/jobs/pickboundingvolumejob.cpp
@@ -433,7 +433,7 @@ void PickBoundingVolumeJob::viewMatrixForCamera(Qt3DCore::QNodeId cameraId,
     if (camNode != nullptr &&
             (lens = camNode->renderComponent<CameraLens>()) != nullptr &&
             lens->isEnabled()) {
-        viewMatrix = *camNode->worldTransform();
+        viewMatrix = lens->viewMatrix(*camNode->worldTransform());
         projectionMatrix = lens->projection();
     }
 }
diff --git a/src/render/jobs/updatelevelofdetailjob.cpp b/src/render/jobs/updatelevelofdetailjob.cpp
index 24891f9b8..709c73cf1 100644
--- a/src/render/jobs/updatelevelofdetailjob.cpp
+++ b/src/render/jobs/updatelevelofdetailjob.cpp
@@ -104,7 +104,7 @@ bool UpdateLevelOfDetailJob::viewMatrixForCamera(const Qt3DCore::QNodeId &camera
     if (camNode != nullptr &&
             (lens = camNode->renderComponent<CameraLens>()) != nullptr &&
             lens->isEnabled()) {
-        viewMatrix = *camNode->worldTransform();
+        viewMatrix = lens->viewMatrix(*camNode->worldTransform());
         projectionMatrix = lens->projection();
         return true;
     }