Fix picking issues when camera has a large far clip

The default QVector3D::unproject() implementation checks for
qFuzzyIsNull on w of the obj vector and inexplicably changes the value
to 1 in that case, which results in completely wrong unprojection.
Fixed by reimplementing unproject without qFuzzyIsNull check.

Task-number: QT3DS-3062
Change-Id: I2d3dea4f5e85f7fcaf576cef04dbff4fe9e3ad22
Reviewed-by: Antti Määttä <antti.maatta@qt.io>
Reviewed-by: Andy Nichols <andy.nichols@qt.io>
Reviewed-by: Laszlo Agocs <laszlo.agocs@qt.io>

1 files changed, 28 insertions, 2 deletions

diff --git a/src/runtime/q3dsinputmanager.cpp b/src/runtime/q3dsinputmanager.cpp
index 323b2a3..4547956 100644
--- a/src/runtime/q3dsinputmanager.cpp
+++ b/src/runtime/q3dsinputmanager.cpp
@@ -141,6 +141,32 @@ QMatrix4x4 calculateCameraViewMatrix(const QMatrix4x4 &cameraWorldTransform)
              QVector3D(upVector));
     return QMatrix4x4(m);
 }
+
+// A copy of QVector3D::unproject with the difference that if obj.w() is nearly zero, we don't
+// set it to one as that is extremely wrong, at least for our purposes.
+// For determining pick rays, nearly zero values are good enough even if they may not result in
+// completely pixel-accurate picks.
+// This allows much larger far clip values to be used in cameras before picking breaks.
+QVector3D unproject(const QVector3D &vector,
+                    const QMatrix4x4 &modelView, const QMatrix4x4 &projection,
+                    const QRect &viewport)
+{
+    QMatrix4x4 inverse = QMatrix4x4( projection * modelView ).inverted();
+
+    QVector4D tmp(vector, 1.0f);
+    tmp.setX((tmp.x() - float(viewport.x())) / float(viewport.width()));
+    tmp.setY((tmp.y() - float(viewport.y())) / float(viewport.height()));
+    tmp = tmp * 2.0f - QVector4D(1.0f, 1.0f, 1.0f, 1.0f);
+
+    QVector4D obj = inverse * tmp;
+    // Don't change the w unless it is actually zero
+    if (obj.w() == 0.f)
+        obj.setW(0.000000001f);
+    obj /= obj.w();
+
+    return obj.toVector3D();
+}
+
 }
 
 void Q3DSInputManager::castRayIntoLayer(Q3DSLayerNode *layer, const QPointF &pos,
@@ -176,9 +202,9 @@ void Q3DSInputManager::castRayIntoLayer(Q3DSLayerNode *layer, const QPointF &pos
     QRect viewport(-1, -1, 2, 2);
 
     QVector3D nearPos(float(pos.x()), float(pos.y()), 0.0f);
-    nearPos = nearPos.unproject(viewMatrix, projectionMatrix, viewport);
+    nearPos = unproject(nearPos, viewMatrix, projectionMatrix, viewport);
     QVector3D farPos(float(pos.x()), float(pos.y()), 1.0f);
-    farPos = farPos.unproject(viewMatrix, projectionMatrix, viewport);
+    farPos = unproject(farPos, viewMatrix, projectionMatrix, viewport);
 
     QVector3D origin(nearPos);
     QVector3D direction((farPos - nearPos).normalized());