Picking: viewport contains check needs to use pos in GL coordinate

The viewport rect is computed in GL coordinates. Therefore we need
to check if the mouse is contained with the position in GL coordinates.
The error is not noticeable unless you are using non full height viewport.

Change-Id: I1c39ff91da4a3649288826b939071a298c83c723
Task-number: QTBUG-72856
Reviewed-by: Mike Krus <mike.krus@kdab.com>

2 files changed, 32 insertions, 5 deletions

diff --git a/src/render/jobs/abstractpickingjob.cpp b/src/render/jobs/abstractpickingjob.cpp
index ccb190cff..74e6a7f80 100644
--- a/src/render/jobs/abstractpickingjob.cpp
+++ b/src/render/jobs/abstractpickingjob.cpp
@@ -122,15 +122,22 @@ RayCasting::QRay3D AbstractPickingJob::rayForViewportAndCamera(const PickingUtil
                                                                const QPoint &pos) const
 {
     static RayCasting::QRay3D invalidRay({}, {}, 0.f);
+
+    if (!vca.area.isValid())
+        return invalidRay;
+
     Matrix4x4 viewMatrix;
     Matrix4x4 projectionMatrix;
     Render::CameraLens::viewMatrixForCamera(m_manager->renderNodesManager(),
                                             vca.cameraId,
                                             viewMatrix,
                                             projectionMatrix);
+    // Returns viewport rect in GL coordinates (y inverted)
     const QRect viewport = windowViewport(vca.area, vca.viewport);
+    // In GL the y is inverted compared to Qt
+    const QPoint glCorrectPos = QPoint(pos.x(), vca.area.height() - pos.y());
 
-    if (vca.area.isValid() && !viewport.contains(pos))
+    if (!viewport.contains(glCorrectPos))
         return invalidRay;
     if (vca.surface) {
         QSurface *surface = nullptr;
@@ -148,8 +155,6 @@ RayCasting::QRay3D AbstractPickingJob::rayForViewportAndCamera(const PickingUtil
             return invalidRay;
     }
 
-    // In GL the y is inverted compared to Qt
-    const QPoint glCorrectPos = QPoint(pos.x(), vca.area.isValid() ? vca.area.height() - pos.y() : pos.y());
     const auto ray = intersectionRay(glCorrectPos, viewMatrix, projectionMatrix, viewport);
     return ray;
 }
diff --git a/tests/manual/picking-qml/main.qml b/tests/manual/picking-qml/main.qml
index c020d5817..5ca50be0b 100644
--- a/tests/manual/picking-qml/main.qml
+++ b/tests/manual/picking-qml/main.qml
@@ -79,7 +79,19 @@ Entity {
         id: camera2
         projectionType: CameraLens.PerspectiveProjection
         fieldOfView: 45
-        aspectRatio: _view.width * 0.5 / _view.height
+        aspectRatio: _view.width * 0.5 / (_view.height * 0.5)
+        nearPlane : 0.1
+        farPlane : 1000.0
+        position: Qt.vector3d( 40.0, 5.0, -20.0 )
+        upVector: Qt.vector3d( 0.0, 1.0, 0.0 )
+        viewCenter: Qt.vector3d( 0.0, 0.0, 0.0 )
+    }
+
+    Camera {
+        id: camera3
+        projectionType: CameraLens.PerspectiveProjection
+        fieldOfView: 45
+        aspectRatio: _view.width * 0.5 / (_view.height * 0.5)
         nearPlane : 0.1
         farPlane : 1000.0
         position: Qt.vector3d( 40.0, 5.0, -20.0 )
@@ -113,7 +125,7 @@ Entity {
                     }
 
                     Viewport {
-                        normalizedRect: Qt.rect(0.5, 0.0, 0.5, 1.0)
+                        normalizedRect: Qt.rect(0.5, 0.0, 0.5, 0.5)
                         CameraSelector {
                             camera: camera2
                             LayerFilter {
@@ -122,6 +134,16 @@ Entity {
                             }
                         }
                     }
+                    Viewport {
+                        normalizedRect: Qt.rect(0.5, 0.5, 0.5, 0.5)
+                        CameraSelector {
+                            camera: camera3
+                            LayerFilter {
+                                // To show Debug volumes
+                                layers: [sceneRoot.contentLayer, sceneRoot.debugLayer]
+                            }
+                        }
+                    }
                 }
             }
         },