Fix multi-view picking

When using Scene3DView, qt3d still has a single scene graph and and a
single framegraph. It automatically creates layers and layer filters
to make sure the right objects are only rendered in the right view.

This affects picking though as it was not aware of layer filters.
This patch collects the filtered layers for each view and makes sure
only entities matching those layers are tested for picking (this uses
code that existed for ray casting).

This changes the behavior of Qt3D though as non rendered objects (ie,
excluded by layer filtering) are no longer pickable. Only way now would
be to provide a shader that just discarded everything.

Note: Scene3DView is not available in Qt6 so on this branch this only
really affects custom scenes with multiple views each showing different
content.

[ChangeLog] Non rendered entities (due to layer filtering) are no
longer pickable

Ticket-number: QTBUG-94214
Change-Id: I8515368e43342b33ac219dff533e92efa72fbe7d
Reviewed-by: Paul Lemire <paul.lemire@kdab.com>
(cherry picked from commit d79376732105dea749e71cdb114251084c7138a9)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

7 files changed, 266 insertions, 4 deletions

diff --git a/src/render/jobs/pickboundingvolumejob.cpp b/src/render/jobs/pickboundingvolumejob.cpp
index f2ae6312e..f53a881ab 100644
--- a/src/render/jobs/pickboundingvolumejob.cpp
+++ b/src/render/jobs/pickboundingvolumejob.cpp
@@ -329,6 +329,7 @@ void PickBoundingVolumeJob::processPickEvent(const PickingUtils::PickConfigurati
         }
 
         PickingUtils::HierarchicalEntityPicker entityPicker(ray);
+        entityPicker.setFilterLayers(vca.layers, vca.layerFilterMode);
         if (entityPicker.collectHits(m_manager, m_node)) {
             if (pickConfiguration.trianglePickingRequested) {
                 PickingUtils::TriangleCollisionGathererFunctor gathererFunctor;
@@ -468,6 +469,7 @@ void PickBoundingVolumeJob::dispatchPickEvents(const QMouseEvent *event,
                 case QEvent::MouseButtonPress: {
                     // Store pressed object handle
                     m_currentPicker = objectPickerHandle;
+                    m_currentViewport = viewportNodeId;
                     // Send pressed event to m_currentPicker
                     d->dispatches.push_back({objectPicker->peerId(), event->type(), pickEvent, viewportNodeId});
                     objectPicker->setPressed(true);
@@ -485,6 +487,7 @@ void PickBoundingVolumeJob::dispatchPickEvents(const QMouseEvent *event,
                                                  PickBoundingVolumeJobPrivate::MouseButtonClick,
                                                  pickEvent, viewportNodeId});
                         m_currentPicker = HObjectPicker();
+                        m_currentViewport = {};
                     }
                     break;
                 }
@@ -529,8 +532,9 @@ void PickBoundingVolumeJob::dispatchPickEvents(const QMouseEvent *event,
         switch (event->type()) {
         case QEvent::MouseButtonRelease: {
             // Send release event to m_currentPicker
-            if (lastCurrentPicker != nullptr) {
+            if (lastCurrentPicker != nullptr && m_currentViewport == viewportNodeId) {
                 m_currentPicker = HObjectPicker();
+                m_currentViewport = {};
                 QPickEventPtr pickEvent(new QPickEvent);
                 lastCurrentPicker->setPressed(false);
                 d->dispatches.push_back({lastCurrentPicker->peerId(), event->type(), pickEvent, viewportNodeId});
diff --git a/src/render/jobs/pickboundingvolumejob_p.h b/src/render/jobs/pickboundingvolumejob_p.h
index f95d4f773..af131b8ca 100644
--- a/src/render/jobs/pickboundingvolumejob_p.h
+++ b/src/render/jobs/pickboundingvolumejob_p.h
@@ -114,6 +114,7 @@ private:
     bool m_pickersDirty;
     bool m_oneHoverAtLeast;
     HObjectPicker m_currentPicker;
+    Qt3DCore::QNodeId m_currentViewport;
     QList<HObjectPicker> m_hoveredPickers;
     QList<HObjectPicker> m_hoveredPickersToClear;
 };
diff --git a/src/render/jobs/pickboundingvolumeutils.cpp b/src/render/jobs/pickboundingvolumeutils.cpp
index d123b99f2..fbd12eb4b 100644
--- a/src/render/jobs/pickboundingvolumeutils.cpp
+++ b/src/render/jobs/pickboundingvolumeutils.cpp
@@ -52,6 +52,7 @@
 #include <Qt3DRender/private/segmentsvisitor_p.h>
 #include <Qt3DRender/private/pointsvisitor_p.h>
 #include <Qt3DRender/private/layer_p.h>
+#include <Qt3DRender/private/layerfilternode_p.h>
 #include <Qt3DRender/private/rendersettings_p.h>
 
 #include <vector>
@@ -131,6 +132,19 @@ ViewportCameraAreaDetails ViewportCameraAreaGatherer::gatherUpViewportCameraArea
                 // prevent picking in the presence of a NoPicking node
                 return {};
             }
+            case FrameGraphNode::LayerFilter: {
+                auto fnode = static_cast<const LayerFilterNode *>(node);
+                const auto &layers = fnode->layerIds();
+                for (const auto &id: layers)
+                    vca.layers.append(id);
+                switch (fnode->filterMode()) {
+                case Qt3DRender::QLayerFilter::AcceptAllMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::AcceptAllMatchingLayers; break;
+                case Qt3DRender::QLayerFilter::AcceptAnyMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::AcceptAnyMatchingLayers; break;
+                case Qt3DRender::QLayerFilter::DiscardAllMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::DiscardAllMatchingLayers; break;
+                case Qt3DRender::QLayerFilter::DiscardAnyMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::DiscardAnyMatchingLayers; break;
+                }
+                break;
+            }
             default:
                 break;
             }
diff --git a/src/render/jobs/pickboundingvolumeutils_p.h b/src/render/jobs/pickboundingvolumeutils_p.h
index bac025dca..d9c8eee22 100644
--- a/src/render/jobs/pickboundingvolumeutils_p.h
+++ b/src/render/jobs/pickboundingvolumeutils_p.h
@@ -84,8 +84,10 @@ struct Q_AUTOTEST_EXPORT ViewportCameraAreaDetails
     QRectF viewport;
     QSize area;
     QSurface *surface = nullptr;
+    Qt3DCore::QNodeIdVector layers;
+    QAbstractRayCaster::FilterMode layerFilterMode = QAbstractRayCaster::AcceptAnyMatchingLayers;
 };
-QT3D_DECLARE_TYPEINFO_3(Qt3DRender, Render, PickingUtils, ViewportCameraAreaDetails, Q_PRIMITIVE_TYPE)
+QT3D_DECLARE_TYPEINFO_3(Qt3DRender, Render, PickingUtils, ViewportCameraAreaDetails, Q_COMPLEX_TYPE)
 
 struct PickConfiguration {
     std::vector<ViewportCameraAreaDetails> vcaDetails;
diff --git a/tests/auto/render/pickboundingvolumejob/CMakeLists.txt b/tests/auto/render/pickboundingvolumejob/CMakeLists.txt
index 53097bc60..25491ba9f 100644
--- a/tests/auto/render/pickboundingvolumejob/CMakeLists.txt
+++ b/tests/auto/render/pickboundingvolumejob/CMakeLists.txt
@@ -39,6 +39,7 @@ set(pickboundingvolumejob_resource_files
     "testscene_pickersdisabled.qml"
     "testscene_priorityoverlapping.qml"
     "testscene_viewports.qml"
+    "testscene_layerfilter.qml"
 )
 
 qt_internal_add_resource(tst_pickboundingvolumejob "pickboundingvolumejob"
diff --git a/tests/auto/render/pickboundingvolumejob/testscene_layerfilter.qml b/tests/auto/render/pickboundingvolumejob/testscene_layerfilter.qml
new file mode 100644
index 000000000..392623a6a
--- /dev/null
+++ b/tests/auto/render/pickboundingvolumejob/testscene_layerfilter.qml
@@ -0,0 +1,162 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 Klaralvdalens Datakonsult AB (KDAB).
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt3D module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:BSD$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** BSD License Usage
+** Alternatively, you may use this file under the terms of the BSD license
+** as follows:
+**
+** "Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions are
+** met:
+**   * Redistributions of source code must retain the above copyright
+**     notice, this list of conditions and the following disclaimer.
+**   * Redistributions in binary form must reproduce the above copyright
+**     notice, this list of conditions and the following disclaimer in
+**     the documentation and/or other materials provided with the
+**     distribution.
+**   * Neither the name of The Qt Company Ltd nor the names of its
+**     contributors may be used to endorse or promote products derived
+**     from this software without specific prior written permission.
+**
+**
+** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+** OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+** LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+import Qt3D.Core 2.0
+import Qt3D.Render 2.0
+import Qt3D.Extras 2.0
+import QtQuick.Window 2.0
+
+Entity {
+    id: sceneRoot
+
+    Window {
+        id: _view
+        width: 600
+        height: 600
+        visible: true
+    }
+
+    Camera {
+        id: camera
+        projectionType: CameraLens.PerspectiveProjection
+        fieldOfView: 45
+        aspectRatio: _view.width / 2 / _view.height
+        nearPlane : 0.1
+        farPlane : 1000.0
+        position: Qt.vector3d( 0.0, 0.0, -10.0 )
+        upVector: Qt.vector3d( 0.0, 1.0, 0.0 )
+        viewCenter: Qt.vector3d( 0.0, 0.0, 0.0 )
+    }
+
+    Camera {
+        id: camera2
+        projectionType: CameraLens.PerspectiveProjection
+        fieldOfView: 45
+        aspectRatio: _view.width / 2 / _view.height
+        nearPlane : 0.1
+        farPlane : 1000.0
+        position: Qt.vector3d( 0.0, 0.0, -10.0 )
+        upVector: Qt.vector3d( 0.0, 1.0, 0.0 )
+        viewCenter: Qt.vector3d( 0.0, 0.0, 0.0 )
+    }
+
+    FirstPersonCameraController {
+        camera: camera
+    }
+
+    DirectionalLight {
+        worldDirection: camera.viewVector.times(-1)
+    }
+
+    // Draw 2 viewports
+    components: [
+        RenderSettings {
+            RenderSurfaceSelector {
+                surface: _view
+
+                Viewport {
+                    normalizedRect: Qt.rect(0.0, 0.0, 0.5, 1.0)
+                    ClearBuffers {
+                        buffers : ClearBuffers.ColorDepthBuffer
+                        clearColor: "white"
+                        CameraSelector {
+                            camera: camera
+
+                            LayerFilter {
+                                layers: [ layer1 ]
+                            }
+                        }
+                    }
+                }
+
+                Viewport {
+                    normalizedRect: Qt.rect(0.5, 0.0, 0.5, 1.0)
+                    CameraSelector {
+                        camera: camera2
+
+                        LayerFilter {
+                            layers: [ layer2 ]
+                        }
+                    }
+                }
+            }
+        }
+    ]
+
+    CuboidMesh { id: cubeMesh }
+
+    Entity {
+        readonly property ObjectPicker objectPicker: ObjectPicker {
+            objectName: "Picker1"
+            onClicked: console.log("o1")
+        }
+        readonly property Transform transform: Transform {
+            scale: 4
+        }
+        readonly property PhongMaterial material: PhongMaterial { diffuse: "red" }
+        readonly property Layer layer: Layer { id: layer1 }
+
+        components: [cubeMesh, transform, material, objectPicker, layer ]
+    }
+
+    Entity {
+        readonly property ObjectPicker objectPicker: ObjectPicker {
+            objectName: "Picker2"
+            onClicked: console.log("o2")
+        }
+        readonly property Transform transform: Transform {
+            scale: 3
+        }
+        readonly property PhongMaterial material: PhongMaterial { diffuse: "green" }
+        readonly property Layer layer: Layer { id: layer2 }
+
+        components: [cubeMesh, transform, material, objectPicker, layer ]
+    }
+}
diff --git a/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp b/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
index a6f47042d..0f942510c 100644
--- a/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
+++ b/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
@@ -300,7 +300,6 @@ private:
     }
 
 private Q_SLOTS:
-
     void viewportCameraAreaGather()
     {
         // GIVEN
@@ -1578,6 +1577,86 @@ private Q_SLOTS:
         }
     }
 
+    void checkPickerAndLayerFilters()
+    {
+        // GIVEN
+        QmlSceneReader sceneReader(QUrl("qrc:/testscene_layerfilter.qml"));
+        QScopedPointer<Qt3DCore::QNode> root(qobject_cast<Qt3DCore::QNode *>(sceneReader.root()));
+        QVERIFY(root);
+
+        QScopedPointer<Qt3DRender::TestAspect> test(new Qt3DRender::TestAspect(root.data()));
+        TestArbiter arbiter;
+
+        // Runs Required jobs
+        runRequiredJobs(test.data());
+
+        // THEN
+        // object partially obscured by another viewport, make sure only visible portion is pickable
+        QList<Qt3DRender::QObjectPicker *> pickers = root->findChildren<Qt3DRender::QObjectPicker *>();
+        QCOMPARE(pickers.size(), 2);
+
+        Qt3DRender::QObjectPicker *picker1 = pickers.front();
+        QCOMPARE(picker1->objectName(), QLatin1String("Picker1"));
+
+        Qt3DRender::Render::ObjectPicker *backendPicker1 = test->nodeManagers()->objectPickerManager()->lookupResource(picker1->id());
+        QVERIFY(backendPicker1);
+
+        QSignalSpy mouseButtonPressedSpy1(picker1, &Qt3DRender::QObjectPicker::pressed);
+
+        QVERIFY(mouseButtonPressedSpy1.isValid());
+
+        Qt3DRender::QObjectPicker *picker2 = pickers.last();
+        QCOMPARE(picker2->objectName(), QLatin1String("Picker2"));
+
+        Qt3DRender::Render::ObjectPicker *backendPicker2 = test->nodeManagers()->objectPickerManager()->lookupResource(picker2->id());
+        QVERIFY(backendPicker2);
+
+        QSignalSpy mouseButtonPressedSpy2(picker2, &Qt3DRender::QObjectPicker::pressed);
+
+        QVERIFY(mouseButtonPressedSpy2.isValid());
+
+        // WHEN -> Pressed on object in vp1
+        Qt3DRender::Render::PickBoundingVolumeJob pickBVJob;
+        initializePickBoundingVolumeJob(&pickBVJob, test.data());
+
+        {
+            auto event = QMouseEvent(QMouseEvent::MouseButtonPress, QPointF(150., 300.),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event);
+            bool earlyReturn = !pickBVJob.runHelper();
+            Qt3DCore::QAspectJobPrivate::get(&pickBVJob)->postFrame(test->aspectManager());
+
+            // THEN -> Pressed
+            QVERIFY(!earlyReturn);
+            QVERIFY(backendPicker1->isPressed());
+            QVERIFY(picker1->isPressed());
+            QVERIFY(!backendPicker2->isPressed());
+            QVERIFY(!picker2->isPressed());
+            QCOMPARE(mouseButtonPressedSpy1.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy2.count(), 0);
+
+            auto event2 = QMouseEvent(QMouseEvent::MouseButtonRelease, QPointF(150., 300.),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event2);
+            pickBVJob.runHelper();
+        }
+
+        {
+            auto event = QMouseEvent(QMouseEvent::MouseButtonPress, QPointF(450., 300.),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event);
+            bool earlyReturn = !pickBVJob.runHelper();
+            Qt3DCore::QAspectJobPrivate::get(&pickBVJob)->postFrame(test->aspectManager());
+
+            // THEN -> Nothing happened
+            QVERIFY(!earlyReturn);
+            QVERIFY(backendPicker2->isPressed());
+            QVERIFY(picker2->isPressed());
+            QCOMPARE(mouseButtonPressedSpy1.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy2.count(), 1);
+        }
+    }
+
     void checkMultipleRayDirections_data()
     {
         QTest::addColumn<QVector3D>("cameraOrigin");
@@ -1910,7 +1989,6 @@ private Q_SLOTS:
         QCOMPARE(vca.cameraId, camera->id());
         QCOMPARE(vca.viewport, QRectF(0., 0., 1., 1.));
     }
-
 };
 
 QTEST_MAIN(tst_PickBoundingVolumeJob)