Fix multi-view picking

When using Scene3DView, qt3d still has a single scene graph and and a
single framegraph. It automatically creates layers and layer filters
to make sure the right objects are only rendered in the right view.

This affects picking though as it was not aware of layer filters.
This patch collects the filtered layers for each view and makes sure
only entities matching those layers are tested for picking (this uses
code that existed for ray casting).

This changes the behavior of Qt3D though as non rendered objects (ie,
excluded by layer filtering) are no longer pickable. Only way now would
be to provide a shader that just discarded everything.

Note: Scene3DView is not available in Qt6 so on this branch this only
really affects custom scenes with multiple views each showing different
content.

[ChangeLog] Non rendered entities (due to layer filtering) are no
longer pickable

Ticket-number: QTBUG-94214
Change-Id: I8515368e43342b33ac219dff533e92efa72fbe7d
Reviewed-by: Paul Lemire <paul.lemire@kdab.com>
(cherry picked from commit d79376732105dea749e71cdb114251084c7138a9)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

4 files changed, 23 insertions, 2 deletions

diff --git a/src/render/jobs/pickboundingvolumejob.cpp b/src/render/jobs/pickboundingvolumejob.cpp
index 290bb8751..720a7e1d0 100644
--- a/src/render/jobs/pickboundingvolumejob.cpp
+++ b/src/render/jobs/pickboundingvolumejob.cpp
@@ -329,6 +329,7 @@ void PickBoundingVolumeJob::processPickEvent(const PickingUtils::PickConfigurati
         }
 
         PickingUtils::HierarchicalEntityPicker entityPicker(ray);
+        entityPicker.setFilterLayers(vca.layers, vca.layerFilterMode);
         if (entityPicker.collectHits(m_manager, m_node)) {
             if (pickConfiguration.trianglePickingRequested) {
                 PickingUtils::TriangleCollisionGathererFunctor gathererFunctor;
@@ -468,6 +469,7 @@ void PickBoundingVolumeJob::dispatchPickEvents(const QMouseEvent *event,
                 case QEvent::MouseButtonPress: {
                     // Store pressed object handle
                     m_currentPicker = objectPickerHandle;
+                    m_currentViewport = viewportNodeId;
                     // Send pressed event to m_currentPicker
                     d->dispatches.push_back({objectPicker->peerId(), event->type(), pickEvent, viewportNodeId});
                     objectPicker->setPressed(true);
@@ -485,6 +487,7 @@ void PickBoundingVolumeJob::dispatchPickEvents(const QMouseEvent *event,
                                                  PickBoundingVolumeJobPrivate::MouseButtonClick,
                                                  pickEvent, viewportNodeId});
                         m_currentPicker = HObjectPicker();
+                        m_currentViewport = {};
                     }
                     break;
                 }
@@ -529,8 +532,9 @@ void PickBoundingVolumeJob::dispatchPickEvents(const QMouseEvent *event,
         switch (event->type()) {
         case QEvent::MouseButtonRelease: {
             // Send release event to m_currentPicker
-            if (lastCurrentPicker != nullptr) {
+            if (lastCurrentPicker != nullptr && m_currentViewport == viewportNodeId) {
                 m_currentPicker = HObjectPicker();
+                m_currentViewport = {};
                 QPickEventPtr pickEvent(new QPickEvent);
                 lastCurrentPicker->setPressed(false);
                 d->dispatches.push_back({lastCurrentPicker->peerId(), event->type(), pickEvent, viewportNodeId});
diff --git a/src/render/jobs/pickboundingvolumejob_p.h b/src/render/jobs/pickboundingvolumejob_p.h
index f95d4f773..af131b8ca 100644
--- a/src/render/jobs/pickboundingvolumejob_p.h
+++ b/src/render/jobs/pickboundingvolumejob_p.h
@@ -114,6 +114,7 @@ private:
     bool m_pickersDirty;
     bool m_oneHoverAtLeast;
     HObjectPicker m_currentPicker;
+    Qt3DCore::QNodeId m_currentViewport;
     QList<HObjectPicker> m_hoveredPickers;
     QList<HObjectPicker> m_hoveredPickersToClear;
 };
diff --git a/src/render/jobs/pickboundingvolumeutils.cpp b/src/render/jobs/pickboundingvolumeutils.cpp
index d123b99f2..fbd12eb4b 100644
--- a/src/render/jobs/pickboundingvolumeutils.cpp
+++ b/src/render/jobs/pickboundingvolumeutils.cpp
@@ -52,6 +52,7 @@
 #include <Qt3DRender/private/segmentsvisitor_p.h>
 #include <Qt3DRender/private/pointsvisitor_p.h>
 #include <Qt3DRender/private/layer_p.h>
+#include <Qt3DRender/private/layerfilternode_p.h>
 #include <Qt3DRender/private/rendersettings_p.h>
 
 #include <vector>
@@ -131,6 +132,19 @@ ViewportCameraAreaDetails ViewportCameraAreaGatherer::gatherUpViewportCameraArea
                 // prevent picking in the presence of a NoPicking node
                 return {};
             }
+            case FrameGraphNode::LayerFilter: {
+                auto fnode = static_cast<const LayerFilterNode *>(node);
+                const auto &layers = fnode->layerIds();
+                for (const auto &id: layers)
+                    vca.layers.append(id);
+                switch (fnode->filterMode()) {
+                case Qt3DRender::QLayerFilter::AcceptAllMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::AcceptAllMatchingLayers; break;
+                case Qt3DRender::QLayerFilter::AcceptAnyMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::AcceptAnyMatchingLayers; break;
+                case Qt3DRender::QLayerFilter::DiscardAllMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::DiscardAllMatchingLayers; break;
+                case Qt3DRender::QLayerFilter::DiscardAnyMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::DiscardAnyMatchingLayers; break;
+                }
+                break;
+            }
             default:
                 break;
             }
diff --git a/src/render/jobs/pickboundingvolumeutils_p.h b/src/render/jobs/pickboundingvolumeutils_p.h
index bac025dca..d9c8eee22 100644
--- a/src/render/jobs/pickboundingvolumeutils_p.h
+++ b/src/render/jobs/pickboundingvolumeutils_p.h
@@ -84,8 +84,10 @@ struct Q_AUTOTEST_EXPORT ViewportCameraAreaDetails
     QRectF viewport;
     QSize area;
     QSurface *surface = nullptr;
+    Qt3DCore::QNodeIdVector layers;
+    QAbstractRayCaster::FilterMode layerFilterMode = QAbstractRayCaster::AcceptAnyMatchingLayers;
 };
-QT3D_DECLARE_TYPEINFO_3(Qt3DRender, Render, PickingUtils, ViewportCameraAreaDetails, Q_PRIMITIVE_TYPE)
+QT3D_DECLARE_TYPEINFO_3(Qt3DRender, Render, PickingUtils, ViewportCameraAreaDetails, Q_COMPLEX_TYPE)
 
 struct PickConfiguration {
     std::vector<ViewportCameraAreaDetails> vcaDetails;