Picking: reuse LayerFilterJob to perform layer filtering

This fixes picking for cases where multiple LayerFilters using different
filter modes are present in the FrameGraph. This also reduces code
duplication.

Pick-to: 6.2 6.1 5.15
Change-Id: I19d0c2c5777930820ab950cbf2bfe08ef7d2484f
Reviewed-by: Mike Krus <mike.krus@kdab.com>

12 files changed, 617 insertions, 98 deletions

diff --git a/src/render/frontend/qrenderaspect.cpp b/src/render/frontend/qrenderaspect.cpp
index e742b1555..2bc31506f 100644
--- a/src/render/frontend/qrenderaspect.cpp
+++ b/src/render/frontend/qrenderaspect.cpp
@@ -318,7 +318,9 @@ QRenderAspectPrivate::QRenderAspectPrivate(QRenderAspect::SubmissionType submiss
     m_expandBoundingVolumeJob->addDependency(m_updateWorldBoundingVolumeJob);
     m_updateLevelOfDetailJob->addDependency(m_expandBoundingVolumeJob);
     m_pickBoundingVolumeJob->addDependency(m_expandBoundingVolumeJob);
+    m_pickBoundingVolumeJob->addDependency(m_updateEntityLayersJob);
     m_rayCastingJob->addDependency(m_expandBoundingVolumeJob);
+    m_rayCastingJob->addDependency(m_updateEntityLayersJob);
 }
 
 /*! \internal */
diff --git a/src/render/jobs/filterlayerentityjob.cpp b/src/render/jobs/filterlayerentityjob.cpp
index ddea1ebf7..c7db55c53 100644
--- a/src/render/jobs/filterlayerentityjob.cpp
+++ b/src/render/jobs/filterlayerentityjob.cpp
@@ -75,6 +75,33 @@ void FilterLayerEntityJob::run()
     std::sort(m_filteredEntities.begin(), m_filteredEntities.end());
 }
 
+void FilterLayerEntityJob::filterEntityAgainstLayers(Entity *entity,
+                                                     const Qt3DCore::QNodeIdVector &layerIds,
+                                                     const QLayerFilter::FilterMode filterMode)
+{
+    // Perform filtering
+    switch (filterMode) {
+    case QLayerFilter::AcceptAnyMatchingLayers: {
+        filterAcceptAnyMatchingLayers(entity, layerIds);
+        break;
+    }
+    case QLayerFilter::AcceptAllMatchingLayers: {
+        filterAcceptAllMatchingLayers(entity, layerIds);
+        break;
+    }
+    case QLayerFilter::DiscardAnyMatchingLayers: {
+        filterDiscardAnyMatchingLayers(entity, layerIds);
+        break;
+    }
+    case QLayerFilter::DiscardAllMatchingLayers: {
+        filterDiscardAllMatchingLayers(entity, layerIds);
+        break;
+    }
+    default:
+        Q_UNREACHABLE();
+    }
+}
+
 // We accept the entity if it contains any of the layers that are in the layer filter
 void FilterLayerEntityJob::filterAcceptAnyMatchingLayers(Entity *entity,
                                                          const Qt3DCore::QNodeIdVector &layerIds)
@@ -181,28 +208,8 @@ void FilterLayerEntityJob::filterLayerAndEntity()
         const QLayerFilter::FilterMode filterMode = layerFilter->filterMode();
 
         // Perform filtering
-        for (Entity *entity : entitiesToFilter) {
-            switch (filterMode) {
-            case QLayerFilter::AcceptAnyMatchingLayers: {
-                filterAcceptAnyMatchingLayers(entity, layerIds);
-                break;
-            }
-            case QLayerFilter::AcceptAllMatchingLayers: {
-                filterAcceptAllMatchingLayers(entity, layerIds);
-                break;
-            }
-            case QLayerFilter::DiscardAnyMatchingLayers: {
-                filterDiscardAnyMatchingLayers(entity, layerIds);
-                break;
-            }
-            case QLayerFilter::DiscardAllMatchingLayers: {
-                filterDiscardAllMatchingLayers(entity, layerIds);
-                break;
-            }
-            default:
-                Q_UNREACHABLE();
-            }
-        }
+        for (Entity *entity : entitiesToFilter)
+            filterEntityAgainstLayers(entity, layerIds, filterMode);
 
         // Entities to filter for the next frame are the filtered result of the
         // current LayerFilter
diff --git a/src/render/jobs/filterlayerentityjob_p.h b/src/render/jobs/filterlayerentityjob_p.h
index cdd309443..691e71c4b 100644
--- a/src/render/jobs/filterlayerentityjob_p.h
+++ b/src/render/jobs/filterlayerentityjob_p.h
@@ -80,6 +80,7 @@ public:
     // QAspectJob interface
     void run() final;
 
+    void filterEntityAgainstLayers(Entity *entity, const Qt3DCore::QNodeIdVector &layerIds, const QLayerFilter::FilterMode filterMode);
     void filterAcceptAnyMatchingLayers(Entity *entity, const Qt3DCore::QNodeIdVector &layerIds);
     void filterAcceptAllMatchingLayers(Entity *entity, const Qt3DCore::QNodeIdVector &layerIds);
     void filterDiscardAnyMatchingLayers(Entity *entity, const Qt3DCore::QNodeIdVector &layerIds);
diff --git a/src/render/jobs/pickboundingvolumejob.cpp b/src/render/jobs/pickboundingvolumejob.cpp
index 720a7e1d0..c3e3cb9bf 100644
--- a/src/render/jobs/pickboundingvolumejob.cpp
+++ b/src/render/jobs/pickboundingvolumejob.cpp
@@ -329,7 +329,8 @@ void PickBoundingVolumeJob::processPickEvent(const PickingUtils::PickConfigurati
         }
 
         PickingUtils::HierarchicalEntityPicker entityPicker(ray);
-        entityPicker.setFilterLayers(vca.layers, vca.layerFilterMode);
+        entityPicker.setLayerFilterIds(vca.layersFilters);
+
         if (entityPicker.collectHits(m_manager, m_node)) {
             if (pickConfiguration.trianglePickingRequested) {
                 PickingUtils::TriangleCollisionGathererFunctor gathererFunctor;
diff --git a/src/render/jobs/pickboundingvolumeutils.cpp b/src/render/jobs/pickboundingvolumeutils.cpp
index fbd12eb4b..10536c3c2 100644
--- a/src/render/jobs/pickboundingvolumeutils.cpp
+++ b/src/render/jobs/pickboundingvolumeutils.cpp
@@ -54,6 +54,7 @@
 #include <Qt3DRender/private/layer_p.h>
 #include <Qt3DRender/private/layerfilternode_p.h>
 #include <Qt3DRender/private/rendersettings_p.h>
+#include <Qt3DRender/private/filterlayerentityjob_p.h>
 
 #include <vector>
 #include <algorithm>
@@ -134,15 +135,7 @@ ViewportCameraAreaDetails ViewportCameraAreaGatherer::gatherUpViewportCameraArea
             }
             case FrameGraphNode::LayerFilter: {
                 auto fnode = static_cast<const LayerFilterNode *>(node);
-                const auto &layers = fnode->layerIds();
-                for (const auto &id: layers)
-                    vca.layers.append(id);
-                switch (fnode->filterMode()) {
-                case Qt3DRender::QLayerFilter::AcceptAllMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::AcceptAllMatchingLayers; break;
-                case Qt3DRender::QLayerFilter::AcceptAnyMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::AcceptAnyMatchingLayers; break;
-                case Qt3DRender::QLayerFilter::DiscardAllMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::DiscardAllMatchingLayers; break;
-                case Qt3DRender::QLayerFilter::DiscardAnyMatchingLayers: vca.layerFilterMode = Qt3DRender::QAbstractRayCaster::DiscardAnyMatchingLayers; break;
-                }
+                vca.layersFilters.push_back(fnode->peerId());
                 break;
             }
             default:
@@ -179,7 +172,8 @@ bool ViewportCameraAreaGatherer::isUnique(const std::vector<ViewportCameraAreaDe
         if (vca.cameraId == listItem.cameraId &&
                 vca.viewport == listItem.viewport &&
                 vca.surface == listItem.surface &&
-                vca.area == listItem.area)
+                vca.area == listItem.area &&
+                vca.layersFilters == listItem.layersFilters)
             return false;
     }
     return true;
@@ -775,16 +769,19 @@ HitList PointCollisionGathererFunctor::pick(const Entity *entity) const
 HierarchicalEntityPicker::HierarchicalEntityPicker(const QRay3D &ray, bool requireObjectPicker)
     : m_ray(ray)
     , m_objectPickersRequired(requireObjectPicker)
-    , m_filterMode(QAbstractRayCaster::AcceptAnyMatchingLayers)
 {
+}
 
+void HierarchicalEntityPicker::setLayerFilterIds(const Qt3DCore::QNodeIdVector &layerFilterIds)
+{
+    m_layerFilterIds = layerFilterIds;
 }
 
-void HierarchicalEntityPicker::setFilterLayers(const Qt3DCore::QNodeIdVector &layerIds, QAbstractRayCaster::FilterMode mode)
+void HierarchicalEntityPicker::setLayerIds(const Qt3DCore::QNodeIdVector &layerIds,
+                                           QAbstractRayCaster::FilterMode mode)
 {
-    m_filterMode = mode;
     m_layerIds = layerIds;
-    std::sort(m_layerIds.begin(), m_layerIds.end());
+    m_layerFilterMode = mode;
 }
 
 bool HierarchicalEntityPicker::collectHits(NodeManagers *manager, Entity *root)
@@ -797,60 +794,37 @@ bool HierarchicalEntityPicker::collectHits(NodeManagers *manager, Entity *root)
     struct EntityData {
         Entity* entity;
         bool hasObjectPicker;
-        Qt3DCore::QNodeIdVector recursiveLayers;
         int priority;
     };
     std::vector<EntityData> worklist;
-    worklist.push_back({root, !root->componentHandle<ObjectPicker>().isNull(), {}, 0});
-
-    LayerManager *layerManager = manager->layerManager();
+    worklist.push_back({root, !root->componentHandle<ObjectPicker>().isNull(), 0});
+
+    // Record all entities that satisfy layerFiltering. We can then check against
+    // that to see if a picked Entity also satisfies the layer filtering
+
+    // Note: PickBoundingVolumeJob filters against LayerFilter nodes (FG) whereas
+    // the RayCastingJob filters only against a set of Layers and a filter Mode
+    const bool hasLayerFilters = m_layerFilterIds.size() > 0;
+    const bool hasLayers = m_layerIds.size() > 0;
+    const bool hasLayerFiltering = hasLayerFilters || hasLayers;
+    std::vector<Entity *> layerFilterEntities;
+    FilterLayerEntityJob layerFilterJob;
+    layerFilterJob.setManager(manager);
+
+    if (hasLayerFilters) {
+        // Note: we expect UpdateEntityLayersJob was called beforehand to handle layer recursivness
+        // Filtering against LayerFilters (PickBoundingVolumeJob)
+        if (m_layerFilterIds.size()) {
+            layerFilterJob.setLayerFilters(m_layerFilterIds);
+            layerFilterJob.run();
+            layerFilterEntities = layerFilterJob.filteredEntities();
+        }
+    }
 
     while (!worklist.empty()) {
         EntityData current = worklist.back();
         worklist.pop_back();
 
-        bool accepted = true;
-        if (m_layerIds.size()) {
-            // TODO investigate reusing logic from LayerFilter job
-            Qt3DCore::QNodeIdVector filterLayers = current.recursiveLayers + current.entity->componentsUuid<Layer>();
-
-            // remove disabled layers
-            filterLayers.erase(std::remove_if(filterLayers.begin(), filterLayers.end(),
-                                              [layerManager](const Qt3DCore::QNodeId layerId) {
-                Layer *layer = layerManager->lookupResource(layerId);
-                return !layer || !layer->isEnabled();
-            }), filterLayers.end());
-
-            std::sort(filterLayers.begin(), filterLayers.end());
-
-            Qt3DCore::QNodeIdVector commonIds;
-            std::set_intersection(m_layerIds.cbegin(), m_layerIds.cend(),
-                                  filterLayers.cbegin(), filterLayers.cend(),
-                                  std::back_inserter(commonIds));
-
-            switch (m_filterMode) {
-            case QAbstractRayCaster::AcceptAnyMatchingLayers: {
-                accepted = !commonIds.empty();
-                break;
-            }
-            case QAbstractRayCaster::AcceptAllMatchingLayers: {
-                accepted = commonIds == m_layerIds;
-                break;
-            }
-            case QAbstractRayCaster::DiscardAnyMatchingLayers: {
-                accepted = commonIds.empty();
-                break;
-            }
-            case QAbstractRayCaster::DiscardAllMatchingLayers: {
-                accepted = !(commonIds == m_layerIds);
-                break;
-            }
-            default:
-                Q_UNREACHABLE();
-                break;
-            }
-        }
-
         // first pick entry sub-scene-graph
         QCollisionQueryResult::Hit queryResult =
                 rayCasting.query(m_ray, current.entity->worldBoundingVolumeWithChildren());
@@ -859,21 +833,27 @@ bool HierarchicalEntityPicker::collectHits(NodeManagers *manager, Entity *root)
 
         // if we get a hit, we check again for this specific entity
         queryResult = rayCasting.query(m_ray, current.entity->worldBoundingVolume());
-        if (accepted && queryResult.m_distance >= 0.f && (current.hasObjectPicker || !m_objectPickersRequired)) {
+
+        // Check Entity is in selected Layers if we have LayerIds or LayerFilterIds
+        // Note: it's not because a parent doesn't satisfy the layerFiltering that a child might not.
+        // Therefore we need to keep traversing children in all cases
+
+        // Are we filtering against layerIds (RayCastingJob)
+        if (hasLayers) {
+            // QLayerFilter::FilterMode and QAbstractRayCaster::FilterMode are the same
+            layerFilterJob.filterEntityAgainstLayers(current.entity, m_layerIds, static_cast<QLayerFilter::FilterMode>(m_layerFilterMode));
+            layerFilterEntities = layerFilterJob.filteredEntities();
+        }
+
+        const bool isInLayers = !hasLayerFiltering || Qt3DCore::contains(layerFilterEntities, current.entity);
+
+        if (isInLayers && queryResult.m_distance >= 0.f && (current.hasObjectPicker || !m_objectPickersRequired)) {
             m_entities.push_back(current.entity);
             m_hits.push_back(queryResult);
             // Record entry for entity/priority
             m_entityToPriorityTable.insert(current.entity->peerId(), current.priority);
         }
 
-        Qt3DCore::QNodeIdVector recursiveLayers;
-        const Qt3DCore::QNodeIdVector entityLayers = current.entity->componentsUuid<Layer>();
-        for (const Qt3DCore::QNodeId &layerId : entityLayers) {
-            Layer *layer = layerManager->lookupResource(layerId);
-            if (layer->recursive())
-                recursiveLayers << layerId;
-        }
-
         // and pick children
         const auto &childrenHandles = current.entity->childrenHandles();
         for (const HEntity &handle : childrenHandles) {
@@ -881,7 +861,6 @@ bool HierarchicalEntityPicker::collectHits(NodeManagers *manager, Entity *root)
             if (child) {
                 ObjectPicker *childPicker = child->renderComponent<ObjectPicker>();
                 worklist.push_back({child, current.hasObjectPicker || childPicker,
-                                    current.recursiveLayers + recursiveLayers,
                                     (childPicker ? childPicker->priority() : current.priority)});
             }
         }
diff --git a/src/render/jobs/pickboundingvolumeutils_p.h b/src/render/jobs/pickboundingvolumeutils_p.h
index d9c8eee22..6a637a1d7 100644
--- a/src/render/jobs/pickboundingvolumeutils_p.h
+++ b/src/render/jobs/pickboundingvolumeutils_p.h
@@ -84,8 +84,7 @@ struct Q_AUTOTEST_EXPORT ViewportCameraAreaDetails
     QRectF viewport;
     QSize area;
     QSurface *surface = nullptr;
-    Qt3DCore::QNodeIdVector layers;
-    QAbstractRayCaster::FilterMode layerFilterMode = QAbstractRayCaster::AcceptAnyMatchingLayers;
+    Qt3DCore::QNodeIdVector layersFilters;
 };
 QT3D_DECLARE_TYPEINFO_3(Qt3DRender, Render, PickingUtils, ViewportCameraAreaDetails, Q_COMPLEX_TYPE)
 
@@ -124,7 +123,8 @@ class Q_AUTOTEST_EXPORT HierarchicalEntityPicker
 public:
     explicit HierarchicalEntityPicker(const RayCasting::QRay3D &ray, bool requireObjectPicker = true);
 
-    void setFilterLayers(const Qt3DCore::QNodeIdVector &layerIds, QAbstractRayCaster::FilterMode mode);
+    void setLayerFilterIds(const Qt3DCore::QNodeIdVector &layerFilterIds);
+    void setLayerIds(const Qt3DCore::QNodeIdVector &layerIds, QAbstractRayCaster::FilterMode mode);
 
     bool collectHits(NodeManagers *manager, Entity *root);
     inline HitList hits() const { return m_hits; }
@@ -136,8 +136,9 @@ private:
     HitList m_hits;
     std::vector<Entity *> m_entities;
     bool m_objectPickersRequired;
+    Qt3DCore::QNodeIdVector m_layerFilterIds;
     Qt3DCore::QNodeIdVector m_layerIds;
-    QAbstractRayCaster::FilterMode m_filterMode;
+    QAbstractRayCaster::FilterMode m_layerFilterMode = QAbstractRayCaster::AcceptAnyMatchingLayers;
     QHash<Qt3DCore::QNodeId, int> m_entityToPriorityTable;
 };
 
diff --git a/src/render/jobs/raycastingjob.cpp b/src/render/jobs/raycastingjob.cpp
index 7c7ddc71a..b6ebdad99 100644
--- a/src/render/jobs/raycastingjob.cpp
+++ b/src/render/jobs/raycastingjob.cpp
@@ -201,7 +201,7 @@ bool RayCastingJob::pick(const QList<QPair<Entity *, RayCaster *>> &entities)
         for (const QRay3D &ray: qAsConst(rays)) {
             PickingUtils::HitList sphereHits;
             PickingUtils::HierarchicalEntityPicker entityPicker(ray, false);
-            entityPicker.setFilterLayers(pair.second->layerIds(), pair.second->filterMode());
+            entityPicker.setLayerIds(pair.second->layerIds(), pair.second->filterMode());
             if (entityPicker.collectHits(m_manager, m_node)) {
                 if (pickConfiguration.trianglePickingRequested) {
                     PickingUtils::TriangleCollisionGathererFunctor gathererFunctor;
diff --git a/tests/auto/render/pickboundingvolumejob/CMakeLists.txt b/tests/auto/render/pickboundingvolumejob/CMakeLists.txt
index 25491ba9f..4e3404870 100644
--- a/tests/auto/render/pickboundingvolumejob/CMakeLists.txt
+++ b/tests/auto/render/pickboundingvolumejob/CMakeLists.txt
@@ -40,6 +40,7 @@ set(pickboundingvolumejob_resource_files
     "testscene_priorityoverlapping.qml"
     "testscene_viewports.qml"
     "testscene_layerfilter.qml"
+    "testscene_nested_layerfilter.qml"
 )
 
 qt_internal_add_resource(tst_pickboundingvolumejob "pickboundingvolumejob"
diff --git a/tests/auto/render/pickboundingvolumejob/pickboundingvolumejob.qrc b/tests/auto/render/pickboundingvolumejob/pickboundingvolumejob.qrc
index 76150da31..63bc4be80 100644
--- a/tests/auto/render/pickboundingvolumejob/pickboundingvolumejob.qrc
+++ b/tests/auto/render/pickboundingvolumejob/pickboundingvolumejob.qrc
@@ -12,5 +12,6 @@
         <file>testscene_cameraposition.qml</file>
         <file>testscene_priorityoverlapping.qml</file>
         <file>testscene_nopicking.qml</file>
+        <file>testscene_nested_layerfilter.qml</file>
     </qresource>
 </RCC>
diff --git a/tests/auto/render/pickboundingvolumejob/testscene_nested_layerfilter.qml b/tests/auto/render/pickboundingvolumejob/testscene_nested_layerfilter.qml
new file mode 100644
index 000000000..b913b6f5f
--- /dev/null
+++ b/tests/auto/render/pickboundingvolumejob/testscene_nested_layerfilter.qml
@@ -0,0 +1,233 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 Klaralvdalens Datakonsult AB (KDAB).
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the Qt3D module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:BSD$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** BSD License Usage
+** Alternatively, you may use this file under the terms of the BSD license
+** as follows:
+**
+** "Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions are
+** met:
+**   * Redistributions of source code must retain the above copyright
+**     notice, this list of conditions and the following disclaimer.
+**   * Redistributions in binary form must reproduce the above copyright
+**     notice, this list of conditions and the following disclaimer in
+**     the documentation and/or other materials provided with the
+**     distribution.
+**   * Neither the name of The Qt Company Ltd nor the names of its
+**     contributors may be used to endorse or promote products derived
+**     from this software without specific prior written permission.
+**
+**
+** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+** OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+** LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+import Qt3D.Core 2.0
+import Qt3D.Render 2.0
+import Qt3D.Extras 2.0
+import QtQuick.Window 2.0
+
+Entity {
+    id: sceneRoot
+
+    Window {
+        id: _view
+        width: 600
+        height: 600
+        visible: true
+    }
+
+    Camera {
+        id: camera
+        projectionType: CameraLens.PerspectiveProjection
+        fieldOfView: 45
+        aspectRatio: _view.width / 2 / _view.height
+        nearPlane : 0.1
+        farPlane : 1000.0
+        position: Qt.vector3d( 0.0, 0.0, -10.0 )
+        upVector: Qt.vector3d( 0.0, 1.0, 0.0 )
+        viewCenter: Qt.vector3d( 0.0, 0.0, 0.0 )
+    }
+
+    Camera {
+        id: camera2
+        projectionType: CameraLens.PerspectiveProjection
+        fieldOfView: 45
+        aspectRatio: _view.width / 2 / _view.height
+        nearPlane : 0.1
+        farPlane : 1000.0
+        position: Qt.vector3d( 0.0, 0.0, -10.0 )
+        upVector: Qt.vector3d( 0.0, 1.0, 0.0 )
+        viewCenter: Qt.vector3d( 0.0, 0.0, 0.0 )
+    }
+
+    FirstPersonCameraController {
+        camera: camera
+    }
+
+    DirectionalLight {
+        worldDirection: camera.viewVector.times(-1)
+    }
+
+    // Draw 2 viewports
+    components: [
+        RenderSettings {
+            RenderSurfaceSelector {
+                surface: _view
+
+                Viewport {
+                    normalizedRect: Qt.rect(0.0, 0.0, 0.5, 1.0)
+                    ClearBuffers {
+                        buffers : ClearBuffers.ColorDepthBuffer
+                        clearColor: "red"
+                        CameraSelector {
+                            camera: camera
+
+                            LayerFilter {
+                                // Any of
+                                layers: [ layer1, layer3 ]
+
+                                LayerFilter {
+                                    filterMode: LayerFilter.DiscardAnyMatchingLayers
+                                    layers: [layer2]
+                                }
+                            }
+                        }
+                    }
+                }
+
+                Viewport {
+                    normalizedRect: Qt.rect(0.5, 0.0, 0.5, 1.0)
+                    CameraSelector {
+                        camera: camera2
+
+                        LayerFilter {
+                            layers: [ layerRecursive ]
+
+                            LayerFilter {
+                                filterMode: LayerFilter.DiscardAnyMatchingLayers
+                                layers: [layer1]
+                            }
+                        }
+                    }
+                }
+            }
+
+            pickingSettings {
+                pickMethod: PickingSettings.TrianglePicking
+            }
+        }
+    ]
+
+    CuboidMesh { id: cubeMesh }
+
+    Layer { id: layerRecursive; recursive: true }
+    Layer { id: layer1 }
+    Layer { id: layer2 }
+    Layer { id: layer3 }
+
+    Entity {
+        id: rootSceneEntity
+        components: [layerRecursive]
+
+        // Selected by Viewport 1
+        Entity {
+            readonly property ObjectPicker objectPicker: ObjectPicker {
+                objectName: "Picker1"
+                onClicked: console.log("o1 " + pick.position)
+            }
+            readonly property Transform transform: Transform {
+                scale: 2
+                translation: Qt.vector3d(-1, 0, 0)
+            }
+            readonly property PhongMaterial material: PhongMaterial { diffuse: "red" }
+
+            components: [cubeMesh, transform, material, objectPicker, layer1 ]
+        }
+
+        // Selected by Viewport 2
+        Entity {
+            readonly property ObjectPicker objectPicker: ObjectPicker {
+                objectName: "Picker2"
+                onClicked: console.log("o2" + pick.position)
+            }
+            readonly property Transform transform: Transform {
+                scale: 3
+                translation: Qt.vector3d(-2, 3, 5)
+            }
+            readonly property PhongMaterial material: PhongMaterial { diffuse: "green" }
+
+            components: [cubeMesh, transform, material, objectPicker, layer2 ]
+        }
+
+        // Not Selected by Any Viewport
+        Entity {
+            readonly property ObjectPicker objectPicker: ObjectPicker {
+                objectName: "Picker3"
+                onClicked: console.log("o3" + pick.position)
+            }
+            readonly property Transform transform: Transform {
+                scale: 4
+            }
+            readonly property PhongMaterial material: PhongMaterial { diffuse: "blue" }
+
+            components: [cubeMesh, transform, material, objectPicker, layer1, layer2 ]
+        }
+
+        // Both Viewport
+        Entity {
+            readonly property ObjectPicker objectPicker: ObjectPicker {
+                objectName: "Picker4"
+                onClicked: console.log("o4" + pick.position)
+            }
+            readonly property Transform transform: Transform {
+                scale: 1
+                translation: Qt.vector3d(0, -2, -1)
+            }
+            readonly property PhongMaterial material: PhongMaterial { diffuse: "orange" }
+
+            components: [cubeMesh, transform, material, objectPicker, layer3 ]
+        }
+
+        // Viewport 1
+        Entity {
+            readonly property ObjectPicker objectPicker: ObjectPicker {
+                objectName: "Picker5"
+                onClicked: console.log("o5" + pick.position)
+            }
+            readonly property Transform transform: Transform {
+                scale: 2
+                translation: Qt.vector3d(0, 1, 3)
+            }
+            readonly property PhongMaterial material: PhongMaterial { diffuse: "purple" }
+
+            components: [cubeMesh, transform, material, objectPicker, layer3, layer1 ]
+        }
+    }
+}
diff --git a/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp b/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
index 98bf268bb..df0eab407 100644
--- a/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
+++ b/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
@@ -60,6 +60,7 @@
 #include <Qt3DRender/private/updateworldtransformjob_p.h>
 #include <Qt3DRender/private/expandboundingvolumejob_p.h>
 #include <Qt3DRender/private/calcboundingvolumejob_p.h>
+#include <Qt3DRender/private/updateentitylayersjob_p.h>
 #include <Qt3DRender/private/buffermanager_p.h>
 #include <Qt3DRender/private/geometryrenderermanager_p.h>
 #include <Qt3DRender/private/qobjectpicker_p.h>
@@ -231,6 +232,10 @@ void runRequiredJobs(Qt3DRender::TestAspect *test)
     expandBVolume.setRoot(test->sceneRoot());
     expandBVolume.setManagers(test->nodeManagers());
     expandBVolume.run();
+
+    Qt3DRender::Render::UpdateEntityLayersJob updateEntityLayer;
+    updateEntityLayer.setManager(test->nodeManagers());
+    updateEntityLayer.run();
 }
 
 void initializePickBoundingVolumeJob(Qt3DRender::Render::PickBoundingVolumeJob *job, Qt3DRender::TestAspect *test)
@@ -1660,6 +1665,289 @@ private Q_SLOTS:
         }
     }
 
+    void checkPickerAndNestedLayerFilters()
+    {
+        // GIVEN
+        QmlSceneReader sceneReader(QUrl("qrc:/testscene_nested_layerfilter.qml"));
+        QScopedPointer<Qt3DCore::QNode> root(qobject_cast<Qt3DCore::QNode *>(sceneReader.root()));
+        QVERIFY(root);
+
+        QScopedPointer<Qt3DRender::TestAspect> test(new Qt3DRender::TestAspect(root.data()));
+        TestArbiter arbiter;
+
+        // Runs Required jobs
+        runRequiredJobs(test.data());
+
+        // THEN
+        // object partially obscured by another viewport, make sure only visible portion is pickable
+        QList<Qt3DRender::QObjectPicker *> pickers = root->findChildren<Qt3DRender::QObjectPicker *>();
+        QCOMPARE(pickers.size(), 5);
+
+        Qt3DRender::QObjectPicker *picker1 = pickers[0];
+        QCOMPARE(picker1->objectName(), QLatin1String("Picker1"));
+        Qt3DRender::QObjectPicker *picker2 = pickers[1];
+        QCOMPARE(picker2->objectName(), QLatin1String("Picker2"));
+        Qt3DRender::QObjectPicker *picker3 = pickers[2];
+        QCOMPARE(picker3->objectName(), QLatin1String("Picker3"));
+        Qt3DRender::QObjectPicker *picker4 = pickers[3];
+        QCOMPARE(picker4->objectName(), QLatin1String("Picker4"));
+        Qt3DRender::QObjectPicker *picker5 = pickers[4];
+        QCOMPARE(picker5->objectName(), QLatin1String("Picker5"));
+
+        Qt3DRender::Render::ObjectPicker *backendPicker1 = test->nodeManagers()->objectPickerManager()->lookupResource(picker1->id());
+        QVERIFY(backendPicker1);
+
+        QSignalSpy mouseButtonPressedSpy1(picker1, &Qt3DRender::QObjectPicker::pressed);
+        QVERIFY(mouseButtonPressedSpy1.isValid());
+
+        Qt3DRender::Render::ObjectPicker *backendPicker2 = test->nodeManagers()->objectPickerManager()->lookupResource(picker2->id());
+        QVERIFY(backendPicker2);
+
+        QSignalSpy mouseButtonPressedSpy2(picker2, &Qt3DRender::QObjectPicker::pressed);
+        QVERIFY(mouseButtonPressedSpy2.isValid());
+
+        Qt3DRender::Render::ObjectPicker *backendPicker3 = test->nodeManagers()->objectPickerManager()->lookupResource(picker3->id());
+        QVERIFY(backendPicker3);
+
+        QSignalSpy mouseButtonPressedSpy3(picker3, &Qt3DRender::QObjectPicker::pressed);
+        QVERIFY(mouseButtonPressedSpy3.isValid());
+
+        Qt3DRender::Render::ObjectPicker *backendPicker4 = test->nodeManagers()->objectPickerManager()->lookupResource(picker4->id());
+        QVERIFY(backendPicker4);
+
+        QSignalSpy mouseButtonPressedSpy4(picker4, &Qt3DRender::QObjectPicker::pressed);
+        QVERIFY(mouseButtonPressedSpy4.isValid());
+
+        Qt3DRender::Render::ObjectPicker *backendPicker5 = test->nodeManagers()->objectPickerManager()->lookupResource(picker5->id());
+        QVERIFY(backendPicker5);
+
+        QSignalSpy mouseButtonPressedSpy5(picker5, &Qt3DRender::QObjectPicker::pressed);
+        QVERIFY(mouseButtonPressedSpy5.isValid());
+
+        Qt3DRender::Render::PickBoundingVolumeJob pickBVJob;
+        initializePickBoundingVolumeJob(&pickBVJob, test.data());
+
+        // Picker1 -> Viewport 1
+        // Picker2 -> Viewport 2
+        // Picker3 -> No Viewport
+        // Picker4 -> Viewport 1 and 2
+        // Picker5 -> Viewport 1
+
+        // WHEN -> Pressed on object1 in VP1
+        {
+            auto event = QMouseEvent(QMouseEvent::MouseButtonPress, QPointF(200.0f, 300.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event);
+            bool earlyReturn = !pickBVJob.runHelper();
+            Qt3DCore::QAspectJobPrivate::get(&pickBVJob)->postFrame(test->aspectManager());
+
+            // THEN -> Pressed
+            QVERIFY(!earlyReturn);
+            QVERIFY(backendPicker1->isPressed());
+            QVERIFY(picker1->isPressed());
+            QVERIFY(!backendPicker2->isPressed());
+            QVERIFY(!picker2->isPressed());
+            QVERIFY(!backendPicker3->isPressed());
+            QVERIFY(!picker3->isPressed());
+            QVERIFY(!backendPicker4->isPressed());
+            QVERIFY(!picker4->isPressed());
+            QVERIFY(!backendPicker5->isPressed());
+            QVERIFY(!picker5->isPressed());
+            QCOMPARE(mouseButtonPressedSpy1.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy2.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy3.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy4.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy5.count(), 0);
+
+            auto event2 = QMouseEvent(QMouseEvent::MouseButtonRelease, QPointF(200.0f, 300.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event2);
+            pickBVJob.runHelper();
+        }
+
+        // WHEN -> Pressed on object2 in VP2
+        {
+            auto event = QMouseEvent(QMouseEvent::MouseButtonPress, QPointF(541.0f, 183.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event);
+            bool earlyReturn = !pickBVJob.runHelper();
+            Qt3DCore::QAspectJobPrivate::get(&pickBVJob)->postFrame(test->aspectManager());
+
+            // THEN -> Pressed
+            QVERIFY(!earlyReturn);
+            QVERIFY(!backendPicker1->isPressed());
+            QVERIFY(!picker1->isPressed());
+            QVERIFY(backendPicker2->isPressed());
+            QVERIFY(picker2->isPressed());
+            QVERIFY(!backendPicker3->isPressed());
+            QVERIFY(!picker3->isPressed());
+            QVERIFY(!backendPicker4->isPressed());
+            QVERIFY(!picker4->isPressed());
+            QVERIFY(!backendPicker5->isPressed());
+            QVERIFY(!picker5->isPressed());
+            QCOMPARE(mouseButtonPressedSpy1.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy2.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy3.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy4.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy5.count(), 0);
+
+            auto event2 = QMouseEvent(QMouseEvent::MouseButtonRelease, QPointF(541.0f, 183.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event2);
+            pickBVJob.runHelper();
+        }
+
+        // WHEN -> Pressed on object3 in VP1
+        {
+            auto event = QMouseEvent(QMouseEvent::MouseButtonPress, QPointF(80.0f, 150.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event);
+            bool earlyReturn = !pickBVJob.runHelper();
+            Qt3DCore::QAspectJobPrivate::get(&pickBVJob)->postFrame(test->aspectManager());
+
+            // THEN -> Nothing Pressed as not selected by Layer
+            QVERIFY(!earlyReturn);
+            QVERIFY(!backendPicker1->isPressed());
+            QVERIFY(!picker1->isPressed());
+            QVERIFY(!backendPicker2->isPressed());
+            QVERIFY(!picker2->isPressed());
+            QVERIFY(!backendPicker3->isPressed());
+            QVERIFY(!picker3->isPressed());
+            QVERIFY(!backendPicker4->isPressed());
+            QVERIFY(!picker4->isPressed());
+            QVERIFY(!backendPicker5->isPressed());
+            QVERIFY(!picker5->isPressed());
+            QCOMPARE(mouseButtonPressedSpy1.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy2.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy3.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy4.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy5.count(), 0);
+        }
+
+        // WHEN -> Pressed on object3 in VP2
+        {
+            auto event = QMouseEvent(QMouseEvent::MouseButtonPress, QPointF(504.0f, 263.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event);
+            bool earlyReturn = !pickBVJob.runHelper();
+            Qt3DCore::QAspectJobPrivate::get(&pickBVJob)->postFrame(test->aspectManager());
+
+            // THEN -> Nothing Pressed as not selected by Layer
+            QVERIFY(!earlyReturn);
+            QVERIFY(!backendPicker1->isPressed());
+            QVERIFY(!picker1->isPressed());
+            QVERIFY(!backendPicker2->isPressed());
+            QVERIFY(!picker2->isPressed());
+            QVERIFY(!backendPicker3->isPressed());
+            QVERIFY(!picker3->isPressed());
+            QVERIFY(!backendPicker4->isPressed());
+            QVERIFY(!picker4->isPressed());
+            QVERIFY(!backendPicker5->isPressed());
+            QVERIFY(!picker5->isPressed());
+            QCOMPARE(mouseButtonPressedSpy1.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy2.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy3.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy4.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy5.count(), 0);
+        }
+
+        // WHEN -> Pressed on object4 in VP1
+        {
+            auto event = QMouseEvent(QMouseEvent::MouseButtonPress, QPointF(160.0f, 431.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event);
+            bool earlyReturn = !pickBVJob.runHelper();
+            Qt3DCore::QAspectJobPrivate::get(&pickBVJob)->postFrame(test->aspectManager());
+
+            // THEN -> Pressed
+            QVERIFY(!earlyReturn);
+            QVERIFY(!backendPicker1->isPressed());
+            QVERIFY(!picker1->isPressed());
+            QVERIFY(!backendPicker2->isPressed());
+            QVERIFY(!picker2->isPressed());
+            QVERIFY(!backendPicker3->isPressed());
+            QVERIFY(!picker3->isPressed());
+            QVERIFY(backendPicker4->isPressed());
+            QVERIFY(picker4->isPressed());
+            QVERIFY(!backendPicker5->isPressed());
+            QVERIFY(!picker5->isPressed());
+            QCOMPARE(mouseButtonPressedSpy1.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy2.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy3.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy4.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy5.count(), 0);
+
+            auto event2 = QMouseEvent(QMouseEvent::MouseButtonRelease, QPointF(160.0f, 431.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event2);
+            pickBVJob.runHelper();
+        }
+
+        // WHEN -> Pressed on object4 in VP2
+        {
+            auto event = QMouseEvent(QMouseEvent::MouseButtonPress, QPointF(447.0f, 472.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event);
+            bool earlyReturn = !pickBVJob.runHelper();
+            Qt3DCore::QAspectJobPrivate::get(&pickBVJob)->postFrame(test->aspectManager());
+
+            // THEN -> Pressed
+            QVERIFY(!earlyReturn);
+            QVERIFY(!backendPicker1->isPressed());
+            QVERIFY(!picker1->isPressed());
+            QVERIFY(!backendPicker2->isPressed());
+            QVERIFY(!picker2->isPressed());
+            QVERIFY(!backendPicker3->isPressed());
+            QVERIFY(!picker3->isPressed());
+            QVERIFY(backendPicker4->isPressed());
+            QVERIFY(picker4->isPressed());
+            QVERIFY(!backendPicker5->isPressed());
+            QVERIFY(!picker5->isPressed());
+            QCOMPARE(mouseButtonPressedSpy1.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy2.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy3.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy4.count(), 2);
+            QCOMPARE(mouseButtonPressedSpy5.count(), 0);
+
+            auto event2 = QMouseEvent(QMouseEvent::MouseButtonRelease, QPointF(447.0f, 472.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event2);
+            pickBVJob.runHelper();
+        }
+
+        // WHEN -> Pressed on object5 in VP1
+        {
+            auto event = QMouseEvent(QMouseEvent::MouseButtonPress, QPointF(153.0f, 195.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event);
+            bool earlyReturn = !pickBVJob.runHelper();
+            Qt3DCore::QAspectJobPrivate::get(&pickBVJob)->postFrame(test->aspectManager());
+
+            // THEN -> Pressed
+            QVERIFY(!earlyReturn);
+            QVERIFY(!backendPicker1->isPressed());
+            QVERIFY(!picker1->isPressed());
+            QVERIFY(!backendPicker2->isPressed());
+            QVERIFY(!picker2->isPressed());
+            QVERIFY(!backendPicker3->isPressed());
+            QVERIFY(!picker3->isPressed());
+            QVERIFY(!backendPicker4->isPressed());
+            QVERIFY(!picker4->isPressed());
+            QVERIFY(backendPicker5->isPressed());
+            QVERIFY(picker5->isPressed());
+            QCOMPARE(mouseButtonPressedSpy1.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy2.count(), 1);
+            QCOMPARE(mouseButtonPressedSpy3.count(), 0);
+            QCOMPARE(mouseButtonPressedSpy4.count(), 2);
+            QCOMPARE(mouseButtonPressedSpy5.count(), 1);
+
+            auto event2 = QMouseEvent(QMouseEvent::MouseButtonRelease, QPointF(153.0f, 195.0f),
+                                     Qt::LeftButton, Qt::LeftButton, Qt::NoModifier);
+            pickBVJob.processMouseEvent(nullptr, &event2);
+            pickBVJob.runHelper();
+        }
+    }
+
     void checkMultipleRayDirections_data()
     {
         QTest::addColumn<QVector3D>("cameraOrigin");
diff --git a/tests/auto/render/raycastingjob/tst_raycastingjob.cpp b/tests/auto/render/raycastingjob/tst_raycastingjob.cpp
index 71b6427a1..72746b219 100644
--- a/tests/auto/render/raycastingjob/tst_raycastingjob.cpp
+++ b/tests/auto/render/raycastingjob/tst_raycastingjob.cpp
@@ -56,6 +56,7 @@
 #include <Qt3DRender/private/updateworldtransformjob_p.h>
 #include <Qt3DRender/private/expandboundingvolumejob_p.h>
 #include <Qt3DRender/private/calcboundingvolumejob_p.h>
+#include <Qt3DRender/private/updateentitylayersjob_p.h>
 #include <Qt3DRender/private/buffermanager_p.h>
 #include <Qt3DRender/private/geometryrenderermanager_p.h>
 
@@ -229,6 +230,10 @@ void runRequiredJobs(Qt3DRender::TestAspect *test)
     expandBVolume.setRoot(test->sceneRoot());
     expandBVolume.setManagers(test->nodeManagers());
     expandBVolume.run();
+
+    Qt3DRender::Render::UpdateEntityLayersJob updateEntityLayer;
+    updateEntityLayer.setManager(test->nodeManagers());
+    updateEntityLayer.run();
 }
 
 void initializeJob(Qt3DRender::Render::RayCastingJob *job, Qt3DRender::TestAspect *test)