Fix and improve layer filtering

We now handle the case where multiple LayerFilter nodes are nested
within the same FrameGraphBranch. We also have improved the Accept/Discard
into Accept Any/All and Discard Any/All which should now handle all
filtering cases.

We now build a list of layerIds stored into Entity based on the recursive
flags on the layers being set. This allows to make layer filtering simpler
but the layerIds list building step will need to be improved as a follow up
commit

Unit tests updated

Change-Id: I93451493e41c6c9486defa7e88eaee073a9bc932
Reviewed-by: Sean Harmer <sean.harmer@kdab.com>

4 files changed, 440 insertions, 152 deletions

diff --git a/tests/auto/render/entity/tst_entity.cpp b/tests/auto/render/entity/tst_entity.cpp
index 96a9683d3..6ad958451 100644
--- a/tests/auto/render/entity/tst_entity.cpp
+++ b/tests/auto/render/entity/tst_entity.cpp
@@ -122,6 +122,7 @@ private slots:
         QVERIFY(entity.componentUuid<Armature>().isNull());
         QVERIFY(!entity.isBoundingVolumeDirty());
         QVERIFY(entity.childrenHandles().isEmpty());
+        QVERIFY(entity.layerIds().isEmpty());
 
         // WHEN
         Q_FOREACH (QComponent *component, components) {
@@ -151,6 +152,7 @@ private slots:
         QVERIFY(!entity.componentUuid<Armature>().isNull());
         QVERIFY(entity.isBoundingVolumeDirty());
         QVERIFY(!entity.childrenHandles().isEmpty());
+        QVERIFY(!entity.layerIds().isEmpty());
         QVERIFY(renderer.dirtyBits() != 0);
         bool containsAll = entity.containsComponentsOfType<Transform,
                 CameraLens, Material, GeometryRenderer, ObjectPicker, ComputeCommand, Armature>();
@@ -172,6 +174,7 @@ private slots:
         QVERIFY(entity.componentUuid<Armature>().isNull());
         QVERIFY(!entity.isBoundingVolumeDirty());
         QVERIFY(entity.childrenHandles().isEmpty());
+        QVERIFY(entity.layerIds().isEmpty());
         containsAll = entity.containsComponentsOfType<Transform,
                 CameraLens, Material, GeometryRenderer, ObjectPicker, ComputeCommand, Armature>();
         QVERIFY(!containsAll);
diff --git a/tests/auto/render/layerfiltering/tst_layerfiltering.cpp b/tests/auto/render/layerfiltering/tst_layerfiltering.cpp
index ed61fce87..c2651c477 100644
--- a/tests/auto/render/layerfiltering/tst_layerfiltering.cpp
+++ b/tests/auto/render/layerfiltering/tst_layerfiltering.cpp
@@ -51,18 +51,14 @@ private Q_SLOTS:
         // THEN
         QCOMPARE(filterJob.hasLayerFilter(), false);
         QCOMPARE(filterJob.filteredEntities().size(), 0);
-        QCOMPARE(filterJob.layers().size(), 0);
-        QCOMPARE(filterJob.filterMode(), Qt3DRender::QLayerFilter::AcceptMatchingLayers);
+        QCOMPARE(filterJob.layerFilters().size(), 0);
         QCOMPARE(frontendLayer.recursive(), false);
     }
 
     void filterEntities_data()
     {
         QTest::addColumn<Qt3DCore::QEntity *>("entitySubtree");
-        QTest::addColumn<Qt3DCore::QNodeIdVector>("layerIds");
-        QTest::addColumn<bool>("hasLayerFilter");
-        QTest::addColumn<Qt3DRender::QLayerFilter::FilterMode>("filterMode");
-        QTest::addColumn<bool>("recursive");
+        QTest::addColumn<Qt3DCore::QNodeIdVector>("layerFilterIds");
         QTest::addColumn<Qt3DCore::QNodeIdVector>("expectedSelectedEntities");
 
 
@@ -77,16 +73,13 @@ private Q_SLOTS:
             Q_UNUSED(childEntity3);
 
             QTest::newRow("EntitiesNoLayerNoLayerFilterNoDiscardNoRecursive-ShouldSelectAll") << rootEntity
-                                                                          << Qt3DCore::QNodeIdVector()
-                                                                          << false
-                                                                          << Qt3DRender::QLayerFilter::AcceptMatchingLayers
-                                                                          << false
-                                                                          << (Qt3DCore::QNodeIdVector()
-                                                                              << rootEntity->id()
-                                                                              << childEntity1->id()
-                                                                              << childEntity2->id()
-                                                                              << childEntity3->id()
-                                                                              );
+                                                                                              << Qt3DCore::QNodeIdVector()
+                                                                                              << (Qt3DCore::QNodeIdVector()
+                                                                                                  << rootEntity->id()
+                                                                                                  << childEntity1->id()
+                                                                                                  << childEntity2->id()
+                                                                                                  << childEntity3->id()
+                                                                                                  );
         }
 
         {
@@ -95,21 +88,15 @@ private Q_SLOTS:
             Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
             Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
 
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+
             Q_UNUSED(childEntity1);
             Q_UNUSED(childEntity2);
             Q_UNUSED(childEntity3);
 
-            QTest::newRow("EntitiesNoLayerNoLayerFilterNoDiscardNoRecursive-ShouldSelectAll") << rootEntity
-                                                                          << Qt3DCore::QNodeIdVector()
-                                                                          << false
-                                                                          << Qt3DRender::QLayerFilter::AcceptMatchingLayers
-                                                                          << false
-                                                                          << (Qt3DCore::QNodeIdVector()
-                                                                              << rootEntity->id()
-                                                                              << childEntity1->id()
-                                                                              << childEntity2->id()
-                                                                              << childEntity3->id()
-                                                                              );
+            QTest::newRow("EntityNoLayerWithLayerFilterWithNoFilter-ShouldSelectNone") << rootEntity
+                                                                                       << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                       << Qt3DCore::QNodeIdVector();
         }
 
         {
@@ -122,18 +109,16 @@ private Q_SLOTS:
             Q_UNUSED(childEntity2);
             Q_UNUSED(childEntity3);
 
-            QTest::newRow("EntitiesNoLayerNoLayerFilterNoDiscardNoRecursive-ShouldSelectAll") << rootEntity
-                                                                          << Qt3DCore::QNodeIdVector()
-                                                                          << false
-                                                                          << Qt3DRender::QLayerFilter::AcceptMatchingLayers
-                                                                          << false
-                                                                          << (Qt3DCore::QNodeIdVector()
-                                                                              << rootEntity->id()
-                                                                              << childEntity1->id()
-                                                                              << childEntity2->id()
-                                                                              << childEntity3->id()
-                                                                              );
+            Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->addLayer(layer);
+
+
+            QTest::newRow("AcceptAny-NoLayerWithLayerFilterWithFilter-ShouldSelectNone") << rootEntity
+                                                                                         << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                         << Qt3DCore::QNodeIdVector();
         }
+
         {
             Qt3DCore::QEntity *rootEntity = new Qt3DCore::QEntity();
             Qt3DCore::QEntity *childEntity1 = new Qt3DCore::QEntity(rootEntity);
@@ -141,16 +126,17 @@ private Q_SLOTS:
             Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
 
 
-            Q_UNUSED(childEntity1);
-            Q_UNUSED(childEntity2);
-            Q_UNUSED(childEntity3);
+            Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            childEntity1->addComponent(layer);
+            childEntity2->addComponent(layer);
+            childEntity3->addComponent(layer);
 
-            QTest::newRow("EntityNoLayerWithLayerFilterWithNoFilter-ShouldSelectNone") << rootEntity
-                                                                                       << Qt3DCore::QNodeIdVector()
-                                                                                       << true
-                                                                                       << Qt3DRender::QLayerFilter::AcceptMatchingLayers
-                                                                                       << false
-                                                                                       << Qt3DCore::QNodeIdVector();
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->addLayer(layer);
+
+            QTest::newRow("AcceptAny-LayerWithLayerFilterWithFilter-ShouldSelectAllButRoot") << rootEntity
+                                                                                             << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                             << (Qt3DCore::QNodeIdVector() << childEntity1->id() << childEntity2->id() << childEntity3->id());
         }
 
         {
@@ -159,19 +145,20 @@ private Q_SLOTS:
             Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
             Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
 
-            Q_UNUSED(childEntity1);
-            Q_UNUSED(childEntity2);
-            Q_UNUSED(childEntity3);
+            Q_UNUSED(childEntity1)
 
             Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
+            childEntity2->addComponent(layer2);
+            childEntity3->addComponent(layer);
 
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->addLayer(layer);
+            layerFilter->addLayer(layer2);
 
-            QTest::newRow("NoLayerWithLayerFilterWithFilter-ShouldSelectNone") << rootEntity
-                                                                               << (Qt3DCore::QNodeIdVector() << layer->id())
-                                                                               << true
-                                                                               << Qt3DRender::QLayerFilter::AcceptMatchingLayers
-                                                                               << false
-                                                                               << Qt3DCore::QNodeIdVector();
+            QTest::newRow("AcceptAny-LayerWithLayerFilterWithFilter-ShouldSelectChild2And3") << rootEntity
+                                                                                             << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                             << (Qt3DCore::QNodeIdVector() << childEntity2->id() << childEntity3->id());
         }
 
         {
@@ -182,16 +169,17 @@ private Q_SLOTS:
 
 
             Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
             childEntity1->addComponent(layer);
             childEntity2->addComponent(layer);
             childEntity3->addComponent(layer);
 
-            QTest::newRow("LayerWithLayerFilterWithFilter-ShouldSelectAllButRoot") << rootEntity
-                                                                                   << (Qt3DCore::QNodeIdVector() << layer->id())
-                                                                                   << true
-                                                                                   << Qt3DRender::QLayerFilter::AcceptMatchingLayers
-                                                                                   << false
-                                                                                   << (Qt3DCore::QNodeIdVector() << childEntity1->id() << childEntity2->id() << childEntity3->id());
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->addLayer(layer2);
+
+            QTest::newRow("AcceptAny-LayerWithLayerFilterWithFilter-ShouldSelectNone") << rootEntity
+                                                                                       << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                       << Qt3DCore::QNodeIdVector();
         }
 
         {
@@ -200,19 +188,41 @@ private Q_SLOTS:
             Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
             Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
 
-            Q_UNUSED(childEntity1)
+            childEntity1->setEnabled(false);
 
             Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
-            Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
-            childEntity2->addComponent(layer2);
+            childEntity1->addComponent(layer);
+            childEntity2->addComponent(layer);
             childEntity3->addComponent(layer);
 
-            QTest::newRow("LayerWithLayerFilterWithFilter-ShouldSelectChild2And3") << rootEntity
-                                                                                   << (Qt3DCore::QNodeIdVector() << layer->id() << layer2->id())
-                                                                                   << true
-                                                                                   << Qt3DRender::QLayerFilter::AcceptMatchingLayers
-                                                                                   << false
-                                                                                   << (Qt3DCore::QNodeIdVector() << childEntity2->id() << childEntity3->id());
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->addLayer(layer);
+
+            QTest::newRow("AcceptAny-LayerWithEntityDisabled-ShouldSelectOnlyEntityEnabled") << rootEntity
+                                                                                             << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                             << (Qt3DCore::QNodeIdVector() << childEntity2->id() << childEntity3->id());
+        }
+
+        {
+            Qt3DCore::QEntity *rootEntity = new Qt3DCore::QEntity();
+            Qt3DCore::QEntity *childEntity1 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
+
+            Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            layer->setRecursive(true);
+            rootEntity->addComponent(layer);
+
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->addLayer(layer);
+
+            QTest::newRow("AcceptAny-RecursiveLayerOnRoot-ShouldSelectAll") << rootEntity
+                                                                            << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                            << (Qt3DCore::QNodeIdVector()
+                                                                                << rootEntity->id()
+                                                                                << childEntity1->id()
+                                                                                << childEntity2->id()
+                                                                                << childEntity3->id());
         }
 
         {
@@ -221,19 +231,104 @@ private Q_SLOTS:
             Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
             Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
 
+            Q_UNUSED(childEntity1);
+            Q_UNUSED(childEntity2);
+            Q_UNUSED(childEntity3);
+
+            Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            layer->setRecursive(true);
+            rootEntity->addComponent(layer);
+
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->setFilterMode(Qt3DRender::QLayerFilter::DiscardAnyMatchingLayers);
+            layerFilter->addLayer(layer);
+
+            QTest::newRow("DiscardAny-RecursiveLayerLayerFilterDiscardOnRoot-ShouldSelectNone") << rootEntity
+                                                                                                << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                                << (Qt3DCore::QNodeIdVector());
+        }
+
+        {
+            Qt3DCore::QEntity *rootEntity = new Qt3DCore::QEntity();
+            Qt3DCore::QEntity *childEntity1 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
+
+            Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            rootEntity->addComponent(layer);
+
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->setFilterMode(Qt3DRender::QLayerFilter::DiscardAnyMatchingLayers);
+            layerFilter->addLayer(layer);
+
+            QTest::newRow("DiscardAny-LayerLayerFilterDiscardOnRoot-ShouldSelectAllButRoot") << rootEntity
+                                                                                             << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                             << (Qt3DCore::QNodeIdVector()
+                                                                                                 << childEntity1->id()
+                                                                                                 << childEntity2->id()
+                                                                                                 << childEntity3->id());
+        }
+
+        {
+            Qt3DCore::QEntity *rootEntity = new Qt3DCore::QEntity();
+            Qt3DCore::QEntity *childEntity1 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
 
             Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
             Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
-            childEntity1->addComponent(layer);
+            Qt3DRender::QLayer *layer3 = new Qt3DRender::QLayer(rootEntity);
+            rootEntity->addComponent(layer);
+
+            childEntity1->addComponent(layer2);
+
+            childEntity2->addComponent(layer3);
+
+            childEntity3->addComponent(layer2);
+            childEntity3->addComponent(layer3);
+            childEntity3->addComponent(layer);
+
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->setFilterMode(Qt3DRender::QLayerFilter::DiscardAnyMatchingLayers);
+            layerFilter->addLayer(layer2);
+            layerFilter->addLayer(layer3);
+
+            QTest::newRow("DiscardAny-LayerLayerFilterDiscardOnRoot-ShouldSelectRoot") << rootEntity
+                                                                                             << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                             << (Qt3DCore::QNodeIdVector()
+                                                                                                 << rootEntity->id());
+        }
+
+        {
+            Qt3DCore::QEntity *rootEntity = new Qt3DCore::QEntity();
+            Qt3DCore::QEntity *childEntity1 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
+
+            Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer3 = new Qt3DRender::QLayer(rootEntity);
+
+            rootEntity->addComponent(layer);
+
+            childEntity1->addComponent(layer3);
+            childEntity1->addComponent(layer2);
+
             childEntity2->addComponent(layer);
+            childEntity2->addComponent(layer3);
+
             childEntity3->addComponent(layer);
+            childEntity3->addComponent(layer2);
 
-            QTest::newRow("LayerWithLayerFilterWithFilter-ShouldSelectNone") << rootEntity
-                                                                             << (Qt3DCore::QNodeIdVector() << layer2->id())
-                                                                             << true
-                                                                             << Qt3DRender::QLayerFilter::AcceptMatchingLayers
-                                                                             << false
-                                                                             << Qt3DCore::QNodeIdVector();
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->setFilterMode(Qt3DRender::QLayerFilter::AcceptAllMatchingLayers);
+            layerFilter->addLayer(layer2);
+            layerFilter->addLayer(layer3);
+
+            QTest::newRow("AcceptAll-LayerFilterWith2LayersNonRecursive-ShouldSelectChild1") << rootEntity
+                                                                                             << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                             << (Qt3DCore::QNodeIdVector()
+                                                                                                 << childEntity1->id());
         }
 
         {
@@ -242,19 +337,99 @@ private Q_SLOTS:
             Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
             Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
 
-            childEntity1->setEnabled(false);
+            Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            layer->setRecursive(true);
+
+            Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer3 = new Qt3DRender::QLayer(rootEntity);
+
+            rootEntity->addComponent(layer);
+
+            childEntity1->addComponent(layer3);
+            childEntity1->addComponent(layer2);
+
+            childEntity2->addComponent(layer3);
+
+            childEntity3->addComponent(layer2);
+
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->setFilterMode(Qt3DRender::QLayerFilter::AcceptAllMatchingLayers);
+            layerFilter->addLayer(layer);
+            layerFilter->addLayer(layer3);
+
+            QTest::newRow("AcceptAll-LayerFilterWith2LayersRecursive-ShouldSelectChild12") << rootEntity
+                                                                                           << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                           << (Qt3DCore::QNodeIdVector()
+                                                                                               << childEntity1->id()
+                                                                                               << childEntity2->id());
+        }
+
+        {
+            Qt3DCore::QEntity *rootEntity = new Qt3DCore::QEntity();
+            Qt3DCore::QEntity *childEntity1 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
 
             Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
-            childEntity1->addComponent(layer);
+            layer->setRecursive(true);
+
+            Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer3 = new Qt3DRender::QLayer(rootEntity);
+
+            rootEntity->addComponent(layer);
+
+            childEntity1->addComponent(layer3);
+            childEntity1->addComponent(layer2);
+
             childEntity2->addComponent(layer);
-            childEntity3->addComponent(layer);
+            childEntity2->addComponent(layer3);
+
+            childEntity3->addComponent(layer2);
 
-            QTest::newRow("LayerWithEntityDisabled-ShouldSelectOnlyEntityEnabled") << rootEntity
-                                                                                   << (Qt3DCore::QNodeIdVector() << layer->id())
-                                                                                   << true
-                                                                                   << Qt3DRender::QLayerFilter::AcceptMatchingLayers
-                                                                                   << false
-                                                                                   << (Qt3DCore::QNodeIdVector() << childEntity2->id() << childEntity3->id());
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->setFilterMode(Qt3DRender::QLayerFilter::AcceptAllMatchingLayers);
+            layerFilter->addLayer(layer);
+            layerFilter->addLayer(layer3);
+
+            QTest::newRow("AcceptAll-LayerFilterWith2LayersRecursiveAndDirectReferenceToRecursiveLayer-ShouldSelectChild12") << rootEntity
+                                                                                                                             << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                                                             << (Qt3DCore::QNodeIdVector()
+                                                                                                                                 << childEntity1->id()
+                                                                                                                                 << childEntity2->id());
+        }
+
+        {
+            Qt3DCore::QEntity *rootEntity = new Qt3DCore::QEntity();
+            Qt3DCore::QEntity *childEntity1 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
+
+            Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer3 = new Qt3DRender::QLayer(rootEntity);
+
+            rootEntity->addComponent(layer);
+
+            childEntity1->addComponent(layer);
+            childEntity1->addComponent(layer2);
+
+            childEntity2->addComponent(layer);
+            childEntity2->addComponent(layer3);
+
+            childEntity3->addComponent(layer2);
+            childEntity3->addComponent(layer3);
+
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->setFilterMode(Qt3DRender::QLayerFilter::DiscardAllMatchingLayers);
+            layerFilter->addLayer(layer2);
+            layerFilter->addLayer(layer3);
+
+            QTest::newRow("DiscardAll-LayerFilterWith2Layers-ShouldSelectRootAndChild12") << rootEntity
+                                                                                          << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                          << (Qt3DCore::QNodeIdVector()
+                                                                                              << rootEntity->id()
+                                                                                              << childEntity1->id()
+                                                                                              << childEntity2->id());
         }
 
         {
@@ -265,18 +440,28 @@ private Q_SLOTS:
 
             Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
             layer->setRecursive(true);
+
+            Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer3 = new Qt3DRender::QLayer(rootEntity);
+
             rootEntity->addComponent(layer);
 
-            QTest::newRow("RecursiveLayerOnRoot-ShouldSelectAll") << rootEntity
-                                                                                   << (Qt3DCore::QNodeIdVector() << layer->id())
-                                                                                   << true
-                                                                                   << Qt3DRender::QLayerFilter::AcceptMatchingLayers
-                                                                                   << true
-                                                                                   << (Qt3DCore::QNodeIdVector()
-                                                                                       << rootEntity->id()
-                                                                                       << childEntity1->id()
-                                                                                       << childEntity2->id()
-                                                                                       << childEntity3->id());
+            childEntity1->addComponent(layer2);
+
+            childEntity2->addComponent(layer3);
+
+            childEntity3->addComponent(layer3);
+
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->setFilterMode(Qt3DRender::QLayerFilter::DiscardAllMatchingLayers);
+            layerFilter->addLayer(layer);
+            layerFilter->addLayer(layer3);
+
+            QTest::newRow("DiscardAll-LayerFilterWith2LayersRecursive-ShouldSelectRootAndChild1") << rootEntity
+                                                                                                  << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                                  << (Qt3DCore::QNodeIdVector()
+                                                                                                      << rootEntity->id()
+                                                                                                      << childEntity1->id());
         }
 
         {
@@ -285,20 +470,76 @@ private Q_SLOTS:
             Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
             Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
 
-            Q_UNUSED(childEntity1);
-            Q_UNUSED(childEntity2);
-            Q_UNUSED(childEntity3);
+            Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            layer->setRecursive(true);
+
+            Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer3 = new Qt3DRender::QLayer(rootEntity);
+
+            rootEntity->addComponent(layer);
+
+            childEntity1->addComponent(layer2);
+            childEntity1->addComponent(layer);
+
+            childEntity2->addComponent(layer3);
+
+            childEntity3->addComponent(layer3);
+
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->setFilterMode(Qt3DRender::QLayerFilter::DiscardAllMatchingLayers);
+            layerFilter->addLayer(layer);
+            layerFilter->addLayer(layer3);
+
+            QTest::newRow("DiscardAll-LayerFilterWith2LayersRecursiveAndDirectReference-ShouldSelectRootAndChild1") << rootEntity
+                                                                                                                    << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                                                    << (Qt3DCore::QNodeIdVector()
+                                                                                                                        << rootEntity->id()
+                                                                                                                        << childEntity1->id());
+        }
+
+        {
+            Qt3DCore::QEntity *rootEntity = new Qt3DCore::QEntity();
+            Qt3DCore::QEntity *childEntity1 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
 
             Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
             layer->setRecursive(true);
+
+            Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer3 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer4 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer5 = new Qt3DRender::QLayer(rootEntity);
+
             rootEntity->addComponent(layer);
 
-            QTest::newRow("RecursiveLayerLayerFilterDiscardOnRoot-ShouldSelectNone") << rootEntity
-                                                                                   << (Qt3DCore::QNodeIdVector() << layer->id())
-                                                                                   << true
-                                                                                   << Qt3DRender::QLayerFilter::DiscardMatchingLayers
-                                                                                   << true
-                                                                                   << (Qt3DCore::QNodeIdVector());
+            childEntity1->addComponent(layer2);
+            childEntity1->addComponent(layer3);
+
+            childEntity2->addComponent(layer2);
+            childEntity2->addComponent(layer3);
+            childEntity2->addComponent(layer4);
+            childEntity2->addComponent(layer5);
+
+            childEntity3->addComponent(layer2);
+            childEntity3->addComponent(layer3);
+            childEntity3->addComponent(layer5);
+
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->setFilterMode(Qt3DRender::QLayerFilter::AcceptAllMatchingLayers);
+            layerFilter->addLayer(layer);
+            layerFilter->addLayer(layer2);
+            layerFilter->addLayer(layer3);
+
+            Qt3DRender::QLayerFilter *layerFilter2 = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter2->setFilterMode(Qt3DRender::QLayerFilter::DiscardAnyMatchingLayers);
+            layerFilter2->addLayer(layer4);
+            layerFilter2->addLayer(layer5);
+
+            QTest::newRow("NestedFiltering-SelectAllOfLayer123AndNoneOf45-ShouldChild1") << rootEntity
+                                                                                         << (Qt3DCore::QNodeIdVector() << layerFilter->id() << layerFilter2->id())
+                                                                                         << (Qt3DCore::QNodeIdVector()
+                                                                                             << childEntity1->id());
         }
 
         {
@@ -306,19 +547,73 @@ private Q_SLOTS:
             Qt3DCore::QEntity *childEntity1 = new Qt3DCore::QEntity(rootEntity);
             Qt3DCore::QEntity *childEntity2 = new Qt3DCore::QEntity(rootEntity);
             Qt3DCore::QEntity *childEntity3 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity4 = new Qt3DCore::QEntity(rootEntity);
+            Qt3DCore::QEntity *childEntity5 = new Qt3DCore::QEntity(rootEntity);
 
             Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(rootEntity);
+            layer->setRecursive(true);
+
+            Qt3DRender::QLayer *layer2 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer3 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer4 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer5 = new Qt3DRender::QLayer(rootEntity);
+            Qt3DRender::QLayer *layer6 = new Qt3DRender::QLayer(rootEntity);
+
             rootEntity->addComponent(layer);
 
-            QTest::newRow("LayerLayerFilterDiscardOnRoot-ShouldSelectAllButRoot") << rootEntity
-                                                                                   << (Qt3DCore::QNodeIdVector() << layer->id())
-                                                                                   << true
-                                                                                   << Qt3DRender::QLayerFilter::DiscardMatchingLayers
-                                                                                   << false
-                                                                                   << (Qt3DCore::QNodeIdVector()
-                                                                                       << childEntity1->id()
-                                                                                       << childEntity2->id()
-                                                                                       << childEntity3->id());
+            childEntity1->addComponent(layer2);
+            childEntity1->addComponent(layer3);
+
+            childEntity2->addComponent(layer2);
+            childEntity2->addComponent(layer3);
+            childEntity2->addComponent(layer4);
+            childEntity2->addComponent(layer5);
+
+            childEntity3->addComponent(layer2);
+            childEntity3->addComponent(layer5);
+
+            childEntity4->addComponent(layer2);
+            childEntity4->addComponent(layer);
+            childEntity4->addComponent(layer3);
+            childEntity4->addComponent(layer6);
+
+            childEntity5->addComponent(layer3);
+            childEntity5->addComponent(layer4);
+            childEntity5->addComponent(layer6);
+
+            Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter->setFilterMode(Qt3DRender::QLayerFilter::DiscardAnyMatchingLayers);
+            layerFilter->addLayer(layer5);
+            layerFilter->addLayer(layer4);
+
+            Qt3DRender::QLayerFilter *layerFilter2 = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter2->setFilterMode(Qt3DRender::QLayerFilter::AcceptAnyMatchingLayers);
+            layerFilter2->addLayer(layer2);
+            layerFilter2->addLayer(layer3);
+
+            Qt3DRender::QLayerFilter *layerFilter3 = new Qt3DRender::QLayerFilter(rootEntity);
+            layerFilter3->setFilterMode(Qt3DRender::QLayerFilter::AcceptAllMatchingLayers);
+            layerFilter3->addLayer(layer);
+            layerFilter3->addLayer(layer6);
+
+            QTest::newRow("NestedFiltering-SelectAllNoneOfAnyLayer45AndAnyOf23AndAllOf16-ShouldSelectChild4-Step1") << rootEntity
+                                                                                                                    << (Qt3DCore::QNodeIdVector() << layerFilter->id())
+                                                                                                                    << (Qt3DCore::QNodeIdVector()
+                                                                                                                        << rootEntity->id()
+                                                                                                                        << childEntity1->id()
+                                                                                                                        << childEntity4->id()
+                                                                                                                        );
+
+            QTest::newRow("NestedFiltering-SelectAllNoneOfAnyLayer45AndAnyOf23AndAllOf16-ShouldSelectChild4-Step2") << rootEntity
+                                                                                                                    << (Qt3DCore::QNodeIdVector() << layerFilter->id() << layerFilter2->id())
+                                                                                                                    << (Qt3DCore::QNodeIdVector()
+                                                                                                                        << childEntity1->id()
+                                                                                                                        << childEntity4->id());
+
+            QTest::newRow("NestedFiltering-SelectAllNoneOfAnyLayer45AndAnyOf23AndAllOf16-ShouldSelectChild4-Step3") << rootEntity
+                                                                                                                    << (Qt3DCore::QNodeIdVector() << layerFilter->id() << layerFilter2->id() << layerFilter3->id())
+                                                                                                                    << (Qt3DCore::QNodeIdVector()
+                                                                                                                        << childEntity4->id());
         }
     }
 
@@ -326,13 +621,9 @@ private Q_SLOTS:
     {
         //QSKIP("Skipping until TestAspect can be registered");
         QFETCH(Qt3DCore::QEntity *, entitySubtree);
-        QFETCH(Qt3DCore::QNodeIdVector, layerIds);
-        QFETCH(bool, hasLayerFilter);
-        QFETCH(Qt3DRender::QLayerFilter::FilterMode, filterMode);
-        QFETCH(bool, recursive);
+        QFETCH(Qt3DCore::QNodeIdVector, layerFilterIds);
         QFETCH(Qt3DCore::QNodeIdVector, expectedSelectedEntities);
 
-        Q_UNUSED(recursive);
 
         // GIVEN
         QScopedPointer<Qt3DRender::TestAspect> aspect(new Qt3DRender::TestAspect(entitySubtree));
@@ -345,17 +636,15 @@ private Q_SLOTS:
 
         // WHEN
         Qt3DRender::Render::FilterLayerEntityJob filterJob;
-        filterJob.setHasLayerFilter(hasLayerFilter);
-        filterJob.setLayers(layerIds);
-        filterJob.setFilterMode(filterMode);
+        filterJob.setLayerFilters(layerFilterIds);
         filterJob.setManager(aspect->nodeManagers());
         filterJob.run();
 
         // THEN
         const QVector<Qt3DRender::Render::Entity *> filterEntities = filterJob.filteredEntities();
-        QCOMPARE(expectedSelectedEntities.size(), filterEntities.size());
+        QCOMPARE(filterEntities.size(), expectedSelectedEntities.size());
         for (auto i = 0, m = expectedSelectedEntities.size(); i < m; ++i)
-            QCOMPARE(expectedSelectedEntities.at(i), filterEntities.at(i)->peerId());
+            QCOMPARE(filterEntities.at(i)->peerId(), expectedSelectedEntities.at(i));
     }
 };
 
diff --git a/tests/auto/render/renderviewbuilder/tst_renderviewbuilder.cpp b/tests/auto/render/renderviewbuilder/tst_renderviewbuilder.cpp
index 1f2f304df..18ad6357e 100644
--- a/tests/auto/render/renderviewbuilder/tst_renderviewbuilder.cpp
+++ b/tests/auto/render/renderviewbuilder/tst_renderviewbuilder.cpp
@@ -361,7 +361,7 @@ private Q_SLOTS:
         // THEN
         QCOMPARE(renderViewBuilder.frustumCullingJob()->isActive(), false);
         QCOMPARE(renderViewBuilder.filterEntityByLayerJob()->hasLayerFilter(), false);
-        QCOMPARE(renderViewBuilder.filterEntityByLayerJob()->layers().size(), 0);
+        QCOMPARE(renderViewBuilder.filterEntityByLayerJob()->layerFilters().size(), 0);
         for (const auto materialGatherer : renderViewBuilder.materialGathererJobs()) {
             QVERIFY(materialGatherer->techniqueFilter() == nullptr);
             QVERIFY(materialGatherer->renderPassFilter() == nullptr);
@@ -374,7 +374,7 @@ private Q_SLOTS:
         // THEN
         QCOMPARE(renderViewBuilder.frustumCullingJob()->isActive(), true);
         QCOMPARE(renderViewBuilder.filterEntityByLayerJob()->hasLayerFilter(), true);
-        QCOMPARE(renderViewBuilder.filterEntityByLayerJob()->layers().size(), 1);
+        QCOMPARE(renderViewBuilder.filterEntityByLayerJob()->layerFilters().size(), 1);
         for (const auto materialGatherer : renderViewBuilder.materialGathererJobs()) {
             QVERIFY(materialGatherer->techniqueFilter() != nullptr);
             QVERIFY(materialGatherer->renderPassFilter() != nullptr);
diff --git a/tests/benchmarks/render/layerfiltering/tst_bench_layerfiltering.cpp b/tests/benchmarks/render/layerfiltering/tst_bench_layerfiltering.cpp
index cea52306f..d1ef468b2 100644
--- a/tests/benchmarks/render/layerfiltering/tst_bench_layerfiltering.cpp
+++ b/tests/benchmarks/render/layerfiltering/tst_bench_layerfiltering.cpp
@@ -38,6 +38,7 @@
 #include <Qt3DRender/private/qrenderaspect_p.h>
 #include <Qt3DRender/private/filterlayerentityjob_p.h>
 #include <Qt3DRender/qlayer.h>
+#include <Qt3DRender/qlayerfilter.h>
 
 QT_BEGIN_NAMESPACE
 
@@ -85,19 +86,21 @@ namespace {
 
 Qt3DCore::QEntity *buildTestScene(int layersCount,
                                   int entityCount,
-                                  QVector<Qt3DCore::QNodeId> &layerIds,
+                                  QVector<Qt3DCore::QNodeId> &layerFilterIds,
                                   bool alwaysEnabled = true)
 {
     Qt3DCore::QEntity *root = new Qt3DCore::QEntity();
+    Qt3DRender::QLayerFilter *layerFilter = new Qt3DRender::QLayerFilter(root);
+    layerFilterIds.push_back(layerFilter->id());
 
     QVector<Qt3DRender::QLayer *> layers;
     layers.reserve(layersCount);
-    layerIds.reserve(layersCount);
+
 
     for (int i = 0; i < layersCount; ++i) {
         Qt3DRender::QLayer *layer = new Qt3DRender::QLayer(root);
         layers.push_back(layer);
-        layerIds.push_back(layer->id());
+        layerFilter->addLayer(layer);
     }
 
     for (int i = 0; i < entityCount; ++i) {
@@ -123,43 +126,38 @@ private Q_SLOTS:
     void filterEntities_data()
     {
         QTest::addColumn<Qt3DCore::QEntity *>("entitySubtree");
-        QTest::addColumn<Qt3DCore::QNodeIdVector>("layerIds");
-        QTest::addColumn<bool>("hasLayerFilter");
+        QTest::addColumn<Qt3DCore::QNodeIdVector>("layerFilterIds");
 
 
         {
-            Qt3DCore::QNodeIdVector layerIds;
-            Qt3DCore::QEntity *rootEntity = buildTestScene(0, 5000, layerIds);
+            Qt3DCore::QNodeIdVector layerFilterIds;
+            Qt3DCore::QEntity *rootEntity = buildTestScene(0, 5000, layerFilterIds);
 
             QTest::newRow("Filter-NoLayerFilterAllEnabled") << rootEntity
-                                                            << layerIds
-                                                            << (layerIds.size() != 0);
+                                                            << layerFilterIds;
         }
 
         {
-            Qt3DCore::QNodeIdVector layerIds;
-            Qt3DCore::QEntity *rootEntity = buildTestScene(0, 5000, layerIds, false);
+            Qt3DCore::QNodeIdVector layerFilterIds;
+            Qt3DCore::QEntity *rootEntity = buildTestScene(0, 5000, layerFilterIds, false);
             QTest::newRow("Filter-NoLayerFilterSomeDisabled") << rootEntity
-                                                              << layerIds
-                                                              << (layerIds.size() != 0);
+                                                              << layerFilterIds;
         }
 
         {
-            Qt3DCore::QNodeIdVector layerIds;
-            Qt3DCore::QEntity *rootEntity = buildTestScene(10, 5000, layerIds);
+            Qt3DCore::QNodeIdVector layerFilterIds;
+            Qt3DCore::QEntity *rootEntity = buildTestScene(10, 5000, layerFilterIds);
 
             QTest::newRow("FilterLayerFilterAllEnabled") << rootEntity
-                                                         << layerIds
-                                                         << (layerIds.size() != 0);
+                                                         << layerFilterIds;
         }
 
         {
-            Qt3DCore::QNodeIdVector layerIds;
-            Qt3DCore::QEntity *rootEntity = buildTestScene(10, 5000, layerIds, false);
+            Qt3DCore::QNodeIdVector layerFilterIds;
+            Qt3DCore::QEntity *rootEntity = buildTestScene(10, 5000, layerFilterIds, false);
 
             QTest::newRow("FilterLayerFilterSomeDisabled") << rootEntity
-                                                         << layerIds
-                                                         << (layerIds.size() != 0);
+                                                         << layerFilterIds;
         }
 
     }
@@ -167,16 +165,14 @@ private Q_SLOTS:
     void filterEntities()
     {
         QFETCH(Qt3DCore::QEntity *, entitySubtree);
-        QFETCH(Qt3DCore::QNodeIdVector, layerIds);
-        QFETCH(bool, hasLayerFilter);
+        QFETCH(Qt3DCore::QNodeIdVector, layerFilterIds);
 
         // GIVEN
         QScopedPointer<Qt3DRender::TestAspect> aspect(new Qt3DRender::TestAspect(entitySubtree));
 
         // WHEN
         Qt3DRender::Render::FilterLayerEntityJob filterJob;
-        filterJob.setHasLayerFilter(hasLayerFilter);
-        filterJob.setLayers(layerIds);
+        filterJob.setLayerFilters(layerFilterIds);
         filterJob.setManager(aspect->nodeManagers());
 
         QBENCHMARK {