Adjust LayerFilter's layers property behavior

Now an empty list filters out all layers instead of letting everything
go through. Also, enabled == false is now honored, a disabled filter
lets everything go through (whatever is the layers list).

Change-Id: I5ef183c27a9dd045f423c1bf13d563c61b432cfa
Task-number: QTBUG-51440
Reviewed-by: Sean Harmer <sean.harmer@kdab.com>

3 files changed, 13 insertions, 8 deletions

diff --git a/src/render/backend/renderview.cpp b/src/render/backend/renderview.cpp
index 1a7005558..514550ce6 100644
--- a/src/render/backend/renderview.cpp
+++ b/src/render/backend/renderview.cpp
@@ -100,11 +100,11 @@ int LIGHT_COLOR_NAMES[MAX_LIGHTS];
 int LIGHT_INTENSITY_NAMES[MAX_LIGHTS];
 QString LIGHT_STRUCT_NAMES[MAX_LIGHTS];
 
-// TODO: Should we treat lack of layer data as implicitly meaning that an
-// entity is in all layers?
-bool isEntityInLayers(const Entity *entity, const QVector<int> &filterLayerIds)
+bool isEntityInLayers(const Entity *entity, bool hasLayerFilter, const QVector<int> &filterLayerIds)
 {
-    if (filterLayerIds.isEmpty())
+    if (hasLayerFilter && filterLayerIds.isEmpty())
+        return false;
+    else if (filterLayerIds.isEmpty())
         return true;
 
     const QList<Layer *> entityLayers = entity->renderComponents<Layer>();
@@ -490,20 +490,20 @@ void RenderView::addClearBuffers(const ClearBuffers *cb) {
 }
 
 // TODO: Convert into a job that caches lookup of entities for layers
-static void findEntitiesInLayers(Entity *e, const QVector<int> &filterLayerIds, QVector<Entity *> &entities)
+static void findEntitiesInLayers(Entity *e, bool hasLayerFilter, const QVector<int> &filterLayerIds, QVector<Entity *> &entities)
 {
     // Bail if sub-tree is disabled
     if (!e->isEnabled())
         return;
 
     // Check this entity
-    if (isEntityInLayers(e, filterLayerIds))
+    if (isEntityInLayers(e, hasLayerFilter, filterLayerIds))
         entities.push_back(e);
 
     // Recurse to children
     const auto children = e->children();
     for (Entity *child : children)
-        findEntitiesInLayers(child, filterLayerIds, entities);
+        findEntitiesInLayers(child, hasLayerFilter, filterLayerIds, entities);
 }
 
 // Tries to order renderCommand by shader so as to minimize shader changes
@@ -517,7 +517,7 @@ void RenderView::buildRenderCommands(Entity *rootEntity, const Plane *planes)
     QVector<Entity *> entities;
     const int entityCount = m_renderer->nodeManagers()->renderNodesManager()->count();
     entities.reserve(entityCount);
-    findEntitiesInLayers(rootEntity, m_data->m_layerIds, entities);
+    findEntitiesInLayers(rootEntity, hasLayerFilter(), layerFilterIds(), entities);
 #if defined(QT3D_RENDER_VIEW_JOB_TIMINGS)
     qDebug() << "Found" << entities.size() << "entities in layers" << m_data->m_layers
              << "in" << timer.nsecsElapsed() / 1.0e6 << "ms";
diff --git a/src/render/backend/renderview_p.h b/src/render/backend/renderview_p.h
index bc634a65d..10d053be9 100644
--- a/src/render/backend/renderview_p.h
+++ b/src/render/backend/renderview_p.h
@@ -155,6 +155,8 @@ public:
     inline void setEyePosition(const QVector3D &eyePos) { m_data->m_eyePos = eyePos; }
     inline QVector3D eyePosition() const { return m_data->m_eyePos; }
 
+    inline void setHasLayerFilter(bool filter) { m_data->m_hasLayerFilter = filter; }
+    inline bool hasLayerFilter() const { return m_data->m_hasLayerFilter; }
     inline void appendLayerFilter(const QStringList &layers) { m_data->m_layers << layers; }
     inline QStringList layerFilters() const { return m_data->m_layers; }
     inline void appendLayerFilter(const QVector<int> &layerIds) { m_data->m_layerIds << layerIds; }
@@ -240,6 +242,7 @@ public:
             , m_techniqueFilter(nullptr)
             , m_passFilter(nullptr)
             , m_viewMatrix(nullptr)
+            , m_hasLayerFilter(false)
         {
         }
         CameraLens *m_renderCamera;
@@ -247,6 +250,7 @@ public:
         const RenderPassFilter *m_passFilter;
         QMatrix4x4 *m_viewMatrix;
         QMatrix4x4 *m_viewProjectionMatrix;
+        bool m_hasLayerFilter;
         QStringList m_layers; // Only for debug
         QVector<int> m_layerIds;
         QVector<Qt3DRender::QSortPolicy::SortType> m_sortingTypes;
diff --git a/src/render/jobs/renderviewjobutils.cpp b/src/render/jobs/renderviewjobutils.cpp
index 249707547..c22f84072 100644
--- a/src/render/jobs/renderviewjobutils.cpp
+++ b/src/render/jobs/renderviewjobutils.cpp
@@ -114,6 +114,7 @@ void setRenderViewConfigFromFrameGraphLeafNode(RenderView *rv, const FrameGraphN
                 }
 
             case FrameGraphNode::LayerFilter: // Can be set multiple times in the tree
+                rv->setHasLayerFilter(true);
                 rv->appendLayerFilter(static_cast<const LayerFilterNode *>(node)->layerIds());
                 rv->appendLayerFilter(static_cast<const LayerFilterNode *>(node)->layers()); // For debug
                 break;