QObjectPicker and QPickingSettings NearestPriorityPick picking mode

Add a new priority property on QObjectPicker and a new QPickingSettings mode.
This will select the result with the highest priority if there are several results
on a given ray. If we have several results with identical properties, the result
with the closest distance is selected.

[ChangeLog][Qt3DRender] QObjectPicker: add a priority based result selection

Change-Id: I7af12db6e163c3c2d9dad696e6d9f9bbbee064ed
Reviewed-by: Mike Krus <mike.krus@kdab.com>

4 files changed, 137 insertions, 31 deletions

diff --git a/src/render/jobs/pickboundingvolumejob.cpp b/src/render/jobs/pickboundingvolumejob.cpp
index 2050b8772..96ec11b4e 100644
--- a/src/render/jobs/pickboundingvolumejob.cpp
+++ b/src/render/jobs/pickboundingvolumejob.cpp
@@ -211,7 +211,6 @@ bool PickBoundingVolumeJob::runHelper()
     const bool edgePickingRequested = (m_renderSettings->pickMethod() & QPickingSettings::LinePicking);
     const bool pointPickingRequested = (m_renderSettings->pickMethod() & QPickingSettings::PointPicking);
     const bool primitivePickingRequested = pointPickingRequested | edgePickingRequested | trianglePickingRequested;
-    const bool allHitsRequested = (m_renderSettings->pickResultMode() == QPickingSettings::AllPicks);
     const bool frontFaceRequested =
             m_renderSettings->faceOrientationPickingMode() != QPickingSettings::BackFace;
     const bool backFaceRequested =
@@ -237,7 +236,7 @@ bool PickBoundingVolumeJob::runHelper()
                 // has moved out of the viewport In case of a button released
                 // outside of the viewport, we still want to notify the
                 // lastCurrent entity about this.
-                dispatchPickEvents(event.second, PickingUtils::HitList(), eventButton, eventButtons, eventModifiers, allHitsRequested);
+                dispatchPickEvents(event.second, PickingUtils::HitList(), eventButton, eventButtons, eventModifiers, m_renderSettings->pickResultMode());
                 continue;
             }
 
@@ -249,14 +248,16 @@ bool PickBoundingVolumeJob::runHelper()
                     gathererFunctor.m_backFaceRequested = backFaceRequested;
                     gathererFunctor.m_manager = m_manager;
                     gathererFunctor.m_ray = ray;
-                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), allHitsRequested);
+                    gathererFunctor.m_entityToPriorityTable = entityPicker.entityToPriorityTable();
+                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), m_renderSettings->pickResultMode());
                 }
                 if (edgePickingRequested) {
                     PickingUtils::LineCollisionGathererFunctor gathererFunctor;
                     gathererFunctor.m_manager = m_manager;
                     gathererFunctor.m_ray = ray;
                     gathererFunctor.m_pickWorldSpaceTolerance = pickWorldSpaceTolerance;
-                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), allHitsRequested);
+                    gathererFunctor.m_entityToPriorityTable = entityPicker.entityToPriorityTable();
+                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), m_renderSettings->pickResultMode());
                     PickingUtils::AbstractCollisionGathererFunctor::sortHits(sphereHits);
                 }
                 if (pointPickingRequested) {
@@ -264,19 +265,20 @@ bool PickBoundingVolumeJob::runHelper()
                     gathererFunctor.m_manager = m_manager;
                     gathererFunctor.m_ray = ray;
                     gathererFunctor.m_pickWorldSpaceTolerance = pickWorldSpaceTolerance;
-                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), allHitsRequested);
+                    gathererFunctor.m_entityToPriorityTable = entityPicker.entityToPriorityTable();
+                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), m_renderSettings->pickResultMode());
                     PickingUtils::AbstractCollisionGathererFunctor::sortHits(sphereHits);
                 }
                 if (!primitivePickingRequested) {
                     sphereHits << entityPicker.hits();
                     PickingUtils::AbstractCollisionGathererFunctor::sortHits(sphereHits);
-                    if (!allHitsRequested)
+                    if (m_renderSettings->pickResultMode() != QPickingSettings::AllPicks)
                         sphereHits = { sphereHits.front() };
                 }
             }
 
             // Dispatch events based on hit results
-            dispatchPickEvents(event.second, sphereHits, eventButton, eventButtons, eventModifiers, allHitsRequested);
+            dispatchPickEvents(event.second, sphereHits, eventButton, eventButtons, eventModifiers, m_renderSettings->pickResultMode());
         }
     }
 
diff --git a/src/render/jobs/pickboundingvolumeutils.cpp b/src/render/jobs/pickboundingvolumeutils.cpp
index 5fed946d6..23e495ecb 100644
--- a/src/render/jobs/pickboundingvolumeutils.cpp
+++ b/src/render/jobs/pickboundingvolumeutils.cpp
@@ -55,6 +55,7 @@
 
 #include <vector>
 #include <algorithm>
+#include <functional>
 
 QT_BEGIN_NAMESPACE
 
@@ -418,6 +419,46 @@ HitList reduceToFirstHit(HitList &result, const HitList &intermediate)
     return result;
 }
 
+
+struct HighestPriorityHitReducer
+{
+    // No need to protect this from concurrent access as the table
+    // is read only
+    const QHash<Qt3DCore::QNodeId, int> entityToPriorityTable;
+
+    HitList operator()(HitList &result, const HitList &intermediate)
+    {
+        // Sort by priority first
+        // If we have equal priorities, we then sort by distance
+
+        if (!intermediate.empty()) {
+            if (result.empty())
+                result.push_back(intermediate.front());
+            int currentPriority = entityToPriorityTable.value(result.front().m_entityId, 0);
+            float closest = result.front().m_distance;
+
+            for (const auto &v : intermediate) {
+                const int newEntryPriority = entityToPriorityTable.value(v.m_entityId, 0);
+                if (newEntryPriority > currentPriority) {
+                    result.push_front(v);
+                    currentPriority = newEntryPriority;
+                    closest = v.m_distance;
+                } else if (newEntryPriority == currentPriority) {
+                    if (v.m_distance < closest) {
+                        result.push_front(v);
+                        closest = v.m_distance;
+                        currentPriority = newEntryPriority;
+                    }
+                }
+            }
+
+            while (result.size() > 1)
+                result.pop_back();
+        }
+        return result;
+    }
+};
+
 HitList reduceToAllHits(HitList &results, const HitList &intermediate)
 {
     if (!intermediate.empty())
@@ -492,9 +533,22 @@ struct MapFunctorHolder
 
 } // anonymous
 
-HitList EntityCollisionGathererFunctor::computeHits(const QVector<Entity *> &entities, bool allHitsRequested)
-{
-    const auto reducerOp = allHitsRequested ? PickingUtils::reduceToAllHits : PickingUtils::reduceToFirstHit;
+HitList EntityCollisionGathererFunctor::computeHits(const QVector<Entity *> &entities,
+                                                    Qt3DRender::QPickingSettings::PickResultMode mode)
+{
+    std::function<HitList (HitList &, const HitList &)> reducerOp;
+    switch (mode) {
+    case QPickingSettings::AllPicks:
+        reducerOp = PickingUtils::reduceToAllHits;
+        break;
+    case QPickingSettings::NearestPriorityPick:
+        reducerOp = HighestPriorityHitReducer{ m_entityToPriorityTable };
+        break;
+    case QPickingSettings::NearestPick:
+        reducerOp = PickingUtils::reduceToFirstHit;
+        break;
+    }
+
     const MapFunctorHolder holder(this);
 #if QT_CONFIG(concurrent)
     return QtConcurrent::blockingMappedReduced<HitList>(entities, holder, reducerOp);
@@ -519,9 +573,22 @@ HitList EntityCollisionGathererFunctor::pick(const Entity *entity) const
     return result;
 }
 
-HitList TriangleCollisionGathererFunctor::computeHits(const QVector<Entity *> &entities, bool allHitsRequested)
-{
-    const auto reducerOp = allHitsRequested ? PickingUtils::reduceToAllHits : PickingUtils::reduceToFirstHit;
+HitList TriangleCollisionGathererFunctor::computeHits(const QVector<Entity *> &entities,
+                                                      Qt3DRender::QPickingSettings::PickResultMode mode)
+{
+    std::function<HitList (HitList &, const HitList &)> reducerOp;
+    switch (mode) {
+    case QPickingSettings::AllPicks:
+        reducerOp = PickingUtils::reduceToAllHits;
+        break;
+    case QPickingSettings::NearestPriorityPick:
+        reducerOp = HighestPriorityHitReducer { m_entityToPriorityTable };
+        break;
+    case QPickingSettings::NearestPick:
+        reducerOp = PickingUtils::reduceToFirstHit;
+        break;
+    }
+
     const MapFunctorHolder holder(this);
 #if QT_CONFIG(concurrent)
     return QtConcurrent::blockingMappedReduced<HitList>(entities, holder, reducerOp);
@@ -553,9 +620,22 @@ HitList TriangleCollisionGathererFunctor::pick(const Entity *entity) const
     return result;
 }
 
-HitList LineCollisionGathererFunctor::computeHits(const QVector<Entity *> &entities, bool allHitsRequested)
-{
-    const auto reducerOp = allHitsRequested ? PickingUtils::reduceToAllHits : PickingUtils::reduceToFirstHit;
+HitList LineCollisionGathererFunctor::computeHits(const QVector<Entity *> &entities,
+                                                  Qt3DRender::QPickingSettings::PickResultMode mode)
+{
+    std::function<HitList (HitList &, const HitList &)> reducerOp;
+    switch (mode) {
+    case QPickingSettings::AllPicks:
+        reducerOp = PickingUtils::reduceToAllHits;
+        break;
+    case QPickingSettings::NearestPriorityPick:
+        reducerOp = HighestPriorityHitReducer { m_entityToPriorityTable };
+        break;
+    case QPickingSettings::NearestPick:
+        reducerOp = PickingUtils::reduceToFirstHit;
+        break;
+    }
+
     const MapFunctorHolder holder(this);
 #if QT_CONFIG(concurrent)
     return QtConcurrent::blockingMappedReduced<HitList>(entities, holder, reducerOp);
@@ -586,9 +666,22 @@ HitList LineCollisionGathererFunctor::pick(const Entity *entity) const
     return result;
 }
 
-HitList PointCollisionGathererFunctor::computeHits(const QVector<Entity *> &entities, bool allHitsRequested)
-{
-    const auto reducerOp = allHitsRequested ? PickingUtils::reduceToAllHits : PickingUtils::reduceToFirstHit;
+HitList PointCollisionGathererFunctor::computeHits(const QVector<Entity *> &entities,
+                                                   Qt3DRender::QPickingSettings::PickResultMode mode)
+{
+    std::function<HitList (HitList &, const HitList &)> reducerOp;
+    switch (mode) {
+    case QPickingSettings::AllPicks:
+        reducerOp = PickingUtils::reduceToAllHits;
+        break;
+    case QPickingSettings::NearestPriorityPick:
+        reducerOp = HighestPriorityHitReducer { m_entityToPriorityTable };
+        break;
+    case QPickingSettings::NearestPick:
+        reducerOp = PickingUtils::reduceToFirstHit;
+        break;
+    }
+
     const MapFunctorHolder holder(this);
 #if QT_CONFIG(concurrent)
     return QtConcurrent::blockingMappedReduced<HitList>(entities, holder, reducerOp);
@@ -641,15 +734,17 @@ bool HierarchicalEntityPicker::collectHits(NodeManagers *manager, Entity *root)
 {
     m_hits.clear();
     m_entities.clear();
+    m_entityToPriorityTable.clear();
 
     QRayCastingService rayCasting;
     struct EntityData {
         Entity* entity;
         bool hasObjectPicker;
         Qt3DCore::QNodeIdVector recursiveLayers;
+        int priority;
     };
     std::vector<EntityData> worklist;
-    worklist.push_back({root, !root->componentHandle<ObjectPicker>().isNull(), {}});
+    worklist.push_back({root, !root->componentHandle<ObjectPicker>().isNull(), {}, 0});
 
     LayerManager *layerManager = manager->layerManager();
 
@@ -710,6 +805,8 @@ bool HierarchicalEntityPicker::collectHits(NodeManagers *manager, Entity *root)
         if (accepted && queryResult.m_distance >= 0.f && (current.hasObjectPicker || !m_objectPickersRequired)) {
             m_entities.push_back(current.entity);
             m_hits.push_back(queryResult);
+            // Record entry for entity/priority
+            m_entityToPriorityTable.insert(current.entity->peerId(), current.priority);
         }
 
         Qt3DCore::QNodeIdVector recursiveLayers;
@@ -722,9 +819,12 @@ bool HierarchicalEntityPicker::collectHits(NodeManagers *manager, Entity *root)
 
         // and pick children
         const auto children = current.entity->children();
-        for (auto child: children)
-            worklist.push_back({child, current.hasObjectPicker || !child->componentHandle<ObjectPicker>().isNull(),
-                                current.recursiveLayers + recursiveLayers});
+        for (Entity *child: children) {
+            ObjectPicker *childPicker = child->renderComponent<ObjectPicker>();
+            worklist.push_back({child, current.hasObjectPicker || childPicker,
+                                current.recursiveLayers + recursiveLayers,
+                                (childPicker ? childPicker->priority() : current.priority)});
+        }
     }
 
     return !m_hits.empty();
diff --git a/src/render/jobs/pickboundingvolumeutils_p.h b/src/render/jobs/pickboundingvolumeutils_p.h
index 780c16cc8..3fc4517c3 100644
--- a/src/render/jobs/pickboundingvolumeutils_p.h
+++ b/src/render/jobs/pickboundingvolumeutils_p.h
@@ -55,6 +55,7 @@
 #include <Qt3DRender/QAbstractRayCaster>
 #include <Qt3DRender/private/qray3d_p.h>
 #include <Qt3DRender/private/qraycastingservice_p.h>
+#include <Qt3DRender/qpickingsettings.h>
 
 
 QT_BEGIN_NAMESPACE
@@ -124,6 +125,7 @@ public:
     bool collectHits(NodeManagers *manager, Entity *root);
     inline HitList hits() const { return m_hits; }
     inline QVector<Entity *> entities() const { return m_entities; }
+    inline QHash<Qt3DCore::QNodeId, int> entityToPriorityTable() const { return m_entityToPriorityTable; }
 
 private:
     RayCasting::QRay3D m_ray;
@@ -132,6 +134,7 @@ private:
     bool m_objectPickersRequired;
     Qt3DCore::QNodeIdVector m_layerIds;
     QAbstractRayCaster::FilterMode m_filterMode;
+    QHash<Qt3DCore::QNodeId, int> m_entityToPriorityTable;
 };
 
 struct Q_AUTOTEST_EXPORT AbstractCollisionGathererFunctor
@@ -142,8 +145,9 @@ struct Q_AUTOTEST_EXPORT AbstractCollisionGathererFunctor
     bool m_objectPickersRequired = true;
     NodeManagers *m_manager = nullptr;
     RayCasting::QRay3D m_ray;
+    QHash<Qt3DCore::QNodeId, int> m_entityToPriorityTable;
 
-    virtual HitList computeHits(const QVector<Entity *> &entities, bool allHitsRequested) = 0;
+    virtual HitList computeHits(const QVector<Entity *> &entities, Qt3DRender::QPickingSettings::PickResultMode mode) = 0;
 
     // This define is required to work with QtConcurrent
     typedef HitList result_type;
@@ -156,7 +160,7 @@ struct Q_AUTOTEST_EXPORT AbstractCollisionGathererFunctor
 
 struct Q_AUTOTEST_EXPORT EntityCollisionGathererFunctor : public AbstractCollisionGathererFunctor
 {
-    HitList computeHits(const QVector<Entity *> &entities, bool allHitsRequested) override;
+    HitList computeHits(const QVector<Entity *> &entities, Qt3DRender::QPickingSettings::PickResultMode mode) override;
     HitList pick(const Entity *entity) const override;
 };
 
@@ -165,7 +169,7 @@ struct Q_AUTOTEST_EXPORT TriangleCollisionGathererFunctor : public AbstractColli
     bool m_frontFaceRequested;
     bool m_backFaceRequested;
 
-    HitList computeHits(const QVector<Entity *> &entities, bool allHitsRequested) override;
+    HitList computeHits(const QVector<Entity *> &entities, Qt3DRender::QPickingSettings::PickResultMode mode) override;
     HitList pick(const Entity *entity) const override;
 };
 
@@ -173,7 +177,7 @@ struct Q_AUTOTEST_EXPORT LineCollisionGathererFunctor : public AbstractCollision
 {
     float m_pickWorldSpaceTolerance;
 
-    HitList computeHits(const QVector<Entity *> &entities, bool allHitsRequested) override;
+    HitList computeHits(const QVector<Entity *> &entities, Qt3DRender::QPickingSettings::PickResultMode mode) override;
     HitList pick(const Entity *entity) const override;
 };
 
@@ -181,7 +185,7 @@ struct Q_AUTOTEST_EXPORT PointCollisionGathererFunctor : public AbstractCollisio
 {
     float m_pickWorldSpaceTolerance;
 
-    HitList computeHits(const QVector<Entity *> &entities, bool allHitsRequested) override;
+    HitList computeHits(const QVector<Entity *> &entities, Qt3DRender::QPickingSettings::PickResultMode mode) override;
     HitList pick(const Entity *entity) const override;
 };
 
diff --git a/src/render/jobs/raycastingjob.cpp b/src/render/jobs/raycastingjob.cpp
index e76b9fe8d..70c7ac374 100644
--- a/src/render/jobs/raycastingjob.cpp
+++ b/src/render/jobs/raycastingjob.cpp
@@ -183,7 +183,7 @@ bool RayCastingJob::runHelper()
                     gathererFunctor.m_manager = m_manager;
                     gathererFunctor.m_ray = ray;
                     gathererFunctor.m_objectPickersRequired = false;
-                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), true);
+                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), QPickingSettings::AllPicks);
                 }
                 if (edgePickingRequested) {
                     PickingUtils::LineCollisionGathererFunctor gathererFunctor;
@@ -191,7 +191,7 @@ bool RayCastingJob::runHelper()
                     gathererFunctor.m_ray = ray;
                     gathererFunctor.m_pickWorldSpaceTolerance = pickWorldSpaceTolerance;
                     gathererFunctor.m_objectPickersRequired = false;
-                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), true);
+                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), QPickingSettings::AllPicks);
                     PickingUtils::AbstractCollisionGathererFunctor::sortHits(sphereHits);
                 }
                 if (pointPickingRequested) {
@@ -200,7 +200,7 @@ bool RayCastingJob::runHelper()
                     gathererFunctor.m_ray = ray;
                     gathererFunctor.m_pickWorldSpaceTolerance = pickWorldSpaceTolerance;
                     gathererFunctor.m_objectPickersRequired = false;
-                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), true);
+                    sphereHits << gathererFunctor.computeHits(entityPicker.entities(), QPickingSettings::AllPicks);
                     PickingUtils::AbstractCollisionGathererFunctor::sortHits(sphereHits);
                 }
                 if (!primitivePickingRequested) {