Reinstate back face picking

If enabled and front face picking fails, try points reverse order.
Handling pick results appropriately in unit tests.

Don’t use the triangle bounding volume and ray casting service to
avoid large amount of unnecessary object creation and data copying.

Task-number: QTBUG-57876
Change-Id: I4c4f570cb8de1fa33fcde117f1543e8625633817
Reviewed-by: Sean Harmer <sean.harmer@kdab.com>

5 files changed, 63 insertions, 17 deletions

diff --git a/src/render/backend/triangleboundingvolume.cpp b/src/render/backend/triangleboundingvolume.cpp
index fcc60702e..deef28a44 100644
--- a/src/render/backend/triangleboundingvolume.cpp
+++ b/src/render/backend/triangleboundingvolume.cpp
@@ -116,7 +116,7 @@ bool TriangleBoundingVolume::intersects(const QRay3D &ray, QVector3D *q) const
 {
     float t = 0.0f;
     QVector3D uvw;
-    const float intersected = intersectsSegmentTriangle(ray, m_c, m_b, m_a, uvw, t);
+    const bool intersected = intersectsSegmentTriangle(ray, m_c, m_b, m_a, uvw, t);
 
     if (intersected && q != nullptr)
         *q = ray.point(t * ray.distance());
diff --git a/src/render/jobs/pickboundingvolumejob.cpp b/src/render/jobs/pickboundingvolumejob.cpp
index 78f65228d..2ce6eb92e 100644
--- a/src/render/jobs/pickboundingvolumejob.cpp
+++ b/src/render/jobs/pickboundingvolumejob.cpp
@@ -183,6 +183,10 @@ bool PickBoundingVolumeJob::runHelper()
 
     const bool trianglePickingRequested = (m_renderSettings->pickMethod() == QPickingSettings::TrianglePicking);
     const bool allHitsRequested = (m_renderSettings->pickResultMode() == QPickingSettings::AllPicks);
+    const bool frontFaceRequested =
+            m_renderSettings->faceOrientationPickingMode() != QPickingSettings::BackFace;
+    const bool backFaceRequested =
+            m_renderSettings->faceOrientationPickingMode() != QPickingSettings::FrontFace;
 
     // Select the best reduction function based on the settings
     const ReducerFunction reducerOp = allHitsRequested ? PickingUtils::reduceToAllHits : PickingUtils::reduceToFirstHit;
@@ -204,6 +208,8 @@ bool PickBoundingVolumeJob::runHelper()
             QRay3D ray = rayForViewportAndCamera(vca.area, event.pos(), vca.viewport, vca.cameraId);
             if (trianglePickingRequested) {
                 PickingUtils::TriangleCollisionGathererFunctor gathererFunctor;
+                gathererFunctor.m_frontFaceRequested = frontFaceRequested;
+                gathererFunctor.m_backFaceRequested = backFaceRequested;
                 gathererFunctor.m_manager = m_manager;
                 gathererFunctor.m_ray = ray;
                 sphereHits = QtConcurrent::blockingMappedReduced<HitList>(entitiesGatherer.entities(), gathererFunctor, reducerOp);
diff --git a/src/render/jobs/pickboundingvolumeutils.cpp b/src/render/jobs/pickboundingvolumeutils.cpp
index 1979c61b5..d0618866b 100644
--- a/src/render/jobs/pickboundingvolumeutils.cpp
+++ b/src/render/jobs/pickboundingvolumeutils.cpp
@@ -146,19 +146,37 @@ QVector<Entity *> EntityGatherer::entities() const
 
 void CollisionVisitor::visit(uint andx, const QVector3D &a, uint bndx, const QVector3D &b, uint cndx, const QVector3D &c)
 {
-    TriangleBoundingVolume volume(m_root->peerId(), a, b, c);
-    volume = volume.transform(*m_root->worldTransform());
+    const QMatrix4x4 &mat = *m_root->worldTransform();
+    const QVector3D tA = mat * a;
+    const QVector3D tB = mat * b;
+    const QVector3D tC = mat * c;
+
+    bool intersected = m_frontFaceRequested &&
+            intersectsSegmentTriangle(cndx, tC, bndx, tB, andx, tA);    // front facing
+    if (!intersected && m_backFaceRequested) {
+        intersected = intersectsSegmentTriangle(andx, tA, bndx, tB, cndx, tC);    // back facing
+    }
+
+    m_triangleIndex++;
+}
 
-    QCollisionQueryResult::Hit queryResult = rayCasting.query(m_ray, &volume);
-    if (queryResult.m_distance > 0.) {
+bool CollisionVisitor::intersectsSegmentTriangle(uint andx, const QVector3D &a, uint bndx, const QVector3D &b, uint cndx, const QVector3D &c)
+{
+    float t = 0.0f;
+    QVector3D uvw;
+    bool intersected = Render::intersectsSegmentTriangle(m_ray, a, b, c, uvw, t);
+    if (intersected) {
+        QCollisionQueryResult::Hit queryResult;
+        queryResult.m_entityId = m_root->peerId();
         queryResult.m_triangleIndex = m_triangleIndex;
         queryResult.m_vertexIndex[0] = andx;
         queryResult.m_vertexIndex[1] = bndx;
         queryResult.m_vertexIndex[2] = cndx;
+        queryResult.m_intersection = m_ray.point(t * m_ray.distance());
+        queryResult.m_distance = m_ray.projectedDistance(queryResult.m_intersection);
         hits.push_back(queryResult);
     }
-
-    m_triangleIndex++;
+    return intersected;
 }
 
 AbstractCollisionGathererFunctor::AbstractCollisionGathererFunctor()
@@ -211,7 +229,7 @@ AbstractCollisionGathererFunctor::result_type TriangleCollisionGathererFunctor::
         return result;
 
     if (rayHitsEntity(rayCasting, entity)) {
-        CollisionVisitor visitor(m_manager, entity, m_ray);
+        CollisionVisitor visitor(m_manager, entity, m_ray, m_frontFaceRequested, m_backFaceRequested);
         visitor.apply(gRenderer, entity->peerId());
         result = visitor.hits;
 
diff --git a/src/render/jobs/pickboundingvolumeutils_p.h b/src/render/jobs/pickboundingvolumeutils_p.h
index 5bc94a13b..08b04f7f0 100644
--- a/src/render/jobs/pickboundingvolumeutils_p.h
+++ b/src/render/jobs/pickboundingvolumeutils_p.h
@@ -110,16 +110,25 @@ public:
     typedef QVector<QCollisionQueryResult::Hit> HitList;
     HitList hits;
 
-    CollisionVisitor(NodeManagers* manager, const Entity *root, const QRay3D& ray) : TrianglesVisitor(manager), m_root(root), m_ray(ray), m_triangleIndex(0) { }
+    CollisionVisitor(NodeManagers* manager, const Entity *root, const QRay3D& ray, bool frontFaceRequested, bool backFaceRequested)
+        : TrianglesVisitor(manager), m_root(root), m_ray(ray), m_triangleIndex(0)
+        , m_frontFaceRequested(frontFaceRequested), m_backFaceRequested(backFaceRequested)
+    {
+    }
+
 private:
     const Entity *m_root;
     QRay3D m_ray;
-    Qt3DRender::QRayCastingService rayCasting;
     uint m_triangleIndex;
+    bool m_frontFaceRequested;
+    bool m_backFaceRequested;
 
     void visit(uint andx, const QVector3D &a,
                uint bndx, const QVector3D &b,
-               uint cndx, const QVector3D &c);
+               uint cndx, const QVector3D &c) Q_DECL_OVERRIDE;
+    bool intersectsSegmentTriangle(uint andx, const QVector3D &a,
+                                   uint bndx, const QVector3D &b,
+                                   uint cndx, const QVector3D &c);
 };
 
 struct Q_AUTOTEST_EXPORT AbstractCollisionGathererFunctor
@@ -143,6 +152,9 @@ struct Q_AUTOTEST_EXPORT EntityCollisionGathererFunctor : public AbstractCollisi
 
 struct Q_AUTOTEST_EXPORT TriangleCollisionGathererFunctor : public AbstractCollisionGathererFunctor
 {
+    bool m_frontFaceRequested;
+    bool m_backFaceRequested;
+
     result_type pick(QAbstractCollisionQueryService *rayCasting, const Entity *entity) const Q_DECL_OVERRIDE;
 
     bool rayHitsEntity(QAbstractCollisionQueryService *rayCasting, const Entity *entity) const;
diff --git a/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp b/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
index 8814bda2c..7e885435b 100644
--- a/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
+++ b/tests/auto/render/pickboundingvolumejob/tst_pickboundingvolumejob.cpp
@@ -758,6 +758,11 @@ private Q_SLOTS:
         QCOMPARE(test->renderSettings()->pickResultMode(), pickResultMode);
         QCOMPARE(test->renderSettings()->faceOrientationPickingMode(), faceOrientationPickingMode);
 
+        const bool backAndFrontPicking =
+                (pickMethod == Qt3DRender::QPickingSettings::TrianglePicking) &&
+                (pickResultMode == Qt3DRender::QPickingSettings::AllPicks) &&
+                (faceOrientationPickingMode == Qt3DRender::QPickingSettings::FrontAndBackFace);
+
         // WHEN -> Pressed on object
         Qt3DRender::Render::PickBoundingVolumeJob pickBVJob;
         initializePickBoundingVolumeJob(&pickBVJob, test.data());
@@ -770,7 +775,7 @@ private Q_SLOTS:
         // THEN -> Pressed
         QVERIFY(!earlyReturn);
         QVERIFY(backendPicker1->isPressed());
-        QCOMPARE(arbiter.events.count(), 1);
+        QCOMPARE(arbiter.events.count(), backAndFrontPicking ? 2 : 1);
         Qt3DCore::QPropertyUpdatedChangePtr change = arbiter.events.first().staticCast<Qt3DCore::QPropertyUpdatedChange>();
         QCOMPARE(change->propertyName(), "pressed");
         Qt3DRender::QPickEventPtr pickEvent = change->value().value<Qt3DRender::QPickEventPtr>();
@@ -789,7 +794,7 @@ private Q_SLOTS:
         // THEN -> Moved
         QVERIFY(!earlyReturn);
         QVERIFY(backendPicker1->isPressed());
-        QCOMPARE(arbiter.events.count(), 1);
+        QCOMPARE(arbiter.events.count(), backAndFrontPicking ? 2 : 1);
         change = arbiter.events.first().staticCast<Qt3DCore::QPropertyUpdatedChange>();
         QCOMPARE(change->propertyName(), "moved");
         pickEvent = change->value().value<Qt3DRender::QPickEventPtr>();
@@ -834,7 +839,7 @@ private Q_SLOTS:
         // THEN -> Released
         QVERIFY(!earlyReturn);
         QVERIFY(!backendPicker1->isPressed());
-        QCOMPARE(arbiter.events.count(), 2);
+        QCOMPARE(arbiter.events.count(), backAndFrontPicking ? 3 : 2);
         change = arbiter.events.first().staticCast<Qt3DCore::QPropertyUpdatedChange>();
         QCOMPARE(change->propertyName(), "pressed");
         change = arbiter.events.last().staticCast<Qt3DCore::QPropertyUpdatedChange>();
@@ -986,6 +991,11 @@ private Q_SLOTS:
         QCOMPARE(test->renderSettings()->pickResultMode(), pickResultMode);
         QCOMPARE(test->renderSettings()->faceOrientationPickingMode(), faceOrientationPickingMode);
 
+        const bool backAndFrontPicking =
+                (pickMethod == Qt3DRender::QPickingSettings::TrianglePicking) &&
+                (pickResultMode == Qt3DRender::QPickingSettings::AllPicks) &&
+                (faceOrientationPickingMode == Qt3DRender::QPickingSettings::FrontAndBackFace);
+
         // WHEN -> Pressed on object
         Qt3DRender::Render::PickBoundingVolumeJob pickBVJob;
         initializePickBoundingVolumeJob(&pickBVJob, test.data());
@@ -998,7 +1008,7 @@ private Q_SLOTS:
         // THEN -> Pressed
         QVERIFY(!earlyReturn);
         QVERIFY(backendPicker->isPressed());
-        QCOMPARE(arbiter.events.count(), 1);
+        QCOMPARE(arbiter.events.count(), backAndFrontPicking ? 2 : 1);
         Qt3DCore::QPropertyUpdatedChangePtr change = arbiter.events.first().staticCast<Qt3DCore::QPropertyUpdatedChange>();
         QCOMPARE(change->propertyName(), "pressed");
 
@@ -1013,7 +1023,7 @@ private Q_SLOTS:
         // THEN -> Moved
         QVERIFY(!earlyReturn);
         QVERIFY(backendPicker->isPressed());
-        QCOMPARE(arbiter.events.count(), 1);
+        QCOMPARE(arbiter.events.count(), backAndFrontPicking ? 2 : 1);
         change = arbiter.events.first().staticCast<Qt3DCore::QPropertyUpdatedChange>();
         QCOMPARE(change->propertyName(), "moved");
 
@@ -1046,7 +1056,7 @@ private Q_SLOTS:
         // THEN -> Released
         QVERIFY(!earlyReturn);
         QVERIFY(!backendPicker->isPressed());
-        QCOMPARE(arbiter.events.count(), 2);
+        QCOMPARE(arbiter.events.count(), backAndFrontPicking ? 3 : 2);
         change = arbiter.events.first().staticCast<Qt3DCore::QPropertyUpdatedChange>();
         QCOMPARE(change->propertyName(), "pressed");
         change = arbiter.events.last().staticCast<Qt3DCore::QPropertyUpdatedChange>();