QTouchEvent: add uniqueId and rotation; TUIO: support fiducial tokens

TUIO supports tracking tagged physical objects on touchscreens by
various means (QR codes, RFIDs etc.)  It can detect both position
and rotation.  Likewise, it may be possible for some touchscreens or
drivers to detect orientation of the fingers.  So, just as QTabletEvent
has rotation, each touchpoint needs to include the rotation value.

When using tokens, each object has a permanent unique ID, whereas
QTouchEvent::TouchPoint::id() is a transient ID which usually auto-
increments each time a finger is pressed to the device.  So we need to
make that available too, to identify each token.  Different platforms
may use different kinds of IDs (int, UUID, QR code etc.); however for
TUIO 1.x, the unique IDs are just 32-bit integers.  QPointerUniqueId
is added, storing only a qint64 for now (like QTabletEvent::uniqueId())
but able to be expanded as necessary later on.

Task-number: QTBUG-51844
Change-Id: I04182042f47fa2954728079139a4664a31184b54
Reviewed-by: Robin Burchell <robin.burchell@viroteck.net>

4 files changed, 382 insertions, 30 deletions

diff --git a/src/plugins/generic/tuiotouch/qtuiohandler.cpp b/src/plugins/generic/tuiotouch/qtuiohandler.cpp
index 6026e06b55..86decd312b 100644
--- a/src/plugins/generic/tuiotouch/qtuiohandler.cpp
+++ b/src/plugins/generic/tuiotouch/qtuiohandler.cpp
@@ -43,10 +43,12 @@
 #include <QWindow>
 #include <QGuiApplication>
 #include <QTouchDevice>
+#include <qmath.h>
 
 #include <qpa/qwindowsysteminterface.h>
 
 #include "qtuiocursor_p.h"
+#include "qtuiotoken_p.h"
 #include "qtuiohandler_p.h"
 #include "qoscbundle_p.h"
 
@@ -55,6 +57,12 @@ QT_BEGIN_NAMESPACE
 Q_LOGGING_CATEGORY(lcTuioSource, "qt.qpa.tuio.source")
 Q_LOGGING_CATEGORY(lcTuioSet, "qt.qpa.tuio.set")
 
+// With TUIO the first application takes exclusive ownership of the "device"
+// we cannot attach more than one application to the same port anyway.
+// Forcing delivery makes it easy to use simulators in the same machine
+// and forget about headaches about unfocused TUIO windows.
+static bool forceDelivery = qEnvironmentVariableIsSet("QT_TUIOTOUCH_DELIVER_WITHOUT_FOCUS");
+
 QTuioHandler::QTuioHandler(const QString &specification)
     : m_device(new QTouchDevice) // not leaked, QTouchDevice cleans up registered devices itself
 {
@@ -157,29 +165,49 @@ void QTuioHandler::processPackets()
             messages.push_back(msg);
         }
 
-        foreach (const QOscMessage &message, messages) {
-            if (message.addressPattern() != "/tuio/2Dcur") {
-                qWarning() << "Ignoring unknown address pattern " << message.addressPattern();
-                continue;
-            }
-
-            QList<QVariant> arguments = message.arguments();
-            if (arguments.count() == 0) {
-                qWarning("Ignoring TUIO message with no arguments");
-                continue;
-            }
-
-            QByteArray messageType = arguments.at(0).toByteArray();
-            if (messageType == "source") {
-                process2DCurSource(message);
-            } else if (messageType == "alive") {
-                process2DCurAlive(message);
-            } else if (messageType == "set") {
-                process2DCurSet(message);
-            } else if (messageType == "fseq") {
-                process2DCurFseq(message);
+        for (const QOscMessage &message : messages) {
+            if (message.addressPattern() == "/tuio/2Dcur") {
+                QList<QVariant> arguments = message.arguments();
+                if (arguments.count() == 0) {
+                    qWarning("Ignoring TUIO message with no arguments");
+                    continue;
+                }
+
+                QByteArray messageType = arguments.at(0).toByteArray();
+                if (messageType == "source") {
+                    process2DCurSource(message);
+                } else if (messageType == "alive") {
+                    process2DCurAlive(message);
+                } else if (messageType == "set") {
+                    process2DCurSet(message);
+                } else if (messageType == "fseq") {
+                    process2DCurFseq(message);
+                } else {
+                    qWarning() << "Ignoring unknown TUIO message type: " << messageType;
+                    continue;
+                }
+            } else if (message.addressPattern() == "/tuio/2Dobj") {
+                QList<QVariant> arguments = message.arguments();
+                if (arguments.count() == 0) {
+                    qWarning("Ignoring TUIO message with no arguments");
+                    continue;
+                }
+
+                QByteArray messageType = arguments.at(0).toByteArray();
+                if (messageType == "source") {
+                    process2DObjSource(message);
+                } else if (messageType == "alive") {
+                    process2DObjAlive(message);
+                } else if (messageType == "set") {
+                    process2DObjSet(message);
+                } else if (messageType == "fseq") {
+                    process2DObjFseq(message);
+                } else {
+                    qWarning() << "Ignoring unknown TUIO message type: " << messageType;
+                    continue;
+                }
             } else {
-                qWarning() << "Ignoring unknown TUIO message type: " << messageType;
+                qWarning() << "Ignoring unknown address pattern " << message.addressPattern();
                 continue;
             }
         }
@@ -328,11 +356,6 @@ void QTuioHandler::process2DCurFseq(const QOscMessage &message)
     Q_UNUSED(message); // TODO: do we need to do anything with the frame id?
 
     QWindow *win = QGuiApplication::focusWindow();
-    // With TUIO the first application takes exclusive ownership of the "device"
-    // we cannot attach more than one application to the same port anyway.
-    // Forcing delivery makes it easy to use simulators in the same machine
-    // and forget about headaches about unfocused TUIO windows.
-    static bool forceDelivery = qEnvironmentVariableIsSet("QT_TUIOTOUCH_DELIVER_WITHOUT_FOCUS");
     if (!win && QGuiApplication::topLevelWindows().length() > 0 && forceDelivery)
           win = QGuiApplication::topLevelWindows().at(0);
 
@@ -342,12 +365,12 @@ void QTuioHandler::process2DCurFseq(const QOscMessage &message)
     QList<QWindowSystemInterface::TouchPoint> tpl;
     tpl.reserve(m_activeCursors.size() + m_deadCursors.size());
 
-    foreach (const QTuioCursor &tc, m_activeCursors) {
+    for (const QTuioCursor &tc : m_activeCursors) {
         QWindowSystemInterface::TouchPoint tp = cursorToTouchPoint(tc, win);
         tpl.append(tp);
     }
 
-    foreach (const QTuioCursor &tc, m_deadCursors) {
+    for (const QTuioCursor &tc : m_deadCursors) {
         QWindowSystemInterface::TouchPoint tp = cursorToTouchPoint(tc, win);
         tp.state = Qt::TouchPointReleased;
         tpl.append(tp);
@@ -357,5 +380,181 @@ void QTuioHandler::process2DCurFseq(const QOscMessage &message)
     m_deadCursors.clear();
 }
 
+void QTuioHandler::process2DObjSource(const QOscMessage &message)
+{
+    QList<QVariant> arguments = message.arguments();
+    if (arguments.count() != 2) {
+        qWarning() << "Ignoring malformed TUIO source message: " << arguments.count();
+        return;
+    }
+
+    if (QMetaType::Type(arguments.at(1).type()) != QMetaType::QByteArray) {
+        qWarning("Ignoring malformed TUIO source message (bad argument type)");
+        return;
+    }
+
+    qCDebug(lcTuioSource) << "Got TUIO source message from: " << arguments.at(1).toByteArray();
+}
+
+void QTuioHandler::process2DObjAlive(const QOscMessage &message)
+{
+    QList<QVariant> arguments = message.arguments();
+
+    // delta the notified tokens that are active, against the ones we already
+    // know of.
+    //
+    // TBD: right now we're assuming one 2DObj alive message corresponds to a
+    // new data source from the input. is this correct, or do we need to store
+    // changes and only process the deltas on fseq?
+    QMap<int, QTuioToken> oldActiveTokens = m_activeTokens;
+    QMap<int, QTuioToken> newActiveTokens;
+
+    for (int i = 1; i < arguments.count(); ++i) {
+        if (QMetaType::Type(arguments.at(i).type()) != QMetaType::Int) {
+            qWarning() << "Ignoring malformed TUIO alive message (bad argument on position" << i << arguments << ')';
+            return;
+        }
+
+        int sessionId = arguments.at(i).toInt();
+        if (!oldActiveTokens.contains(sessionId)) {
+            // newly active
+            QTuioToken token(sessionId);
+            token.setState(Qt::TouchPointPressed);
+            newActiveTokens.insert(sessionId, token);
+        } else {
+            // we already know about it, remove it so it isn't marked as released
+            QTuioToken token = oldActiveTokens.value(sessionId);
+            token.setState(Qt::TouchPointStationary); // position change in SET will update if needed
+            newActiveTokens.insert(sessionId, token);
+            oldActiveTokens.remove(sessionId);
+        }
+    }
+
+    // anything left is dead now
+    QMap<int, QTuioToken>::ConstIterator it = oldActiveTokens.constBegin();
+
+    // deadTokens should be cleared from the last FSEQ now
+    m_deadTokens.reserve(oldActiveTokens.size());
+
+    // TODO: there could be an issue of resource exhaustion here if FSEQ isn't
+    // sent in a timely fashion. we should probably track message counts and
+    // force-flush if we get too many built up.
+    while (it != oldActiveTokens.constEnd()) {
+        m_deadTokens.append(it.value());
+        ++it;
+    }
+
+    m_activeTokens = newActiveTokens;
+}
+
+void QTuioHandler::process2DObjSet(const QOscMessage &message)
+{
+    QList<QVariant> arguments = message.arguments();
+    if (arguments.count() < 7) {
+        qWarning() << "Ignoring malformed TUIO set message with too few arguments: " << arguments.count();
+        return;
+    }
+
+    if (QMetaType::Type(arguments.at(1).type()) != QMetaType::Int ||
+            QMetaType::Type(arguments.at(2).type()) != QMetaType::Int ||
+            QMetaType::Type(arguments.at(3).type()) != QMetaType::Float ||
+            QMetaType::Type(arguments.at(4).type()) != QMetaType::Float ||
+            QMetaType::Type(arguments.at(5).type()) != QMetaType::Float ||
+            QMetaType::Type(arguments.at(6).type()) != QMetaType::Float ||
+            QMetaType::Type(arguments.at(7).type()) != QMetaType::Float ||
+            QMetaType::Type(arguments.at(8).type()) != QMetaType::Float ||
+            QMetaType::Type(arguments.at(9).type()) != QMetaType::Float ||
+            QMetaType::Type(arguments.at(10).type()) != QMetaType::Float) {
+        qWarning() << "Ignoring malformed TUIO set message with bad types: " << arguments;
+        return;
+    }
+
+    int id = arguments.at(1).toInt();
+    int classId = arguments.at(2).toInt();
+    float x = arguments.at(3).toFloat();
+    float y = arguments.at(4).toFloat();
+    float angle = arguments.at(5).toFloat();
+    float vx = arguments.at(6).toFloat();
+    float vy = arguments.at(7).toFloat();
+    float angularVelocity = arguments.at(8).toFloat();
+    float acceleration = arguments.at(9).toFloat();
+    float angularAcceleration = arguments.at(10).toFloat();
+
+    QMap<int, QTuioToken>::Iterator it = m_activeTokens.find(id);
+    if (it == m_activeTokens.end()) {
+        qWarning() << "Ignoring malformed TUIO set for nonexistent token " << classId;
+        return;
+    }
+
+    qCDebug(lcTuioSet) << "Processing SET for token " << classId << id << " @ " << x << y << "∡" << angle <<
+                          "vel" << vx << vy << angularVelocity << "acc" << acceleration << angularAcceleration;
+    QTuioToken &tok = *it;
+    tok.setClassId(classId);
+    tok.setX(x);
+    tok.setY(y);
+    tok.setVX(vx);
+    tok.setVY(vy);
+    tok.setAcceleration(acceleration);
+    tok.setAngle(angle);
+    tok.setAngularVelocity(angularAcceleration);
+    tok.setAngularAcceleration(angularAcceleration);
+}
+
+QWindowSystemInterface::TouchPoint QTuioHandler::tokenToTouchPoint(const QTuioToken &tc, QWindow *win)
+{
+    QWindowSystemInterface::TouchPoint tp;
+    tp.id = tc.id();
+    tp.uniqueId = tc.classId(); // TODO TUIO 2.0: populate a QVariant, and register the mapping from int to arbitrary UID data
+    tp.flags = QTouchEvent::TouchPoint::Token;
+    tp.pressure = 1.0f;
+
+    tp.normalPosition = QPointF(tc.x(), tc.y());
+
+    if (!m_transform.isIdentity())
+        tp.normalPosition = m_transform.map(tp.normalPosition);
+
+    tp.state = tc.state();
+    tp.area = QRectF(0, 0, 1, 1);
+
+    // We map the token position to the size of the window.
+    QPointF relPos = QPointF(win->size().width() * tp.normalPosition.x(), win->size().height() * tp.normalPosition.y());
+    QPointF delta = relPos - relPos.toPoint();
+    tp.area.moveCenter(win->mapToGlobal(relPos.toPoint()) + delta);
+    tp.velocity = QVector2D(win->size().width() * tc.vx(), win->size().height() * tc.vy());
+    tp.rotation = tc.angle() * 180.0 / M_PI; // convert radians to degrees
+    return tp;
+}
+
+
+void QTuioHandler::process2DObjFseq(const QOscMessage &message)
+{
+    Q_UNUSED(message); // TODO: do we need to do anything with the frame id?
+
+    QWindow *win = QGuiApplication::focusWindow();
+    if (!win && QGuiApplication::topLevelWindows().length() > 0 && forceDelivery)
+          win = QGuiApplication::topLevelWindows().at(0);
+
+    if (!win)
+        return;
+
+    QList<QWindowSystemInterface::TouchPoint> tpl;
+    tpl.reserve(m_activeTokens.size() + m_deadTokens.size());
+
+    for (const QTuioToken & t : m_activeTokens) {
+        QWindowSystemInterface::TouchPoint tp = tokenToTouchPoint(t, win);
+        tpl.append(tp);
+    }
+
+    for (const QTuioToken & t : m_deadTokens) {
+        QWindowSystemInterface::TouchPoint tp = tokenToTouchPoint(t, win);
+        tp.state = Qt::TouchPointReleased;
+        tp.velocity = QVector2D();
+        tpl.append(tp);
+    }
+    QWindowSystemInterface::handleTouchEvent(win, m_device, tpl);
+
+    m_deadTokens.clear();
+}
+
 QT_END_NAMESPACE
 
diff --git a/src/plugins/generic/tuiotouch/qtuiohandler_p.h b/src/plugins/generic/tuiotouch/qtuiohandler_p.h
index 3034872aae..2e444f2a0d 100644
--- a/src/plugins/generic/tuiotouch/qtuiohandler_p.h
+++ b/src/plugins/generic/tuiotouch/qtuiohandler_p.h
@@ -54,6 +54,7 @@ QT_BEGIN_NAMESPACE
 class QTouchDevice;
 class QOscMessage;
 class QTuioCursor;
+class QTuioToken;
 
 class QTuioHandler : public QObject
 {
@@ -69,14 +70,21 @@ private slots:
     void process2DCurAlive(const QOscMessage &message);
     void process2DCurSet(const QOscMessage &message);
     void process2DCurFseq(const QOscMessage &message);
+    void process2DObjSource(const QOscMessage &message);
+    void process2DObjAlive(const QOscMessage &message);
+    void process2DObjSet(const QOscMessage &message);
+    void process2DObjFseq(const QOscMessage &message);
 
 private:
     QWindowSystemInterface::TouchPoint cursorToTouchPoint(const QTuioCursor &tc, QWindow *win);
+    QWindowSystemInterface::TouchPoint tokenToTouchPoint(const QTuioToken &tc, QWindow *win);
 
     QTouchDevice *m_device;
     QUdpSocket m_socket;
     QMap<int, QTuioCursor> m_activeCursors;
     QVector<QTuioCursor> m_deadCursors;
+    QMap<int, QTuioToken> m_activeTokens;
+    QVector<QTuioToken> m_deadTokens;
     QTransform m_transform;
 };
 
diff --git a/src/plugins/generic/tuiotouch/qtuiotoken_p.h b/src/plugins/generic/tuiotouch/qtuiotoken_p.h
new file mode 100644
index 0000000000..5084aeed11
--- /dev/null
+++ b/src/plugins/generic/tuiotouch/qtuiotoken_p.h
@@ -0,0 +1,144 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QTUIOOBJECT_P_H
+#define QTUIOOBJECT_P_H
+
+#include <Qt>
+#include <qmath.h>
+
+QT_BEGIN_NAMESPACE
+
+/*!
+    \internal
+
+    A fiducial object, or token, represented by 2Dobj in TUIO 1.x and tok in TUIO 2:
+    a physical object whose position and rotation can be uniquely tracked
+    on the touchscreen surface.
+*/
+class QTuioToken
+{
+public:
+    QTuioToken(int id = -1)
+        : m_id(id)
+        , m_classId(-1)
+        , m_x(0)
+        , m_y(0)
+        , m_vx(0)
+        , m_vy(0)
+        , m_acceleration(0)
+        , m_angle(0)
+        , m_angularVelocity(0)
+        , m_angularAcceleration(0)
+        , m_state(Qt::TouchPointPressed)
+    {
+    }
+
+    int id() const { return m_id; }
+
+    int classId() const { return m_classId; }
+    void setClassId(int classId) { m_classId = classId; }
+
+    void setX(float x)
+    {
+        if (state() == Qt::TouchPointStationary &&
+            !qFuzzyCompare(m_x + 2.0, x + 2.0)) { // +2 because 1 is a valid value, and qFuzzyCompare can't cope with 0.0
+            setState(Qt::TouchPointMoved);
+        }
+        m_x = x;
+    }
+    float x() const { return m_x; }
+
+    void setY(float y)
+    {
+        if (state() == Qt::TouchPointStationary &&
+            !qFuzzyCompare(m_y + 2.0, y + 2.0)) { // +2 because 1 is a valid value, and qFuzzyCompare can't cope with 0.0
+            setState(Qt::TouchPointMoved);
+        }
+        m_y = y;
+    }
+    float y() const { return m_y; }
+
+    void setVX(float vx) { m_vx = vx; }
+    float vx() const { return m_vx; }
+
+    void setVY(float vy) { m_vy = vy; }
+    float vy() const { return m_vy; }
+
+    void setAcceleration(float acceleration) { m_acceleration = acceleration; }
+    float acceleration() const { return m_acceleration; }
+
+    float angle() const { return m_angle; }
+    void setAngle(float angle)
+    {
+        if (angle > M_PI)
+            angle = angle - M_PI * 2.0; // zero is pointing upwards, and is the default; but we want to have negative angles when rotating left
+        if (state() == Qt::TouchPointStationary &&
+            !qFuzzyCompare(m_angle + 2.0, angle + 2.0)) { // +2 because 1 is a valid value, and qFuzzyCompare can't cope with 0.0
+            setState(Qt::TouchPointMoved);
+        }
+        m_angle = angle;
+    }
+
+    float angularVelocity() const { return m_angularVelocity; }
+    void setAngularVelocity(float angularVelocity) { m_angularVelocity = angularVelocity; }
+
+    float angularAcceleration() const { return m_angularAcceleration; }
+    void setAngularAcceleration(float angularAcceleration) { m_angularAcceleration = angularAcceleration; }
+
+    void setState(const Qt::TouchPointState &state) { m_state = state; }
+    Qt::TouchPointState state() const { return m_state; }
+
+private:
+    int m_id;       // sessionID, temporary object ID
+    int m_classId;  // classID (e.g. marker ID)
+    float m_x;
+    float m_y;
+    float m_vx;
+    float m_vy;
+    float m_acceleration;
+    float m_angle;
+    float m_angularVelocity;
+    float m_angularAcceleration;
+    Qt::TouchPointState m_state;
+};
+
+QT_END_NAMESPACE
+
+#endif // QTUIOOBJECT_P_H
diff --git a/src/plugins/generic/tuiotouch/tuiotouch.pro b/src/plugins/generic/tuiotouch/tuiotouch.pro
index ae2ccde058..ad6a1c6876 100644
--- a/src/plugins/generic/tuiotouch/tuiotouch.pro
+++ b/src/plugins/generic/tuiotouch/tuiotouch.pro
@@ -15,7 +15,8 @@ HEADERS += \
     qoscbundle_p.h \
     qoscmessage_p.h \
     qtuiohandler_p.h \
-    qtuiocursor_p.h
+    qtuiocursor_p.h \
+    qtuiotoken_p.h
 
 OTHER_FILES += \
     tuiotouch.json