Run lipi HWR task in parallel with the recognition timer

This change improves the user experience by reducing the time
spent waiting the HWR task to complete.

Previously the recognition was started after the recognition
timer expired. This caused the total overhead of the recognition
to be summed on top of the recognition timer, reducing the user
experience in low performance devices. Now the idle time (during
the recognition timer) is used for the benefit of HWR task, so that
in optimal case, the recognition result is available immediately
after the recognition timer expires. If the HWR task takes longer
to process than the recognition timer, the results will be provided
as soon as the HWR task is done.

Ongoing recognition task is cancelled if the user continues drawing
while the recognition timer is running. If the recognition task is
already completed, the current result is ignored and the result
from the next recognition task is used instead.

Change-Id: I9ba797223d8a9b8daf423e500fcf9d5250caaa5a
Reviewed-by: Mitch Curtis <mitch.curtis@theqtcompany.com>

5 files changed, 160 insertions, 53 deletions

diff --git a/src/virtualkeyboard/lipiinputmethod.cpp b/src/virtualkeyboard/lipiinputmethod.cpp
index 9b92cb58..c8b9eda5 100644
--- a/src/virtualkeyboard/lipiinputmethod.cpp
+++ b/src/virtualkeyboard/lipiinputmethod.cpp
@@ -141,10 +141,10 @@ public:
 
         setContext(patternRecognitionMode, traceCaptureDeviceInfo, traceScreenInfo);
 
-        if (!recognitionTask) {
-            recognitionTask = recognizer.newRecognition(*deviceInfo, *screenContext, subsetOfClasses, 0.0f, 4);
-            if (!recognitionTask)
-                return 0;
+        if (recognitionTask) {
+            recognizer.cancelRecognitionTask(recognitionTask);
+            recognitionTask.reset();
+            delayedResult.clear();
         }
 
         DeclarativeTrace *trace = new DeclarativeTrace();
@@ -164,10 +164,11 @@ public:
             addPointsToTraceGroup(trace);
         }
         if (!traceList.isEmpty() && countActiveTraces() == 0)
-            resetRecognizeTimer();
+            restartRecognition();
     }
 
-    int countActiveTraces() const {
+    int countActiveTraces() const
+    {
         int count = 0;
         foreach (DeclarativeTrace *trace, traceList) {
             if (!trace->isFinal())
@@ -180,12 +181,11 @@ public:
     {
         qDeleteAll(traceList);
         traceList.clear();
+        traceGroup.emptyAllTraces();
     }
 
     void addPointsToTraceGroup(DeclarativeTrace *trace)
     {
-        if (!recognitionTask)
-            return;
         vector<LTKChannel> channels;
         channels.push_back(LTKChannel("X", DT_INT, true));
         channels.push_back(LTKChannel("Y", DT_INT, true));
@@ -209,24 +209,36 @@ public:
             }
             ltktrace.addPoint(point);
         }
-        recognitionTask->traceGroup.addTrace(ltktrace);
+        traceGroup.addTrace(ltktrace);
     }
 
-    void recognize()
+    void finishRecognition()
     {
 #ifdef QT_VIRTUALKEYBOARD_LIPI_RECORD_TRACE_INPUT
         dumpTraces();
 #endif
         stopRecognizeTimer();
         clearTraces();
+        if (recognitionTask && !delayedResult.isEmpty() && recognitionTask->resultId() == delayedResult["resultId"].toInt())
+            processResult(delayedResult);
+        delayedResult.clear();
+        recognitionTask.reset();
+    }
 
+    void restartRecognition()
+    {
+        recognitionTask = recognizer.newRecognition(*deviceInfo, *screenContext, subsetOfClasses, 0.0f, 4);
         if (recognitionTask) {
             Q_Q(LipiInputMethod);
 
+            recognitionTask->traceGroup = traceGroup;
+
             QSharedPointer<LipiRecognitionResultsTask> resultsTask = recognizer.startRecognition(recognitionTask);
             q->connect(resultsTask.data(), SIGNAL(resultsAvailable(const QVariantList &)), SLOT(resultsAvailable(const QVariantList &)));
 
-            recognitionTask.reset();
+            resetRecognizeTimer();
+        } else {
+            stopRecognizeTimer();
         }
     }
 
@@ -234,6 +246,7 @@ public:
     {
         stopRecognizeTimer();
         clearTraces();
+        delayedResult.clear();
         bool result = !recognitionTask.isNull();
         recognitionTask.reset();
         return recognizer.cancelRecognition() || result;
@@ -255,6 +268,50 @@ public:
         }
     }
 
+    void resultsAvailable(const QVariantList &resultList)
+    {
+        if (!resultList.isEmpty()) {
+            const QVariantMap result = resultList.at(0).toMap();
+            if (recognitionTask && recognitionTask->resultId() == result["resultId"].toInt())
+                delayedResult = result;
+            else
+                processResult(result);
+        }
+    }
+
+    void processResult(const QVariantMap &result)
+    {
+        const QChar ch = result["unicode"].toChar();
+        const QChar chUpper = ch.toUpper();
+#ifdef QT_VIRTUALKEYBOARD_LIPI_RECORD_TRACE_INPUT
+        // In recording mode, the text case must match with the current text case
+        if (!ch.isLetter() || (ch.isUpper() == (textCase == DeclarativeInputEngine::Upper))) {
+            QStringList homeLocations = QStandardPaths::standardLocations(QStandardPaths::HomeLocation);
+            if (!homeLocations.isEmpty()) {
+                uint confidence = qRound(result["confidence"].toDouble() * 100);
+                QString filePath = QStringLiteral("%1/%2").arg(homeLocations.at(0)).arg("VIRTUAL_KEYBOARD_TRACES");
+                QDir fileDir(filePath);
+                if (!fileDir.exists())
+                    fileDir.mkpath(filePath);
+                if (fileDir.exists()) {
+                    QString fileName;
+                    int fileIndex = 0;
+                    do {
+                        fileName = fileDir.absoluteFilePath(QStringLiteral("%1_%2_%3.txt").arg((uint)ch.unicode()).arg(confidence, 3, 10, QLatin1Char('0')).arg(fileIndex++));
+                    } while (QFileInfo(fileName).exists());
+                    saveTraces(fileName);
+                }
+            }
+        } else {
+            return;
+        }
+#endif
+        Q_Q(LipiInputMethod);
+        q->inputContext()->inputEngine()->virtualKeyClick((Qt::Key)chUpper.unicode(),
+                    textCase == DeclarativeInputEngine::Lower ? QString(ch.toLower()) : QString(chUpper),
+                    Qt::NoModifier);
+    }
+
 #ifdef QT_VIRTUALKEYBOARD_LIPI_RECORD_TRACE_INPUT
     QStringList recordedData;
 
@@ -316,10 +373,12 @@ public:
     QScopedPointer<LTKCaptureDevice> deviceInfo;
     QScopedPointer<LTKScreenContext> screenContext;
     QSharedPointer<LipiRecognitionTask> recognitionTask;
+    LTKTraceGroup traceGroup;
     QList<DeclarativeTrace *> traceList;
     int recognizeTimer;
     DeclarativeInputEngine::TextCase textCase;
     vector<int> subsetOfClasses;
+    QVariantMap delayedResult;
 };
 
 LipiInputMethod::LipiInputMethod(QObject *parent) :
@@ -441,7 +500,7 @@ void LipiInputMethod::timerEvent(QTimerEvent *timerEvent)
 {
     Q_D(LipiInputMethod);
     if (timerEvent->timerId() == d->recognizeTimer) {
-        d->recognize();
+        d->finishRecognition();
     }
 }
 
@@ -456,36 +515,6 @@ void LipiInputMethod::resultsAvailable(const QVariantList &resultList)
         }
     }
 #endif
-    if (!resultList.isEmpty()) {
-        Q_D(LipiInputMethod);
-        const QVariantMap result = resultList.at(0).toMap();
-        const QChar ch = result["unicode"].toChar();
-        const QChar chUpper = ch.toUpper();
-#ifdef QT_VIRTUALKEYBOARD_LIPI_RECORD_TRACE_INPUT
-        // In recording mode, the text case must match with the current text case
-        if (!ch.isLetter() || (ch.isUpper() == (d->textCase == DeclarativeInputEngine::Upper))) {
-            QStringList homeLocations = QStandardPaths::standardLocations(QStandardPaths::HomeLocation);
-            if (!homeLocations.isEmpty()) {
-                uint confidence = qRound(result["confidence"].toDouble() * 100);
-                QString filePath = QStringLiteral("%1/%2").arg(homeLocations.at(0)).arg("VIRTUAL_KEYBOARD_TRACES");
-                QDir fileDir(filePath);
-                if (!fileDir.exists())
-                    fileDir.mkpath(filePath);
-                if (fileDir.exists()) {
-                    QString fileName;
-                    int fileIndex = 0;
-                    do {
-                        fileName = fileDir.absoluteFilePath(QStringLiteral("%1_%2_%3.txt").arg((uint)ch.unicode()).arg(confidence, 3, 10, QLatin1Char('0')).arg(fileIndex++));
-                    } while (QFileInfo(fileName).exists());
-                    d->saveTraces(fileName);
-                }
-            }
-        } else {
-            return;
-        }
-#endif
-        inputContext()->inputEngine()->virtualKeyClick((Qt::Key)chUpper.unicode(),
-                    d->textCase == DeclarativeInputEngine::Lower ? QString(ch.toLower()) : QString(chUpper),
-                    Qt::NoModifier);
-    }
+    Q_D(LipiInputMethod);
+    d->resultsAvailable(resultList);
 }
diff --git a/src/virtualkeyboard/lipisharedrecognizer.cpp b/src/virtualkeyboard/lipisharedrecognizer.cpp
index 0c8b125f..feede348 100644
--- a/src/virtualkeyboard/lipisharedrecognizer.cpp
+++ b/src/virtualkeyboard/lipisharedrecognizer.cpp
@@ -46,6 +46,7 @@ LipiWorker *LipiSharedRecognizer::s_lipiWorker = 0;
 QMap<int, QChar> LipiSharedRecognizer::s_unicodeMap;
 QString LipiSharedRecognizer::s_activeModel;
 stringStringMap LipiSharedRecognizer::s_lipiEngineConfigEntries;
+int LipiSharedRecognizer::s_recognitionCount = 0;
 
 LipiSharedRecognizer::LipiSharedRecognizer()
 {
@@ -117,7 +118,10 @@ QSharedPointer<LipiRecognitionTask> LipiSharedRecognizer::newRecognition(const L
                                                                      screenContext,
                                                                      inSubsetOfClasses,
                                                                      confThreshold,
-                                                                     numChoices));
+                                                                     numChoices,
+                                                                     s_recognitionCount));
+
+    ++s_recognitionCount;
 
     return task;
 }
@@ -128,7 +132,8 @@ QSharedPointer<LipiRecognitionResultsTask> LipiSharedRecognizer::startRecognitio
         return QSharedPointer<LipiRecognitionResultsTask>();
 
     QSharedPointer<LipiRecognitionResultsTask> resultsTask(new LipiRecognitionResultsTask(recognitionTask->resultVector,
-                                                                                          s_unicodeMap));
+                                                                                          s_unicodeMap,
+                                                                                          recognitionTask->resultId()));
 
     s_lipiWorker->addTask(recognitionTask);
     s_lipiWorker->addTask(resultsTask);
@@ -144,6 +149,14 @@ bool LipiSharedRecognizer::cancelRecognition()
     return s_lipiWorker->removeAllTasks() > 0;
 }
 
+bool LipiSharedRecognizer::cancelRecognitionTask(QSharedPointer<LipiRecognitionTask> &recognitionTask)
+{
+    if (!s_lipiEngine || !s_shapeRecognizer || !s_lipiWorker || !recognitionTask)
+        return false;
+
+    return recognitionTask->cancelRecognition() || s_lipiWorker->removeTask(recognitionTask) > 0;
+}
+
 int LipiSharedRecognizer::loadLipiInterface()
 {
     VIRTUALKEYBOARD_DEBUG() << "LipiSharedRecognizer::loadLipiInterface():" << s_lipiEngineRefCount;
diff --git a/src/virtualkeyboard/lipisharedrecognizer.h b/src/virtualkeyboard/lipisharedrecognizer.h
index a53ac2d8..2ce7a69f 100644
--- a/src/virtualkeyboard/lipisharedrecognizer.h
+++ b/src/virtualkeyboard/lipisharedrecognizer.h
@@ -45,6 +45,7 @@ public:
                                                        int numChoices);
     QSharedPointer<LipiRecognitionResultsTask> startRecognition(QSharedPointer<LipiRecognitionTask> &recognitionTask);
     bool cancelRecognition();
+    bool cancelRecognitionTask(QSharedPointer<LipiRecognitionTask> &recognitionTask);
 
 private:
     static int loadLipiInterface();
@@ -69,6 +70,7 @@ private:
     static QMap<int, QChar> s_unicodeMap;
     static QString s_activeModel;
     static stringStringMap s_lipiEngineConfigEntries;
+    static int s_recognitionCount;
 };
 
 #endif // LIPISHAREDRECOGNIZER_H
diff --git a/src/virtualkeyboard/lipiworker.cpp b/src/virtualkeyboard/lipiworker.cpp
index 16fafc3e..6b26a8f0 100644
--- a/src/virtualkeyboard/lipiworker.cpp
+++ b/src/virtualkeyboard/lipiworker.cpp
@@ -43,14 +43,18 @@ LipiRecognitionTask::LipiRecognitionTask(const LTKCaptureDevice& deviceInfo,
                                          const LTKScreenContext& screenContext,
                                          const vector<int>& inSubsetOfClasses,
                                          float confThreshold,
-                                         int numChoices) :
+                                         int numChoices,
+                                         int resultId) :
     LipiTask(),
     deviceInfo(deviceInfo),
     screenContext(screenContext),
     inSubsetOfClasses(inSubsetOfClasses),
     confThreshold(confThreshold),
     numChoices(numChoices),
-    resultVector(new vector<LTKShapeRecoResult>())
+    resultVector(new vector<LTKShapeRecoResult>()),
+    _resultId(resultId),
+    stateRunning(false),
+    stateCancelled(false)
 {
 }
 
@@ -61,6 +65,12 @@ void LipiRecognitionTask::run()
     if (!shapeRecognizer || !resultVector)
         return;
 
+    {
+        QMutexLocker stateGuard(&stateLock);
+        stateRunning = true;
+    }
+
+    resultVector->clear();
     resultVector->reserve(numChoices);
 
     shapeRecognizer->setDeviceContext(deviceInfo);
@@ -72,16 +82,44 @@ void LipiRecognitionTask::run()
     shapeRecognizer->recognize(traceGroup, screenContext,
                                inSubsetOfClasses, confThreshold,
                                numChoices, *resultVector);
+
+#ifdef QT_VIRTUALKEYBOARD_DEBUG
+    int perfElapsed = perf.elapsed();
+#endif
+
+    {
+        QMutexLocker stateGuard(&stateLock);
+        stateRunning = false;
+        if (stateCancelled)
+            resultVector->clear();
 #ifdef QT_VIRTUALKEYBOARD_DEBUG
-    VIRTUALKEYBOARD_DEBUG() << "LipiRecognitionTask::run(): time:" << perf.elapsed() << "ms";
+        VIRTUALKEYBOARD_DEBUG() << "LipiRecognitionTask::run(): time:" << perfElapsed << "ms" << (stateCancelled ? "(cancelled)" : "");
 #endif
+    }
+}
+
+bool LipiRecognitionTask::cancelRecognition()
+{
+    QMutexLocker stateGuard(&stateLock);
+    stateCancelled = true;
+    bool result = (stateRunning && shapeRecognizer);
+    if (result)
+        shapeRecognizer->requestCancelRecognition();
+    return result;
+}
+
+int LipiRecognitionTask::resultId() const
+{
+    return _resultId;
 }
 
 LipiRecognitionResultsTask::LipiRecognitionResultsTask(QSharedPointer<vector<LTKShapeRecoResult> > resultVector,
-                                                       const QMap<int, QChar> &unicodeMap) :
+                                                       const QMap<int, QChar> &unicodeMap,
+                                                       int resultId) :
     LipiTask(),
     resultVector(resultVector),
-    unicodeMap(unicodeMap)
+    unicodeMap(unicodeMap),
+    _resultId(resultId)
 {
 }
 
@@ -95,12 +133,16 @@ void LipiRecognitionResultsTask::run()
          i != resultVector->end(); i++) {
         QVariantMap result;
         int shapeId = i->getShapeId();
+        result["resultId"] = _resultId;
         result["shapeId"] = shapeId;
         result["unicode"] = unicodeMap.value(shapeId);
         result["confidence"] = i->getConfidence();
         resultList.append(result);
     }
 
+    if (resultList.isEmpty())
+        return;
+
     emit resultsAvailable(resultList);
 }
 
@@ -131,6 +173,17 @@ void LipiWorker::addTask(QSharedPointer<LipiTask> task)
     }
 }
 
+int LipiWorker::removeTask(QSharedPointer<LipiTask> task)
+{
+    int count = 0;
+    if (task) {
+        QMutexLocker guard(&taskLock);
+        count = taskList.removeAll(task);
+        taskSema.acquire(qMin(count, taskSema.available()));
+    }
+    return count;
+}
+
 int LipiWorker::removeAllTasks()
 {
     QMutexLocker guard(&taskLock);
diff --git a/src/virtualkeyboard/lipiworker.h b/src/virtualkeyboard/lipiworker.h
index 01eac2e0..a3e7a50a 100644
--- a/src/virtualkeyboard/lipiworker.h
+++ b/src/virtualkeyboard/lipiworker.h
@@ -65,9 +65,12 @@ public:
                                  const LTKScreenContext& screenContext,
                                  const vector<int>& inSubsetOfClasses,
                                  float confThreshold,
-                                 int numChoices);
+                                 int numChoices,
+                                 int resultId);
 
     void run();
+    bool cancelRecognition();
+    int resultId() const;
 
     LTKTraceGroup traceGroup;
 
@@ -80,6 +83,10 @@ private:
     const float confThreshold;
     const int numChoices;
     QSharedPointer<vector<LTKShapeRecoResult> > resultVector;
+    const int _resultId;
+    QMutex stateLock;
+    bool stateRunning;
+    bool stateCancelled;
 };
 
 class LipiRecognitionResultsTask : public LipiTask
@@ -87,7 +94,8 @@ class LipiRecognitionResultsTask : public LipiTask
     Q_OBJECT
 public:
     explicit LipiRecognitionResultsTask(QSharedPointer<vector<LTKShapeRecoResult> > resultVector,
-                                        const QMap<int, QChar> &unicodeMap);
+                                        const QMap<int, QChar> &unicodeMap,
+                                        int resultId);
 
     void run();
 
@@ -97,6 +105,7 @@ signals:
 private:
     QSharedPointer<vector<LTKShapeRecoResult> > resultVector;
     const QMap<int, QChar> &unicodeMap;
+    const int _resultId;
 };
 
 class LipiWorker : public QThread
@@ -107,6 +116,7 @@ public:
     ~LipiWorker();
 
     void addTask(QSharedPointer<LipiTask> task);
+    int removeTask(QSharedPointer<LipiTask> task);
     int removeAllTasks();
 
 protected: