/**************************************************************************** ** ** Copyright (C) 2016 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the Qt Virtual Keyboard module of the Qt Toolkit. ** ** $QT_BEGIN_LICENSE:GPL$ ** Commercial License Usage ** Licensees holding valid commercial Qt licenses may use this file in ** accordance with the commercial license agreement provided with the ** Software or, alternatively, in accordance with the terms contained in ** a written agreement between you and The Qt Company. For licensing terms ** and conditions see https://www.qt.io/terms-conditions. For further ** information use the contact form at https://www.qt.io/contact-us. ** ** GNU General Public License Usage ** Alternatively, this file may be used under the terms of the GNU ** General Public License version 3 or (at your option) any later version ** approved by the KDE Free Qt Foundation. The licenses are as published by ** the Free Software Foundation and appearing in the file LICENSE.GPL3 ** included in the packaging of this file. Please review the following ** information to ensure the GNU General Public License requirements will ** be met: https://www.gnu.org/licenses/gpl-3.0.html. ** ** $QT_END_LICENSE$ ** ****************************************************************************/ #include "lipiinputmethod.h" #include "lipisharedrecognizer.h" #include "inputengine.h" #include "inputcontext.h" #include "shifthandler.h" #include "virtualkeyboarddebug.h" #include "trace.h" #include "handwritinggesturerecognizer.h" #ifdef HAVE_HUNSPELL #include "hunspellinputmethod_p.h" #endif #include "LTKCaptureDevice.h" #include "LTKScreenContext.h" #include "LTKTraceGroup.h" #include "LTKChannel.h" #include "LTKTraceFormat.h" #include "LTKTrace.h" #include "LTKShapeRecoResult.h" #include #ifdef QT_VIRTUALKEYBOARD_RECORD_TRACE_INPUT #include "unipentrace.h" #include #endif #ifdef HAVE_HUNSPELL #define LipiInputMethodPrivateBase HunspellInputMethodPrivate #else #define LipiInputMethodPrivateBase AbstractInputMethodPrivate #endif namespace QtVirtualKeyboard { class LipiInputMethodPrivate : public LipiInputMethodPrivateBase { Q_DECLARE_PUBLIC(LipiInputMethod) public: LipiInputMethodPrivate(LipiInputMethod *q_ptr) : #ifdef HAVE_HUNSPELL LipiInputMethodPrivateBase(static_cast(q_ptr)), #else LipiInputMethodPrivateBase(), #endif q_ptr(q_ptr), recognizeTimer(0), textCase(InputEngine::Lower) #ifdef QT_VIRTUALKEYBOARD_RECORD_TRACE_INPUT , unipenTrace(0) #endif { } ~LipiInputMethodPrivate() { cancelRecognition(); } QByteArray getContext(InputEngine::PatternRecognitionMode patternRecognitionMode, const QVariantMap &traceCaptureDeviceInfo, const QVariantMap &traceScreenInfo) const { QCryptographicHash hash(QCryptographicHash::Md5); hash.addData((const char *)&patternRecognitionMode, sizeof(patternRecognitionMode)); QByteArray mapData; QDataStream ds(&mapData, QIODevice::WriteOnly); ds << traceCaptureDeviceInfo; ds << traceScreenInfo; hash.addData(mapData); return hash.result(); } void setContext(InputEngine::PatternRecognitionMode patternRecognitionMode, const QVariantMap &traceCaptureDeviceInfo, const QVariantMap &traceScreenInfo) { QByteArray context = getContext(patternRecognitionMode, traceCaptureDeviceInfo, traceScreenInfo); if (context == currentContext) return; VIRTUALKEYBOARD_DEBUG() << "LipiInputMethodPrivate::setContext():" << QString(context.toHex()); clearTraces(); deviceInfo.reset(new LTKCaptureDevice()); deviceInfo->setSamplingRate(traceCaptureDeviceInfo.value("sampleRate", 60).toInt()); deviceInfo->setXDPI(traceCaptureDeviceInfo.value("dpi", 96).toInt()); deviceInfo->setYDPI(deviceInfo->getXDPI()); deviceInfo->setLatency(traceCaptureDeviceInfo.value("latency", 0.0).toFloat()); deviceInfo->setUniformSampling(traceCaptureDeviceInfo.value("uniform", false).toBool()); screenContext.reset(new LTKScreenContext()); QRectF boundingBox(traceScreenInfo.value("boundingBox").toRectF()); if (!boundingBox.isEmpty()) { screenContext->setBboxLeft(boundingBox.left()); screenContext->setBboxTop(boundingBox.top()); screenContext->setBboxRight(boundingBox.right()); screenContext->setBboxBottom(boundingBox.bottom()); } QVariantList horizontalRulers(traceScreenInfo.value("horizontalRulers", QVariantList()).toList()); if (!horizontalRulers.isEmpty()) { for (QVariantList::ConstIterator i = horizontalRulers.constBegin(); i != horizontalRulers.constEnd(); i++) { screenContext->addHLine(i->toFloat()); } } QVariantList verticalRulers(traceScreenInfo.value("verticalRulers", QVariantList()).toList()); if (!horizontalRulers.isEmpty()) { for (QVariantList::ConstIterator i = verticalRulers.constBegin(); i != verticalRulers.constEnd(); i++) { screenContext->addVLine(i->toFloat()); } } gestureRecognizer.setDpi(deviceInfo->getXDPI()); currentContext = context; } Trace *traceBegin(int traceId, InputEngine::PatternRecognitionMode patternRecognitionMode, const QVariantMap &traceCaptureDeviceInfo, const QVariantMap &traceScreenInfo) { Q_UNUSED(traceId) stopRecognizeTimer(); setContext(patternRecognitionMode, traceCaptureDeviceInfo, traceScreenInfo); if (recognitionTask) { recognizer.cancelRecognitionTask(recognitionTask); recognitionTask.reset(); delayedResult.clear(); } #ifdef QT_VIRTUALKEYBOARD_RECORD_TRACE_INPUT if (!unipenTrace) { Q_Q(LipiInputMethod); unipenTrace = new UnipenTrace(traceCaptureDeviceInfo, traceScreenInfo, q); } #endif Trace *trace = new Trace(); trace->setChannels(QStringList("t")); traceList.append(trace); return trace; } void traceEnd(Trace *trace) { if (trace->isCanceled()) { VIRTUALKEYBOARD_DEBUG() << "LipiInputMethodPrivate::traceEnd(): discarded" << trace; traceList.removeOne(trace); delete trace; } else { addPointsToTraceGroup(trace); } handleGesture(); if (!traceList.isEmpty() && countActiveTraces() == 0) restartRecognition(); } int countActiveTraces() const { int count = 0; for (Trace *trace : qAsConst(traceList)) { if (!trace->isFinal()) count++; } return count; } void handleGesture() { if (countActiveTraces() > 0) return; QVariantMap gesture = gestureRecognizer.recognize(traceList); if (gesture.isEmpty()) return; VIRTUALKEYBOARD_DEBUG() << "LipiInputMethodPrivate::handleGesture():" << gesture; if (gesture[QLatin1String("type")].toString() == QLatin1String("swipe")) { static const int SWIPE_MIN_LENGTH = 25; // mm static const int SWIPE_ANGLE_THRESHOLD = 15; // degrees +- qreal swipeLength = gesture[QLatin1String("length_mm")].toReal(); if (swipeLength >= SWIPE_MIN_LENGTH) { Q_Q(LipiInputMethod); InputContext *ic = q->inputContext(); if (!ic) return; qreal swipeAngle = gesture[QLatin1String("angle_degrees")].toReal(); int swipeTouchCount = gesture[QLatin1String("touch_count")].toInt(); // Swipe left if (swipeAngle <= 180 + SWIPE_ANGLE_THRESHOLD && swipeAngle >= 180 - SWIPE_ANGLE_THRESHOLD) { if (swipeTouchCount == 1) { // Single swipe: backspace #ifdef QT_VIRTUALKEYBOARD_RECORD_TRACE_INPUT dumpTraces(); saveTraces(Qt::Key_Backspace, 100); #endif cancelRecognition(); ic->inputEngine()->virtualKeyClick(Qt::Key_Backspace, QString(), Qt::NoModifier); } else if (swipeTouchCount == 2) { // Double swipe: reset word, or backspace cancelRecognition(); if (!ic->preeditText().isEmpty()) { q->reset(); ic->setPreeditText(QString()); } else { ic->inputEngine()->virtualKeyClick(Qt::Key_Backspace, QString(), Qt::NoModifier); } } return; } // Swipe right if (swipeAngle <= SWIPE_ANGLE_THRESHOLD || swipeAngle >= 360 - SWIPE_ANGLE_THRESHOLD) { if (swipeTouchCount == 1) { // Single swipe: space #ifdef QT_VIRTUALKEYBOARD_RECORD_TRACE_INPUT dumpTraces(); saveTraces(Qt::Key_Space, 100); #endif cancelRecognition(); ic->inputEngine()->virtualKeyClick(Qt::Key_Space, QString(" "), Qt::NoModifier); } else if (swipeTouchCount == 2) { // Double swipe: commit word, or insert space cancelRecognition(); #ifdef HAVE_HUNSPELL if (activeWordIndex != -1) { q->selectionListItemSelected(SelectionListModel::WordCandidateList, activeWordIndex); return; } #endif ic->inputEngine()->virtualKeyClick(Qt::Key_Space, QString(" "), Qt::NoModifier); } return; } // Swipe up if (swipeAngle <= 270 + SWIPE_ANGLE_THRESHOLD && swipeAngle >= 270 - SWIPE_ANGLE_THRESHOLD) { if (swipeTouchCount == 1) { // Single swipe: toggle input mode #ifdef QT_VIRTUALKEYBOARD_RECORD_TRACE_INPUT dumpTraces(); saveTraces(Qt::Key_Mode_switch, 100); #endif cancelRecognition(); if (!(ic->inputMethodHints() & (Qt::ImhDialableCharactersOnly | Qt::ImhFormattedNumbersOnly | Qt::ImhDigitsOnly))) { InputEngine::InputMode inputMode = ic->inputEngine()->inputMode(); inputMode = inputMode == InputEngine::Latin ? InputEngine::Numeric : InputEngine::Latin; ic->inputEngine()->setInputMode(inputMode); } } else if (swipeTouchCount == 2) { // Double swipe: toggle text case cancelRecognition(); ic->shiftHandler()->toggleShift(); } return; } } } } void clearTraces() { qDeleteAll(traceList); traceList.clear(); traceGroup.emptyAllTraces(); } void addPointsToTraceGroup(Trace *trace) { vector channels; channels.push_back(LTKChannel("X", DT_INT, true)); channels.push_back(LTKChannel("Y", DT_INT, true)); bool hasTime = trace->channels().contains("t"); if (hasTime) channels.push_back(LTKChannel("T", DT_FLOAT, true)); LTKTraceFormat traceFormat(channels); LTKTrace ltktrace(traceFormat); const QVariantList points = trace->points(); const QVariantList timeData = hasTime ? trace->channelData("t") : QVariantList(); QVariantList::ConstIterator t = timeData.constBegin(); for (const QVariant &p : points) { const QPointF pt(p.toPointF()); vector point; point.push_back(pt.x()); point.push_back(pt.y()); if (hasTime) { point.push_back(t->toFloat()); t++; } ltktrace.addPoint(point); } traceGroup.addTrace(ltktrace); } void finishRecognition() { #ifdef QT_VIRTUALKEYBOARD_RECORD_TRACE_INPUT dumpTraces(); #endif stopRecognizeTimer(); clearTraces(); if (recognitionTask && !delayedResult.isEmpty() && recognitionTask->resultId() == delayedResult["resultId"].toInt()) processResult(delayedResult); delayedResult.clear(); recognitionTask.reset(); } void restartRecognition() { recognitionTask = recognizer.newRecognition(*deviceInfo, *screenContext, subsetOfClasses, 0.0f, 4); if (recognitionTask) { Q_Q(LipiInputMethod); recognitionTask->traceGroup = traceGroup; QSharedPointer resultsTask = recognizer.startRecognition(recognitionTask); q->connect(resultsTask.data(), SIGNAL(resultsAvailable(const QVariantList &)), SLOT(resultsAvailable(const QVariantList &))); resetRecognizeTimer(); } else { stopRecognizeTimer(); } } bool cancelRecognition() { stopRecognizeTimer(); clearTraces(); delayedResult.clear(); bool result = !recognitionTask.isNull(); recognitionTask.reset(); return recognizer.cancelRecognition() || result; } void resetRecognizeTimer() { Q_Q(LipiInputMethod); stopRecognizeTimer(); recognizeTimer = q->startTimer(300); } void stopRecognizeTimer() { if (recognizeTimer) { Q_Q(LipiInputMethod); q->killTimer(recognizeTimer); recognizeTimer = 0; } } void resultsAvailable(const QVariantList &resultList) { if (!resultList.isEmpty()) { const QVariantMap result = resultList.at(0).toMap(); if (recognitionTask && recognitionTask->resultId() == result["resultId"].toInt()) delayedResult = result; else processResult(result); } } void processResult(const QVariantMap &result) { const QChar ch = result["unicode"].toChar(); const QChar chUpper = ch.toUpper(); #ifdef QT_VIRTUALKEYBOARD_RECORD_TRACE_INPUT // In recording mode, the text case must match with the current text case if (unipenTrace) { if (!ch.isLetter() || (ch.isUpper() == (textCase == InputEngine::Upper))) saveTraces(ch.unicode(), qRound(result["confidence"].toDouble() * 100)); delete unipenTrace; unipenTrace = 0; } #endif Q_Q(LipiInputMethod); q->inputContext()->inputEngine()->virtualKeyClick((Qt::Key)chUpper.unicode(), textCase == InputEngine::Lower ? QString(ch.toLower()) : QString(chUpper), Qt::NoModifier); } #ifdef QT_VIRTUALKEYBOARD_RECORD_TRACE_INPUT void dumpTraces() { if (unipenTrace) unipenTrace->record(traceList); } void saveTraces(uint unicode, uint confidence) { if (!unipenTrace) return; QStringList homeLocations = QStandardPaths::standardLocations(QStandardPaths::HomeLocation); if (!homeLocations.isEmpty()) { QString filePath = QStringLiteral("%1/%2").arg(homeLocations.at(0)).arg("VIRTUAL_KEYBOARD_TRACES"); unipenTrace->setDirectory(filePath); unipenTrace->save(unicode, confidence); } } #endif LipiInputMethod *q_ptr; LipiSharedRecognizer recognizer; QByteArray currentContext; QScopedPointer deviceInfo; QScopedPointer screenContext; QSharedPointer recognitionTask; LTKTraceGroup traceGroup; QList traceList; int recognizeTimer; InputEngine::TextCase textCase; vector subsetOfClasses; QVariantMap delayedResult; HandwritingGestureRecognizer gestureRecognizer; #ifdef QT_VIRTUALKEYBOARD_RECORD_TRACE_INPUT UnipenTrace *unipenTrace; #endif }; /*! \class QtVirtualKeyboard::LipiInputMethod \internal */ LipiInputMethod::LipiInputMethod(QObject *parent) : LipiInputMethodBase(*new LipiInputMethodPrivate(this), parent) { } LipiInputMethod::~LipiInputMethod() { } QList LipiInputMethod::inputModes(const QString &locale) { Q_UNUSED(locale) QList availableInputModes; const Qt::InputMethodHints inputMethodHints(inputContext()->inputMethodHints()); if (inputMethodHints.testFlag(Qt::ImhDialableCharactersOnly) || inputMethodHints.testFlag(Qt::ImhDigitsOnly)) { availableInputModes.append(InputEngine::Dialable); } else if (inputMethodHints.testFlag(Qt::ImhFormattedNumbersOnly)) { availableInputModes.append(InputEngine::Numeric); } else { availableInputModes.append(InputEngine::Latin); availableInputModes.append(InputEngine::Numeric); } return availableInputModes; } bool LipiInputMethod::setInputMode(const QString &locale, InputEngine::InputMode inputMode) { Q_D(LipiInputMethod); #ifdef HAVE_HUNSPELL HunspellInputMethod::setInputMode(locale, inputMode); #else Q_UNUSED(locale) #endif bool result = d->recognizer.setModel(QStringLiteral("SHAPEREC_ALPHANUM")); if (!result) return false; d->subsetOfClasses.clear(); switch (inputMode) { case InputEngine::Latin: d->recognizer.subsetOfClasses(QStringLiteral("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz?,.@"), d->subsetOfClasses); break; case InputEngine::Numeric: case InputEngine::Dialable: d->recognizer.subsetOfClasses(QStringLiteral("1234567890,.+"), d->subsetOfClasses); break; default: break; } return true; } bool LipiInputMethod::setTextCase(InputEngine::TextCase textCase) { Q_D(LipiInputMethod); d->textCase = textCase; #ifdef HAVE_HUNSPELL HunspellInputMethod::setTextCase(textCase); #endif return true; } bool LipiInputMethod::keyEvent(Qt::Key key, const QString &text, Qt::KeyboardModifiers modifiers) { #ifdef HAVE_HUNSPELL Q_D(LipiInputMethod); switch (key) { case Qt::Key_Enter: case Qt::Key_Return: d->cancelRecognition(); break; case Qt::Key_Backspace: if (d->cancelRecognition()) return true; break; default: break; } return HunspellInputMethod::keyEvent(key, text, modifiers); #else Q_UNUSED(key) Q_UNUSED(text) Q_UNUSED(modifiers) return false; #endif } void LipiInputMethod::reset() { LipiInputMethodBase::reset(); Q_D(LipiInputMethod); d->cancelRecognition(); } void LipiInputMethod::update() { LipiInputMethodBase::update(); } void LipiInputMethod::selectionListItemSelected(SelectionListModel::Type type, int index) { LipiInputMethodBase::selectionListItemSelected(type, index); Q_D(LipiInputMethod); d->cancelRecognition(); } QList LipiInputMethod::patternRecognitionModes() const { return QList() << InputEngine::HandwritingRecoginition; } Trace *LipiInputMethod::traceBegin(int traceId, InputEngine::PatternRecognitionMode patternRecognitionMode, const QVariantMap &traceCaptureDeviceInfo, const QVariantMap &traceScreenInfo) { Q_D(LipiInputMethod); return d->traceBegin(traceId, patternRecognitionMode, traceCaptureDeviceInfo, traceScreenInfo); } bool LipiInputMethod::traceEnd(Trace *trace) { Q_D(LipiInputMethod); d->traceEnd(trace); return true; } void LipiInputMethod::timerEvent(QTimerEvent *timerEvent) { Q_D(LipiInputMethod); if (timerEvent->timerId() == d->recognizeTimer) { d->finishRecognition(); } } void LipiInputMethod::resultsAvailable(const QVariantList &resultList) { #ifdef QT_VIRTUALKEYBOARD_DEBUG { VIRTUALKEYBOARD_DEBUG() << "LipiInputMethod::resultsAvailable():"; for (int i = 0; i < resultList.size(); i++) { QVariantMap result = resultList.at(i).toMap(); VIRTUALKEYBOARD_DEBUG() << QString("%1: %2 (%3)").arg(i + 1).arg(result["unicode"].toChar()).arg(result["confidence"].toFloat()).toUtf8().constData(); } } #endif Q_D(LipiInputMethod); d->resultsAvailable(resultList); } } // namespace QtVirtualKeyboard