Add Cerence HWR and Cerence XT9 extension plugins

Cerence SDK enables two extension plugins for the Qt Virtual Keyboard.

- Cerence HWR: Handwriting extension know previously as T9 Write
- Cerence XT9: Advanced keyboard extension, supporting various languages

This commit removes the previous T9 Write extension and reintroduces
it as the Cerence HWR extension. This commit also adds the Cerence XT9
as a new feature. The Cerence HWR can also utilize the XT9 for some
additional features.

Here is a list of changes made to the handwriting extension:

- Move the T9 Write extension to the general "cerence" directory,
  which allows data and code to be shared between T9 Write and XT9.
- Update unpack.py to match the latest Cerence SDK. Drop support
  for legacy delivery rules, as they interfere with the latest
  SDK.
- Alphabetic API was renamed from decuma* to decumaUcr at version
  number 21
- Add extra parameter to BeginArcAddition and set
  bMinimizePreProcessing to 1 to avoid defining the parameter.
- Do not pass the support lines to engine as they are not
  supported/needed. For example, the latest CJK SDK returns error
  when the support lines are defined.
- Remove reference to decumaFunctionalSupport.h, which is not part
  of the official SDK. As a consequence, disable support for
  recognition interrupt, which is not critical feature anyway.
- Fix several compiler warnings.
- Enable UCR mode for new languages.
- Filter out duplicate candidates (after case formatting)
- Use handwriting recognition timeout setting
- Check and recover from an init failure when UCR not supported
- Use common dictionary management with XT9
- Add user defined search path for HWR db
  QT_VIRTUALKEYBOARD_T9WRITE_DB_PATH
- Use generic dictionary API
- Add user dictionary (DLM)
- Limit too many simultaneous input
- Add support for the latest SDK (removed support from previous
  versions of the SDK)
- Add support for background recognition supported by the engine.
  This improves latencies and removes delays in certain situations.
- Clear old traces from screen after specified delay.
  Previously, old traces were cleared from the screen based on the
  information from the engine to identify strokes of the recognized
  characters. Unfortunately, the engine does not provide this
  information at the same detail in UCR mode.
  This change introduces a timer-based approach, where traces are
  removed after a specified time, calculated from the recognition
  result. This time can be adjusted in the virtual keyboard settings.
- Add auto correction for alphabetic languages
- Add predictions from custom dictionary (XT9 Nav)
- Fix language id mapping between Qt6 QLocale::Language and Cerence
  HWR.

Change-Id: Iff4daea67cbb8adee1caf2e9513198482da48a38
Reviewed-by: Mitch Curtis <mitch.curtis@qt.io>
Reviewed-by: Jarkko Koivikko <jarkko.koivikko@code-q.fi>

3 files changed, 50 insertions, 19 deletions

diff --git a/tests/auto/inputpanel/data/inputpanel/inputpanel.qml b/tests/auto/inputpanel/data/inputpanel/inputpanel.qml
index 73b08413..807e32ab 100644
--- a/tests/auto/inputpanel/data/inputpanel/inputpanel.qml
+++ b/tests/auto/inputpanel/data/inputpanel/inputpanel.qml
@@ -689,6 +689,9 @@ InputPanel {
             origIndex = inputPanel.wordCandidateView.currentIndex
         }
         if (origIndex !== -1) {
+            while (inputPanel.wordCandidateView.currentIndex > 0) {
+                inputPanel.wordCandidateView.decrementCurrentIndex()
+            }
             while (true) {
                 if (inputPanel.wordCandidateView.model.dataAt(inputPanel.wordCandidateView.currentIndex) === suggestion) {
                     suggestionFound = true
diff --git a/tests/auto/inputpanel/data/inputpanel/utils.js b/tests/auto/inputpanel/data/inputpanel/utils.js
index de5117b1..376da88a 100644
--- a/tests/auto/inputpanel/data/inputpanel/utils.js
+++ b/tests/auto/inputpanel/data/inputpanel/utils.js
@@ -66,3 +66,13 @@ function findChild(parent, param, matchCb) {
     }
     return obj
 }
+
+function toUnicodeHex(str) {
+    var result = ''
+    for (var i = 0; i < str.length; i++) {
+        if (result.length > 0)
+            result += ", "
+        result += "U+" + ("000" + str.charCodeAt(i).toString(16)).slice(-4)
+    }
+    return result
+}
diff --git a/tests/auto/inputpanel/data/tst_inputpanel.qml b/tests/auto/inputpanel/data/tst_inputpanel.qml
index b92b2712..03ca7cbf 100644
--- a/tests/auto/inputpanel/data/tst_inputpanel.qml
+++ b/tests/auto/inputpanel/data/tst_inputpanel.qml
@@ -30,6 +30,7 @@
 import QtTest
 import QtQuick
 import QtQuick.Window
+import "inputpanel/utils.js" as Utils
 
 Rectangle {
     id: container
@@ -867,7 +868,7 @@ Rectangle {
         function test_spellCorrectionAutomaticSpaceInsertion_data() {
             return [
                 { inputSequence: ['h','e','l','l','o',Qt.Key_Select,'w','o','r','l','d'], outputText: "Hello world" },
-                { inputSequence: ['h','e','l','l','o','\'','s',Qt.Key_Select,'w','o','r','l','d'], outputText: "Hello's world" },
+                { inputSequence: ['k','e','l','l','y','\'','s',Qt.Key_Select,'w','o','r','l','d'], outputText: "Kelly's world" },
                 { inputSequence: ['h','e','l','l','o','s','\'',Qt.Key_Select,'w','o','r','l','d'], outputText: "Hellos' world" },
                 { inputSequence: ['h','e','l','l','o','-','w','o','r','l','d'], outputText: "Hello-world" },
                 { inputSequence: ['h','e','l','l','o',Qt.Key_Select,'.','w','o','r','l','d'], outputText: "Hello. World" },
@@ -940,7 +941,7 @@ Rectangle {
                 verify(inputPanel.selectionListSelectCurrentItem())
             }
 
-            compare(textInput.text, data.outputText)
+            compare(Utils.toUnicodeHex(textInput.text), Utils.toUnicodeHex(data.outputText))
         }
 
         function test_cangjieInputMethod_data() {
@@ -979,6 +980,9 @@ Rectangle {
         function test_cangjieInputMethod(data) {
             prepareTest(data, true)
 
+            if (!inputPanel.inputMethod.hasOwnProperty("simplified"))
+                skip("Input method does not support simplified mode")
+
             if (data.hasOwnProperty("initSimplified")) {
                 if (inputPanel.inputMethod.simplified !== data.initSimplified)
                     verify(inputPanel.virtualKeyClick(Qt.Key_Mode_switch))
@@ -1120,7 +1124,7 @@ Rectangle {
                 { initLocale: "ko_KR", inputSequence: "\u3131\u314F\u3139\u314D", outputText: "\uAC0E" },
                 { initLocale: "ko_KR", inputSequence: "\u3131\u314F\u3139\u314E", outputText: "\uAC0F" },
                 { initLocale: "ko_KR", inputSequence: "\u3131\u314F\u3142\u3145", outputText: "\uAC12" },
-                { initLocale: "ko_KR", inputSequence: "\u3131\u314F\u3145\u3145", outputText: "\uAC14" },
+                //{ initLocale: "ko_KR", inputSequence: "\u3131\u314F\u3145\u3145", outputText: "\uAC14" },   // Actually not standard
                 // Test using the final Jamo of the first syllable as an initial
                 // Jamo of the following syllable
                 { initLocale: "ko_KR", inputSequence: "\u3131\u314F\u3131\u314F", outputText: "\uAC00\uAC00" },
@@ -1137,24 +1141,32 @@ Rectangle {
 
             compare(Qt.inputMethod.locale.name, Qt.locale(data.initLocale).name)
 
+            function textInputContents() {
+                return textInput.text.substring(0, textInput.cursorPosition) +
+                        textInput.preeditText +
+                        textInput.text.substring(textInput.cursorPosition)
+            }
+
             // Add Jamos one by one
             var intermediateResult = []
             for (var inputIndex in data.inputSequence) {
                 verify(inputPanel.virtualKeyClick(data.inputSequence[inputIndex]))
-                intermediateResult.push(textInput.text)
+                intermediateResult.push(textInputContents())
             }
 
-            compare(textInput.text, data.outputText)
+            compare(Utils.toUnicodeHex(textInputContents()), Utils.toUnicodeHex(data.outputText))
 
             // Remove Jamos one by one.
             // The number of removed characters must match to the number of Jamos entered.
             for (inputIndex = data.inputSequence.length - 1; inputIndex >= 0; inputIndex--) {
-                compare(textInput.text, intermediateResult.pop())
+                waitForRendering(inputPanel)
+                compare(Utils.toUnicodeHex(textInputContents()), Utils.toUnicodeHex(intermediateResult.pop()))
                 inputPanel.virtualKeyClick(Qt.Key_Backspace)
             }
 
             waitForRendering(inputPanel)
-            compare(textInput.text, data.initText !== undefined ? data.initText : "")
+            compare(Utils.toUnicodeHex(textInputContents()),
+                    Utils.toUnicodeHex(data.initText !== undefined ? data.initText : ""))
         }
 
         function test_japaneseInputModes_data() {
@@ -1241,6 +1253,8 @@ Rectangle {
             // The Japanese keyboard uses the BaseKey.noModifier flag for the arrow keys.
             // Without this flag the arrow key + shift would extend the text selection.
             prepareTest({ initLocale: "ja_JP", initInputMethodHints: Qt.ImhLatinOnly })
+            if (!inputPanel.findVirtualKey(Qt.Key_Left) || inputPanel.virtualKeyClick(Qt.Key_Right))
+                skip("Custom layout detected")
             verify(inputPanel.virtualKeyClick("a"))
             verify(inputPanel.virtualKeyClick(Qt.Key_Left))
             compare(textInput.cursorPosition, 0)
@@ -2159,21 +2173,21 @@ Rectangle {
                 { inputSequence: ['a','s','d'], initShift: false, expectedSuggestion: "asdf", suggestionIsFromUserDictionary: true },
                 { inputSequence: ['a','s','d'], initShift: true, expectedSuggestion: "Asdf", suggestionIsFromUserDictionary: true },
                 //
-                { inputSequence: ['s','d','f','a'], initShift: true },
-                { inputSequence: ['s','d','f'], initShift: true, expectedSuggestion: "Sdfa", suggestionIsFromUserDictionary: true },
+                { inputSequence: ['S','d','f','a'], initShift: true },
+                { inputSequence: ['S','d','f'], initShift: true, expectedSuggestion: "Sdfa", suggestionIsFromUserDictionary: true },
                 { inputSequence: ['s','d','f'], initShift: false, expectedSuggestion: "sdfa", suggestionIsFromUserDictionary: true, removeSuggestion: true },
                 //
-                { inputSequence: ['d','f','a','s'], initCapsLock: true },
-                { inputSequence: ['d','f','a'], initCapsLock: true, expectedSuggestion: "DFAS", suggestionIsFromUserDictionary: true },
+                { inputSequence: ['D','F','A','S'], initCapsLock: true },
+                { inputSequence: ['D','F','A'], initCapsLock: true, expectedSuggestion: "DFAS", suggestionIsFromUserDictionary: true },
                 { inputSequence: ['d','f','a'], initShift: false, unexpectedSuggestion: "dfas", suggestionIsFromUserDictionary: true },
                 //
                 { inputSequence: ['f','a','s','d'], initShift: false, initInputMethodHints: Qt.ImhSensitiveData },
                 { inputSequence: ['f','a','s'], initShift: false, unexpectedSuggestion: "fasd" },
-                { inputSequence: ['f','a','s'], initShift: true, unexpectedSuggestion: "Fasd"},
+                { inputSequence: ['F','a','s'], initShift: true, unexpectedSuggestion: "Fasd"},
                 //
-                { initLocale: "en_GB", inputSequence: "windo", expectedSuggestion: "Window", suggestionIsFromUserDictionary: false, removeSuggestion: true },
-                { initLocale: "en_GB", inputSequence: "window", },
-                { initLocale: "en_GB", inputSequence: "windo", expectedSuggestion: "Window", suggestionIsFromUserDictionary: false },
+                { initLocale: "en_GB", inputSequence: "windo", initShift: false, expectedSuggestion: "window", suggestionIsFromUserDictionary: false, removeSuggestion: true },
+                { initLocale: "en_GB", inputSequence: "window",initShift: false,  },
+                { initLocale: "en_GB", inputSequence: "windo", initShift: false, expectedSuggestion: "window", suggestionIsFromUserDictionary: false },
             ]
         }
 
@@ -2188,11 +2202,14 @@ Rectangle {
             if (data.hasOwnProperty("initCapsLock"))
                 inputPanel.setCapsLockActive(data.initCapsLock)
 
-            for (var inputIndex in data.inputSequence)
+            var exactWord = ""
+            for (var inputIndex in data.inputSequence) {
                 inputPanel.virtualKeyClick(data.inputSequence[inputIndex])
+                exactWord += data.inputSequence[inputIndex]
+            }
 
             if (data.hasOwnProperty("expectedSuggestion")) {
-                tryVerify(function() {return inputPanel.selectionListSearchSuggestion(data.expectedSuggestion)}, 1000, "The expected spell correction suggestion \"%1\" was not found".arg(data.expectedSuggestion))
+                tryVerify(function() {return inputPanel.selectionListSearchSuggestion(data.expectedSuggestion)}, 10000, "The expected spell correction suggestion \"%1\" was not found".arg(data.expectedSuggestion))
                 verify(inputPanel.selectionListCurrentIndex() > 0)
                 if (data.hasOwnProperty("suggestionIsFromUserDictionary"))
                     compare(inputPanel.selectionListSuggestionIsFromUserDictionary(), data.suggestionIsFromUserDictionary)
@@ -2201,15 +2218,16 @@ Rectangle {
                     inputPanel.wordCandidateListChangedSpy.clear()
                     verify(inputPanel.selectItemFromWordCandidateContextMenu(0))
                     inputPanel.wordCandidateListChangedSpy.wait()
-                    tryVerify(function() {return !inputPanel.selectionListSearchSuggestion(data.expectedSuggestion)}, 1000, "An unexpected spell correction suggestion \"%1\" was found".arg(data.unexpectedSuggestion))
+                    tryVerify(function() {return !inputPanel.selectionListSearchSuggestion(data.expectedSuggestion)}, 10000, "An unexpected spell correction suggestion \"%1\" was found".arg(data.unexpectedSuggestion))
                 } else {
                     inputPanel.selectionListSelectCurrentItem()
                 }
             } else if (data.hasOwnProperty("unexpectedSuggestion")) {
                 var oldIndex = inputPanel.selectionListCurrentIndex()
-                tryVerify(function() {return !inputPanel.selectionListSearchSuggestion(data.unexpectedSuggestion)}, 1000, "An unexpected spell correction suggestion \"%1\" was found".arg(data.unexpectedSuggestion))
+                tryVerify(function() {return !inputPanel.selectionListSearchSuggestion(data.unexpectedSuggestion)}, 10000, "An unexpected spell correction suggestion \"%1\" was found".arg(data.unexpectedSuggestion))
                 compare(inputPanel.selectionListCurrentIndex(), oldIndex)
             } else {
+                tryVerify(function() {return inputPanel.selectionListSearchSuggestion(exactWord)}, 1000, "The exact word \"%1\" was not found".arg(exactWord))
                 inputPanel.selectionListSelectCurrentItem()
             }