Make qsTr() work with Unicode (non-Latin-1) strings

Converting the source string/context/comment to Latin-1 is very
broken, since JS strings are UTF-16.
The strings should be converted to UTF-8, which should also be
the default encoding for qsTranslate().
Effectively, this bug meant that only Latin-1 characters could
be used in source strings; the translations themselves could
have non-Latin-1 characters. But there was data loss in the
case where you passed a source string for which no translation
was found (since the Latin-1-ized string would be returned).

Task-number: QTBUG-14989
Reviewed-by: Jedrzej Nowacki

8 files changed, 170 insertions, 0 deletions

diff --git a/tests/auto/qscriptengine/idtranslatable-unicode.js b/tests/auto/qscriptengine/idtranslatable-unicode.js
new file mode 100644
index 0000000000..e17d6172bc
--- /dev/null
+++ b/tests/auto/qscriptengine/idtranslatable-unicode.js
@@ -0,0 +1,5 @@
+qsTrId('\u01F8\u01D2\u0199\u01D0\u01E1');
+
+QT_TRID_NOOP("\u0191\u01CE\u0211\u0229\u019C\u018E\u019A\u01D0");
+
+qsTrId("\u0181\u01A1\u0213\u018F\u018C", 10);
diff --git a/tests/auto/qscriptengine/qscriptengine.qrc b/tests/auto/qscriptengine/qscriptengine.qrc
index fa55a5b5b1..d05d115966 100644
--- a/tests/auto/qscriptengine/qscriptengine.qrc
+++ b/tests/auto/qscriptengine/qscriptengine.qrc
@@ -2,5 +2,7 @@
 <qresource>
    <file>translations/translatable_la.qm</file>
    <file>translations/idtranslatable_la.qm</file>
+   <file>translations/translatable-unicode.qm</file>
+   <file>translations/idtranslatable-unicode.qm</file>
 </qresource>
 </RCC>
diff --git a/tests/auto/qscriptengine/translatable-unicode.js b/tests/auto/qscriptengine/translatable-unicode.js
new file mode 100644
index 0000000000..afe2aff21c
--- /dev/null
+++ b/tests/auto/qscriptengine/translatable-unicode.js
@@ -0,0 +1,9 @@
+qsTr("H\u2082O");
+qsTranslate("\u010C\u0101\u011F\u0115", "CO\u2082");
+
+var unicode_strings = [
+    QT_TR_NOOP("\u0391\u0392\u0393"),
+    QT_TRANSLATE_NOOP("\u010C\u0101\u011F\u0115", "\u0414\u0415\u0416")
+];
+
+qsTr("H\u2082O", "not the same H\u2082O");
diff --git a/tests/auto/qscriptengine/translations/idtranslatable-unicode.qm b/tests/auto/qscriptengine/translations/idtranslatable-unicode.qm
new file mode 100644
index 0000000000..8c5fb91b1d
--- /dev/null
+++ b/tests/auto/qscriptengine/translations/idtranslatable-unicode.qm
diff --git a/tests/auto/qscriptengine/translations/idtranslatable-unicode.ts b/tests/auto/qscriptengine/translations/idtranslatable-unicode.ts
new file mode 100644
index 0000000000..74ebf43c47
--- /dev/null
+++ b/tests/auto/qscriptengine/translations/idtranslatable-unicode.ts
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE TS>
+<TS version="2.0" language="nb_NO">
+<defaultcodec>UTF-8</defaultcodec>
+<context>
+    <name></name>
+    <message id="Ǹǒƙǐǡ">
+        <location filename="idtranslatable-unicode.js" line="1"/>
+        <source></source>
+        <translation>Ƨưƈȼȝȿș</translation>
+    </message>
+    <message id="ƑǎȑȩƜƎƚǐ">
+        <location filename="idtranslatable-unicode.js" line="3"/>
+        <source></source>
+        <translation>Ǡȡȋȅȕ</translation>
+    </message>
+    <message id="ƁơȓƏƌ" numerus="yes">
+        <location filename="idtranslatable-unicode.js" line="5"/>
+        <source></source>
+        <translation>
+            <numerusform>Ƒưǹ</numerusform>
+            <numerusform>%n ƒơǒ(ș)</numerusform>
+        </translation>
+    </message>
+</context>
+</TS>
diff --git a/tests/auto/qscriptengine/translations/translatable-unicode.qm b/tests/auto/qscriptengine/translations/translatable-unicode.qm
new file mode 100644
index 0000000000..aa75ce61df
--- /dev/null
+++ b/tests/auto/qscriptengine/translations/translatable-unicode.qm
diff --git a/tests/auto/qscriptengine/translations/translatable-unicode.ts b/tests/auto/qscriptengine/translations/translatable-unicode.ts
new file mode 100644
index 0000000000..1b8b4d2309
--- /dev/null
+++ b/tests/auto/qscriptengine/translations/translatable-unicode.ts
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE TS>
+<TS version="2.0">
+<defaultcodec>UTF-8</defaultcodec>
+<context>
+    <name>translatable-unicode</name>
+    <message>
+        <location filename="translatable-unicode.js" line="1"/>
+        <source>H₂O</source>
+        <translation>ͻͼͽ</translation>
+    </message>
+    <message>
+        <location filename="translatable-unicode.js" line="5"/>
+        <source>ΑΒΓ</source>
+        <translation>ӜҴѼ</translation>
+    </message>
+    <message>
+        <location filename="translatable-unicode.js" line="9"/>
+        <source>H₂O</source>
+        <comment>not the same H₂O</comment>
+        <translation>ԶՊՒ</translation>
+    </message>
+</context>
+<context>
+    <name>Čāğĕ</name>
+    <message>
+        <location filename="translatable-unicode.js" line="2"/>
+        <source>CO₂</source>
+        <translation>בךע</translation>
+    </message>
+    <message>
+        <location filename="translatable-unicode.js" line="6"/>
+        <source>ДЕЖ</source>
+        <translation>خسس</translation>
+    </message>
+</context>
+</TS>
diff --git a/tests/auto/qscriptengine/tst_qscriptengine.cpp b/tests/auto/qscriptengine/tst_qscriptengine.cpp
index 5f38c2255c..d529b8b0d2 100644
--- a/tests/auto/qscriptengine/tst_qscriptengine.cpp
+++ b/tests/auto/qscriptengine/tst_qscriptengine.cpp
@@ -186,6 +186,10 @@ private slots:
     void translationContext_data();
     void translationContext();
     void translateScriptIdBased();
+    void translateScriptUnicode_data();
+    void translateScriptUnicode();
+    void translateScriptUnicodeIdBased_data();
+    void translateScriptUnicodeIdBased();
     void functionScopes();
     void nativeFunctionScopes();
     void evaluateProgram();
@@ -4950,6 +4954,93 @@ void tst_QScriptEngine::translateScriptIdBased()
              QString::fromLatin1("qtn_foo_bar")); // Doesn't have plural
 }
 
+// How to add a new test row:
+// - Find a nice list of Unicode characters to choose from
+// - Write source string/context/comment in .js using Unicode escape sequences (\uABCD)
+// - Update corresponding .ts file (e.g. lupdate foo.js -ts foo.ts -codecfortr UTF-8)
+// - Enter translation in Linguist
+// - Update corresponding .qm file (e.g. lrelease foo.ts)
+// - Evaluate script that performs translation; make sure the correct result is returned
+//   (e.g. by setting the resulting string as the text of a QLabel and visually verifying
+//   that it looks the same as what you entered in Linguist :-) )
+// - Generate the expectedTranslation column data using toUtf8().toHex()
+void tst_QScriptEngine::translateScriptUnicode_data()
+{
+    QTest::addColumn<QString>("expression");
+    QTest::addColumn<QString>("fileName");
+    QTest::addColumn<QString>("expectedTranslation");
+
+    QString fileName = QString::fromLatin1("translatable-unicode.js");
+    QTest::newRow("qsTr('H\\u2082O')@translatable-unicode.js")
+            << QString::fromLatin1("qsTr('H\\u2082O')") << fileName << QString::fromUtf8("\xcd\xbb\xcd\xbc\xcd\xbd");
+    QTest::newRow("qsTranslate('\\u010C\\u0101\\u011F\\u0115', 'CO\\u2082')@translatable-unicode.js")
+            << QString::fromLatin1("qsTranslate('\\u010C\\u0101\\u011F\\u0115', 'CO\\u2082')") << fileName << QString::fromUtf8("\xd7\x91\xd7\x9a\xd7\xa2");
+    QTest::newRow("qsTr('\\u0391\\u0392\\u0393')@translatable-unicode.js")
+            << QString::fromLatin1("qsTr('\\u0391\\u0392\\u0393')") << fileName << QString::fromUtf8("\xd3\x9c\xd2\xb4\xd1\xbc");
+    QTest::newRow("qsTranslate('\\u010C\\u0101\\u011F\\u0115', '\\u0414\\u0415\\u0416')@translatable-unicode.js")
+            << QString::fromLatin1("qsTranslate('\\u010C\\u0101\\u011F\\u0115', '\\u0414\\u0415\\u0416')") << fileName << QString::fromUtf8("\xd8\xae\xd8\xb3\xd8\xb3");
+    QTest::newRow("qsTr('H\\u2082O', 'not the same H\\u2082O')@translatable-unicode.js")
+            << QString::fromLatin1("qsTr('H\\u2082O', 'not the same H\\u2082O')") << fileName << QString::fromUtf8("\xd4\xb6\xd5\x8a\xd5\x92");
+    QTest::newRow("qsTr('H\\u2082O')")
+            << QString::fromLatin1("qsTr('H\\u2082O')") << QString() << QString::fromUtf8("\x48\xe2\x82\x82\x4f");
+    QTest::newRow("qsTranslate('\\u010C\\u0101\\u011F\\u0115', 'CO\\u2082')")
+            << QString::fromLatin1("qsTranslate('\\u010C\\u0101\\u011F\\u0115', 'CO\\u2082')") << QString() << QString::fromUtf8("\xd7\x91\xd7\x9a\xd7\xa2");
+}
+
+void tst_QScriptEngine::translateScriptUnicode()
+{
+    QFETCH(QString, expression);
+    QFETCH(QString, fileName);
+    QFETCH(QString, expectedTranslation);
+
+    QScriptEngine engine;
+
+    QTranslator translator;
+    QVERIFY(translator.load(":/translations/translatable-unicode"));
+    QCoreApplication::instance()->installTranslator(&translator);
+    engine.installTranslatorFunctions();
+
+    QCOMPARE(engine.evaluate(expression, fileName).toString(), expectedTranslation);
+    QVERIFY(!engine.hasUncaughtException());
+
+    QCoreApplication::instance()->removeTranslator(&translator);
+}
+
+void tst_QScriptEngine::translateScriptUnicodeIdBased_data()
+{
+    QTest::addColumn<QString>("expression");
+    QTest::addColumn<QString>("expectedTranslation");
+
+    QTest::newRow("qsTrId('\\u01F8\\u01D2\\u0199\\u01D0\\u01E1'')")
+            << QString::fromLatin1("qsTrId('\\u01F8\\u01D2\\u0199\\u01D0\\u01E1')") << QString::fromUtf8("\xc6\xa7\xc6\xb0\xc6\x88\xc8\xbc\xc8\x9d\xc8\xbf\xc8\x99");
+    QTest::newRow("qsTrId('\\u0191\\u01CE\\u0211\\u0229\\u019C\\u018E\\u019A\\u01D0')")
+            << QString::fromLatin1("qsTrId('\\u0191\\u01CE\\u0211\\u0229\\u019C\\u018E\\u019A\\u01D0')") << QString::fromUtf8("\xc7\xa0\xc8\xa1\xc8\x8b\xc8\x85\xc8\x95");
+    QTest::newRow("qsTrId('\\u0181\\u01A1\\u0213\\u018F\\u018C', 10)")
+            << QString::fromLatin1("qsTrId('\\u0181\\u01A1\\u0213\\u018F\\u018C', 10)") << QString::fromUtf8("\x31\x30\x20\xc6\x92\xc6\xa1\xc7\x92\x28\xc8\x99\x29");
+    QTest::newRow("qsTrId('\\u0181\\u01A1\\u0213\\u018F\\u018C')")
+            << QString::fromLatin1("qsTrId('\\u0181\\u01A1\\u0213\\u018F\\u018C')") << QString::fromUtf8("\xc6\x91\xc6\xb0\xc7\xb9");
+    QTest::newRow("qsTrId('\\u01CD\\u0180\\u01A8\\u0190\\u019E\\u01AB')")
+            << QString::fromLatin1("qsTrId('\\u01CD\\u0180\\u01A8\\u0190\\u019E\\u01AB')") << QString::fromUtf8("\xc7\x8d\xc6\x80\xc6\xa8\xc6\x90\xc6\x9e\xc6\xab");
+}
+
+void tst_QScriptEngine::translateScriptUnicodeIdBased()
+{
+    QFETCH(QString, expression);
+    QFETCH(QString, expectedTranslation);
+
+    QScriptEngine engine;
+
+    QTranslator translator;
+    QVERIFY(translator.load(":/translations/idtranslatable-unicode"));
+    QCoreApplication::instance()->installTranslator(&translator);
+    engine.installTranslatorFunctions();
+
+    QCOMPARE(engine.evaluate(expression).toString(), expectedTranslation);
+    QVERIFY(!engine.hasUncaughtException());
+
+    QCoreApplication::instance()->removeTranslator(&translator);
+}
+
 void tst_QScriptEngine::functionScopes()
 {
     QScriptEngine eng;