Get rid of the QTextCodec dependency in linguist

Change-Id: I9f09578b816ff24a23e17b3255ac93c5eb7c321e
Reviewed-by: Liang Qi <liang.qi@qt.io>

11 files changed, 44 insertions, 66 deletions

diff --git a/src/linguist/linguist/phrase.cpp b/src/linguist/linguist/phrase.cpp
index eafca3aa6..505e02941 100644
--- a/src/linguist/linguist/phrase.cpp
+++ b/src/linguist/linguist/phrase.cpp
@@ -34,7 +34,6 @@
 #include <QFile>
 #include <QFileInfo>
 #include <QMessageBox>
-#include <QTextCodec>
 #include <QTextStream>
 #include <QXmlStreamReader>
 
@@ -269,7 +268,6 @@ bool PhraseBook::save(const QString &fileName)
     m_fileName = fileName;
 
     QTextStream t(&f);
-    t.setCodec( QTextCodec::codecForName("UTF-8") );
 
     t << "<!DOCTYPE QPH>\n<QPH";
     if (sourceLanguage() != QLocale::C)
diff --git a/src/linguist/lupdate/cpp.cpp b/src/linguist/lupdate/cpp.cpp
index 1214d00ae..b7823e622 100644
--- a/src/linguist/lupdate/cpp.cpp
+++ b/src/linguist/lupdate/cpp.cpp
@@ -31,7 +31,6 @@
 #include <translator.h>
 #include <QtCore/QBitArray>
 #include <QtCore/QStack>
-#include <QtCore/QTextCodec>
 #include <QtCore/QTextStream>
 #include <QtCore/QRegularExpression>
 
@@ -212,7 +211,7 @@ private:
     int yyParenLineNo;
 
     // the string to read from and current position in the string
-    QTextCodec *yySourceCodec;
+    QStringConverter::Encoding yySourceEncoding = QStringConverter::Utf8;
     QString yyInStr;
     const ushort *yyInPtr;
 
@@ -275,14 +274,14 @@ void CppParser::setInput(const QString &in)
 {
     yyInStr = in;
     yyFileName = QString();
-    yySourceCodec = 0;
+    yySourceEncoding = QStringConverter::Utf8;
 }
 
 void CppParser::setInput(QTextStream &ts, const QString &fileName)
 {
     yyInStr = ts.readAll();
     yyFileName = fileName;
-    yySourceCodec = ts.codec();
+    yySourceEncoding = ts.encoding();
 }
 
 /*
@@ -1305,7 +1304,7 @@ void CppParser::processInclude(const QString &file, ConversionData &cd, const QS
     }
 
     QTextStream ts(&f);
-    ts.setCodec(yySourceCodec);
+    ts.setEncoding(yySourceEncoding);
     ts.setAutoDetectUnicode(true);
 
     inclusions.insert(cleanFile);
@@ -1391,12 +1390,12 @@ bool CppParser::matchEncoding()
         if (yyTok == Tok_ColonColon)
             yyTok = getToken();
     }
-    if (yyWord == strUnicodeUTF8 || yyWord == strDefaultCodec || yyWord == strCodecForTr) {
+    if (yyWord == strUnicodeUTF8) {
         yyTok = getToken();
         return true;
     }
-    if (yyWord == strLatin1)
-        yyMsg() << qPrintable(LU::tr("Unsupported encoding Latin1\n"));
+    if (yyWord == strLatin1 || yyWord == strDefaultCodec || yyWord == strCodecForTr)
+        yyMsg() << qPrintable(LU::tr("Unsupported encoding Latin1/DefaultCodec/CodecForTr\n"));
     return false;
 }
 
@@ -2229,7 +2228,7 @@ const ParseResults *CppParser::recordResults(bool isHeader)
 
 void loadCPP(Translator &translator, const QStringList &filenames, ConversionData &cd)
 {
-    QTextCodec *codec = QTextCodec::codecForName(cd.m_sourceIsUtf16 ? "UTF-16" : "UTF-8");
+    QStringConverter::Encoding e = cd.m_sourceIsUtf16 ? QStringConverter::Utf16 : QStringConverter::Utf8;
 
     foreach (const QString &filename, filenames) {
         if (!CppFiles::getResults(filename).isEmpty() || CppFiles::isBlacklisted(filename))
@@ -2243,7 +2242,7 @@ void loadCPP(Translator &translator, const QStringList &filenames, ConversionDat
 
         CppParser parser;
         QTextStream ts(&file);
-        ts.setCodec(codec);
+        ts.setEncoding(e);
         ts.setAutoDetectUnicode(true);
         parser.setInput(ts, filename);
         Translator *tor = new Translator;
diff --git a/src/linguist/lupdate/java.cpp b/src/linguist/lupdate/java.cpp
index b5f2dc82a..14ba926a8 100644
--- a/src/linguist/lupdate/java.cpp
+++ b/src/linguist/lupdate/java.cpp
@@ -35,7 +35,6 @@
 #include <QtCore/QStack>
 #include <QtCore/QStack>
 #include <QtCore/QString>
-#include <QtCore/QTextCodec>
 #include <QtCore/QCoreApplication>
 
 #include <iostream>
@@ -597,7 +596,7 @@ bool loadJava(Translator &translator, const QString &filename, ConversionData &c
     yyParenLineNo = 1;
 
     QTextStream ts(&file);
-    ts.setCodec(QTextCodec::codecForName(cd.m_sourceIsUtf16 ? "UTF-16" : "UTF-8"));
+    ts.setEncoding(cd.m_sourceIsUtf16 ? QStringConverter::Utf16 : QStringConverter::Utf8);
     ts.setAutoDetectUnicode(true);
     yyInStr = ts.readAll();
     yyInPos = 0;
diff --git a/src/linguist/lupdate/qdeclarative.cpp b/src/linguist/lupdate/qdeclarative.cpp
index 7d995ae86..468a5258c 100644
--- a/src/linguist/lupdate/qdeclarative.cpp
+++ b/src/linguist/lupdate/qdeclarative.cpp
@@ -469,8 +469,6 @@ static bool load(Translator &translator, const QString &filename, ConversionData
         code = QTextStream(&file).readAll();
     } else {
         QTextStream ts(&file);
-        ts.setCodec("UTF-8");
-        ts.setAutoDetectUnicode(true);
         code = ts.readAll();
     }
 
diff --git a/src/linguist/shared/po.cpp b/src/linguist/shared/po.cpp
index 62c0402e6..dd6fcbbf2 100644
--- a/src/linguist/shared/po.cpp
+++ b/src/linguist/shared/po.cpp
@@ -33,7 +33,7 @@
 #include <QtCore/QHash>
 #include <QtCore/QRegularExpression>
 #include <QtCore/QString>
-#include <QtCore/QTextCodec>
+#include <QtCore/QStringConverter>
 #include <QtCore/QTextStream>
 
 #include <ctype.h>
@@ -397,7 +397,7 @@ static QByteArray QByteArrayList_join(const QList<QByteArray> &that, char sep)
 
 bool loadPO(Translator &translator, QIODevice &dev, ConversionData &cd)
 {
-    QTextCodec *codec = QTextCodec::codecForName("UTF-8");
+    QStringDecoder toUnicode(QStringConverter::Utf8, QStringDecoder::Flag::Stateless);
     bool error = false;
 
     // format of a .po file entry:
@@ -476,18 +476,18 @@ bool loadPO(Translator &translator, QIODevice &dev, ConversionData &cd)
                                     .arg(QString::fromLatin1(hdrValue)));
                                 error = true;
                                 // This will avoid a flood of conversion errors.
-                                codec = QTextCodec::codecForName("latin1");
+                                toUnicode = QStringConverter::Latin1;
                             } else {
                                 QByteArray cod = hdrValue.mid(20);
-                                QTextCodec *cdc = QTextCodec::codecForName(cod);
-                                if (!cdc) {
-                                    cd.appendError(QString::fromLatin1("Unsupported codec '%1'")
+                                auto enc = QStringConverter::encodingForName(cod);
+                                if (!enc) {
+                                    cd.appendError(QString::fromLatin1("Unsupported encoding '%1'")
                                             .arg(QString::fromLatin1(cod)));
                                     error = true;
                                     // This will avoid a flood of conversion errors.
-                                    codec = QTextCodec::codecForName("latin1");
+                                    toUnicode = QStringConverter::Latin1;
                                 } else {
-                                    codec = cdc;
+                                    toUnicode = *enc;
                                 }
                             }
                     } else if (hdrName == "Content-Transfer-Encoding") {
@@ -537,17 +537,17 @@ bool loadPO(Translator &translator, QIODevice &dev, ConversionData &cd)
                 for (QHash<QString, QByteArray>::ConstIterator it = extras.constBegin(),
                                                                end = extras.constEnd();
                      it != end; ++it)
-                    translator.setExtra(it.key(), codec->toUnicode(it.value()));
+                    translator.setExtra(it.key(), toUnicode(it.value()));
                 item = PoItem();
                 continue;
             }
             // build translator message
             TranslatorMessage msg;
-            msg.setContext(codec->toUnicode(item.context));
+            msg.setContext(toUnicode(item.context));
             if (!item.references.isEmpty()) {
                 QString xrefs;
                 foreach (const QString &ref,
-                         codec->toUnicode(item.references).split(
+                         QString(toUnicode(item.references)).split(
                                  QRegularExpression(QLatin1String("\\s")), Qt::SkipEmptyParts)) {
                     int pos = ref.indexOf(QLatin1Char(':'));
                     int lpos = ref.lastIndexOf(QLatin1Char(':'));
@@ -566,17 +566,17 @@ bool loadPO(Translator &translator, QIODevice &dev, ConversionData &cd)
                 if (!xrefs.isEmpty())
                     item.extra[QLatin1String("po-references")] = xrefs;
             }
-            msg.setId(codec->toUnicode(item.id));
-            msg.setSourceText(codec->toUnicode(item.msgId));
-            msg.setOldSourceText(codec->toUnicode(item.oldMsgId));
-            msg.setComment(codec->toUnicode(item.tscomment));
-            msg.setOldComment(codec->toUnicode(item.oldTscomment));
-            msg.setExtraComment(codec->toUnicode(item.automaticComments));
-            msg.setTranslatorComment(codec->toUnicode(item.translatorComments));
+            msg.setId(toUnicode(item.id));
+            msg.setSourceText(toUnicode(item.msgId));
+            msg.setOldSourceText(toUnicode(item.oldMsgId));
+            msg.setComment(toUnicode(item.tscomment));
+            msg.setOldComment(toUnicode(item.oldTscomment));
+            msg.setExtraComment(toUnicode(item.automaticComments));
+            msg.setTranslatorComment(toUnicode(item.translatorComments));
             msg.setPlural(item.isPlural || item.msgStr.size() > 1);
             QStringList translations;
             foreach (const QByteArray &bstr, item.msgStr) {
-                QString str = codec->toUnicode(bstr);
+                QString str = toUnicode(bstr);
                 str.replace(QChar(Translator::TextVariantSeparator),
                             QChar(Translator::BinaryVariantSeparator));
                 translations << str;
@@ -642,14 +642,14 @@ bool loadPO(Translator &translator, QIODevice &dev, ConversionData &cd)
                         QByteArray extra = slurpEscapedString(lines, l, 16, "#| ", cd);
                         if (extra != item.oldMsgId)
                             item.extra[QLatin1String("po-old_msgid_plural")] =
-                                    codec->toUnicode(extra);
+                                    toUnicode(extra);
                     } else if (line.startsWith("#| msgctxt ")) {
                         item.oldTscomment = slurpEscapedString(lines, l, 11, "#| ", cd);
                         if (qtContexts)
                             splitContext(&item.oldTscomment, &item.context);
                     } else {
                         cd.appendError(QString(QLatin1String("PO-format parse error in line %1: '%2'"))
-                            .arg(l + 1).arg(codec->toUnicode(lines[l])));
+                            .arg(l + 1).arg(toUnicode(lines[l])));
                         error = true;
                     }
                     break;
@@ -660,7 +660,7 @@ bool loadPO(Translator &translator, QIODevice &dev, ConversionData &cd)
                         QByteArray extra = slurpEscapedString(lines, l, 16, "#~ ", cd);
                         if (extra != item.msgId)
                             item.extra[QLatin1String("po-msgid_plural")] =
-                                    codec->toUnicode(extra);
+                                    toUnicode(extra);
                         item.isPlural = true;
                     } else if (line.startsWith("#~ msgctxt ")) {
                         item.tscomment = slurpEscapedString(lines, l, 11, "#~ ", cd);
@@ -672,20 +672,20 @@ bool loadPO(Translator &translator, QIODevice &dev, ConversionData &cd)
                         QByteArray extra = slurpEscapedString(lines, l, 17, "#~| ", cd);
                         if (extra != item.oldMsgId)
                             item.extra[QLatin1String("po-old_msgid_plural")] =
-                                    codec->toUnicode(extra);
+                                    toUnicode(extra);
                     } else if (line.startsWith("#~| msgctxt ")) {
                         item.oldTscomment = slurpEscapedString(lines, l, 12, "#~| ", cd);
                         if (qtContexts)
                             splitContext(&item.oldTscomment, &item.context);
                     } else {
                         cd.appendError(QString(QLatin1String("PO-format parse error in line %1: '%2'"))
-                            .arg(l + 1).arg(codec->toUnicode(lines[l])));
+                            .arg(l + 1).arg(toUnicode(lines[l])));
                         error = true;
                     }
                     break;
                 default:
                     cd.appendError(QString(QLatin1String("PO-format parse error in line %1: '%2'"))
-                        .arg(l + 1).arg(codec->toUnicode(lines[l])));
+                        .arg(l + 1).arg(toUnicode(lines[l])));
                     error = true;
                     break;
             }
@@ -699,11 +699,11 @@ bool loadPO(Translator &translator, QIODevice &dev, ConversionData &cd)
         } else if (line.startsWith("msgid_plural ")) {
             QByteArray extra = slurpEscapedString(lines, l, 13, QByteArray(), cd);
             if (extra != item.msgId)
-                item.extra[QLatin1String("po-msgid_plural")] = codec->toUnicode(extra);
+                item.extra[QLatin1String("po-msgid_plural")] = toUnicode(extra);
             item.isPlural = true;
         } else {
             cd.appendError(QString(QLatin1String("PO-format error in line %1: '%2'"))
-                .arg(l + 1).arg(codec->toUnicode(lines[l])));
+                .arg(l + 1).arg(toUnicode(lines[l])));
             error = true;
         }
     }
@@ -735,7 +735,6 @@ bool savePO(const Translator &translator, QIODevice &dev, ConversionData &)
 
     bool ok = true;
     QTextStream out(&dev);
-    out.setCodec("UTF-8");
 
     bool qtContexts = false;
     foreach (const TranslatorMessage &msg, translator.messages())
@@ -754,7 +753,7 @@ bool savePO(const Translator &translator, QIODevice &dev, ConversionData &)
     // Keep in sync with loadPO
     addPoHeader(headers, hdrOrder, "MIME-Version", QLatin1String("1.0"));
     addPoHeader(headers, hdrOrder, "Content-Type",
-                QLatin1String("text/plain; charset=" + out.codec()->name()));
+                QLatin1String("text/plain; charset=UTF-8"));
     addPoHeader(headers, hdrOrder, "Content-Transfer-Encoding", QLatin1String("8bit"));
     if (!translator.languageCode().isEmpty()) {
         QLocale::Language l;
diff --git a/src/linguist/shared/qm.cpp b/src/linguist/shared/qm.cpp
index 288607824..3c047bc8d 100644
--- a/src/linguist/shared/qm.cpp
+++ b/src/linguist/shared/qm.cpp
@@ -38,7 +38,6 @@
 #include <QtCore/QFileInfo>
 #include <QtCore/QMap>
 #include <QtCore/QString>
-#include <QtCore/QTextCodec>
 
 QT_BEGIN_NAMESPACE
 
@@ -456,10 +455,9 @@ static quint32 read32(const uchar *data)
 
 static void fromBytes(const char *str, int len, QString *out, bool *utf8Fail)
 {
-    static QTextCodec *utf8Codec = QTextCodec::codecForName("UTF-8");
-    QTextCodec::ConverterState cvtState;
-    *out = utf8Codec->toUnicode(str, len, &cvtState);
-    *utf8Fail = cvtState.invalidChars;
+    QStringDecoder toUnicode(QStringDecoder::Utf8, QStringDecoder::Flag::Stateless);
+    *out = toUnicode(str, len);
+    *utf8Fail = toUnicode.hasError();
 }
 
 bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
@@ -623,7 +621,7 @@ bool loadQM(Translator &translator, QIODevice &dev, ConversionData &cd)
         translator.append(msg);
     }
     if (utf8Fail) {
-        cd.appendError(QLatin1String("Cannot read file with UTF-8 codec"));
+        cd.appendError(QLatin1String("Error: File contains invalid UTF-8 sequences."));
         return false;
     }
     return ok;
diff --git a/src/linguist/shared/qph.cpp b/src/linguist/shared/qph.cpp
index f29b3789b..a1895d387 100644
--- a/src/linguist/shared/qph.cpp
+++ b/src/linguist/shared/qph.cpp
@@ -30,7 +30,6 @@
 
 #include <QtCore/QByteArray>
 #include <QtCore/QDebug>
-#include <QtCore/QTextCodec>
 #include <QtCore/QTextStream>
 
 #include <QtCore/QXmlStreamReader>
@@ -147,7 +146,6 @@ static QString protect(const QString &str)
 static bool saveQPH(const Translator &translator, QIODevice &dev, ConversionData &)
 {
     QTextStream t(&dev);
-    t.setCodec(QTextCodec::codecForName("UTF-8"));
     t << "<!DOCTYPE QPH>\n<QPH";
     QString languageCode = translator.languageCode();
     if (!languageCode.isEmpty() && languageCode != QLatin1String("C"))
diff --git a/src/linguist/shared/ts.cpp b/src/linguist/shared/ts.cpp
index 5d727b6ab..2d59fab30 100644
--- a/src/linguist/shared/ts.cpp
+++ b/src/linguist/shared/ts.cpp
@@ -31,7 +31,6 @@
 #include <QtCore/QByteArray>
 #include <QtCore/QDebug>
 #include <QtCore/QRegularExpression>
-#include <QtCore/QTextCodec>
 #include <QtCore/QTextStream>
 
 #include <QtCore/QXmlStreamReader>
@@ -191,7 +190,6 @@ bool TSReader::read(Translator &translator)
     STRING(catalog);
     STRING(comment);
     STRING(context);
-    STRING(defaultcodec);
     STRING(dependencies);
     STRING(dependency);
     STRING(extracomment);
@@ -250,11 +248,6 @@ bool TSReader::read(Translator &translator)
                     break;
                 } else if (isWhiteSpace()) {
                     // ignore these, just whitespace
-                } else if (elementStarts(strdefaultcodec)) {
-                    // <defaultcodec>
-                    readElementText();
-                    m_cd.appendError(QString::fromLatin1("Warning: ignoring <defaultcodec> element"));
-                    // </defaultcodec>
                 } else if (isStartElement()
                         && name().toString().startsWith(strextrans)) {
                     // <extra-...>
@@ -516,8 +509,6 @@ bool saveTS(const Translator &translator, QIODevice &dev, ConversionData &cd)
 {
     bool result = true;
     QTextStream t(&dev);
-    t.setCodec(QTextCodec::codecForName("UTF-8"));
-    //qDebug() << translator.codecName();
 
     // The xml prolog allows processors to easily detect the correct encoding
     t << "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!DOCTYPE TS>\n";
diff --git a/src/linguist/shared/xliff.cpp b/src/linguist/shared/xliff.cpp
index e1cc64c59..7d869e06f 100644
--- a/src/linguist/shared/xliff.cpp
+++ b/src/linguist/shared/xliff.cpp
@@ -34,7 +34,6 @@
 #include <QtCore/QRegularExpression>
 #include <QtCore/QStack>
 #include <QtCore/QString>
-#include <QtCore/QTextCodec>
 #include <QtCore/QTextStream>
 
 // The string value is historical and reflects the main purpose: Keeping
@@ -757,7 +756,6 @@ bool saveXLIFF(const Translator &translator, QIODevice &dev, ConversionData &cd)
     int indent = 0;
 
     QTextStream ts(&dev);
-    ts.setCodec(QTextCodec::codecForName("UTF-8"));
 
     QStringList dtgs = cd.dropTags();
     dtgs << QLatin1String("po-(old_)?msgid_plural");
diff --git a/tests/auto/linguist/lupdate/testdata/good/codecforsrc/main.cpp b/tests/auto/linguist/lupdate/testdata/good/codecforsrc/main.cpp
index 7d9f15565..169c63cd1 100644
--- a/tests/auto/linguist/lupdate/testdata/good/codecforsrc/main.cpp
+++ b/tests/auto/linguist/lupdate/testdata/good/codecforsrc/main.cpp
@@ -29,7 +29,7 @@
 #include <QApplication>
 #include <QtGui>
 #include <QtCore>
-#include <QTextCodec>
+
 
 int main(int argc, char **argv)
 {
diff --git a/tests/auto/linguist/lupdate/testdata/good/parsecpp/expectedoutput.txt b/tests/auto/linguist/lupdate/testdata/good/parsecpp/expectedoutput.txt
index b67fb4944..c4dbae544 100644
--- a/tests/auto/linguist/lupdate/testdata/good/parsecpp/expectedoutput.txt
+++ b/tests/auto/linguist/lupdate/testdata/good/parsecpp/expectedoutput.txt
@@ -1,7 +1,7 @@
 .*/lupdate/testdata/good/parsecpp/finddialog.cpp:72: Qualifying with unknown namespace/class ::FindDialog
 .*/lupdate/testdata/good/parsecpp/finddialog.cpp:122: Qualifying with unknown namespace/class ::FindDialog
 .*/lupdate/testdata/good/parsecpp/finddialog.cpp:162: Qualifying with unknown namespace/class ::FindDialog
-.*/lupdate/testdata/good/parsecpp/finddialog.cpp:168: Unsupported encoding Latin1
+.*/lupdate/testdata/good/parsecpp/finddialog.cpp:168: Unsupported encoding Latin1/DefaultCodec/CodecForTr
 .*/lupdate/testdata/good/parsecpp/main.cpp:620: tr\(\) cannot be called without context
 .*/lupdate/testdata/good/parsecpp/main.cpp:622: tr\(\) cannot be called without context
 lupdate warning: Message with id 'yet_another_id' has no source.