Get rid of QSettings::iniCodec()

Always encodee INI files as utf-8 in Qt6. This is mostly backwards
compatible, as old ini files would encode all non ascii characters.

[ChangeLog][Important behavioral changes] QSettings will now always
encode INI files as utf-8 (and the iniCodec/setIniCode methods are
removed). This is a change from Qt 5 and earlier, where QSettings would
by default escape all non ascii characters. The behavior is equivalent to
what you got in Qt5 by setting a utf-8 iniCodec on the settings object.
Settings files written in Qt 5 will still be readable in Qt 6 (unless
an iniCodec different from utf-8 was used), but to read Qt6 based ini
files in Qt 5 applications, setting the iniCodec to utf-8 is required.

Change-Id: Ic7dffcca17779bd5e3dae50d42ce633170289f6c
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
Reviewed-by: Alex Blasche <alexander.blasche@qt.io>

4 files changed, 43 insertions, 224 deletions

diff --git a/src/corelib/io/qsettings.cpp b/src/corelib/io/qsettings.cpp
index 2eeed3f57c..eeed759bc0 100644
--- a/src/corelib/io/qsettings.cpp
+++ b/src/corelib/io/qsettings.cpp
@@ -52,10 +52,7 @@
 #include "qtemporaryfile.h"
 #include "qstandardpaths.h"
 #include <qdatastream.h>
-
-#if QT_CONFIG(textcodec)
-#  include "qtextcodec.h"
-#endif
+#include <qstringconverter.h>
 
 #ifndef QT_NO_GEOM_VARIANT
 #include "qsize.h"
@@ -230,7 +227,7 @@ void QConfFile::clearCache()
 // QSettingsPrivate
 
 QSettingsPrivate::QSettingsPrivate(QSettings::Format format)
-    : format(format), scope(QSettings::UserScope /* nothing better to put */), iniCodec(nullptr), fallbacks(true),
+    : format(format), scope(QSettings::UserScope /* nothing better to put */), fallbacks(true),
       pendingChanges(false), status(QSettings::NoError)
 {
 }
@@ -238,7 +235,7 @@ QSettingsPrivate::QSettingsPrivate(QSettings::Format format)
 QSettingsPrivate::QSettingsPrivate(QSettings::Format format, QSettings::Scope scope,
                                    const QString &organization, const QString &application)
     : format(format), scope(scope), organizationName(organization), applicationName(application),
-      iniCodec(nullptr), fallbacks(true), pendingChanges(false), status(QSettings::NoError)
+      fallbacks(true), pendingChanges(false), status(QSettings::NoError)
 {
 }
 
@@ -616,16 +613,18 @@ bool QSettingsPrivate::iniUnescapedKey(const QByteArray &key, int from, int to,
     return lowercaseOnly;
 }
 
-void QSettingsPrivate::iniEscapedString(const QString &str, QByteArray &result, QTextCodec *codec)
+void QSettingsPrivate::iniEscapedString(const QString &str, QByteArray &result)
 {
     bool needsQuotes = false;
     bool escapeNextIfDigit = false;
-    bool useCodec = codec && !str.startsWith(QLatin1String("@ByteArray("))
+    bool useCodec = !str.startsWith(QLatin1String("@ByteArray("))
                     && !str.startsWith(QLatin1String("@Variant("));
 
     int i;
     int startPos = result.size();
 
+    QStringEncoder toUtf8(QStringEncoder::Utf8);
+
     result.reserve(startPos + str.size() * 3 / 2);
     const QChar *unicode = str.unicode();
     for (i = 0; i < str.size(); ++i) {
@@ -678,11 +677,9 @@ void QSettingsPrivate::iniEscapedString(const QString &str, QByteArray &result,
             if (ch <= 0x1F || (ch >= 0x7F && !useCodec)) {
                 result += "\\x" + QByteArray::number(ch, 16);
                 escapeNextIfDigit = true;
-#if QT_CONFIG(textcodec)
             } else if (useCodec) {
                 // slow
-                result += codec->fromUnicode(&unicode[i], 1);
-#endif
+                result += toUtf8(&unicode[i], 1);
             } else {
                 result += (char)ch;
             }
@@ -705,7 +702,7 @@ inline static void iniChopTrailingSpaces(QString &str, int limit)
         str.truncate(n--);
 }
 
-void QSettingsPrivate::iniEscapedStringList(const QStringList &strs, QByteArray &result, QTextCodec *codec)
+void QSettingsPrivate::iniEscapedStringList(const QStringList &strs, QByteArray &result)
 {
     if (strs.isEmpty()) {
         /*
@@ -721,14 +718,13 @@ void QSettingsPrivate::iniEscapedStringList(const QStringList &strs, QByteArray
         for (int i = 0; i < strs.size(); ++i) {
             if (i != 0)
                 result += ", ";
-            iniEscapedString(strs.at(i), result, codec);
+            iniEscapedString(strs.at(i), result);
         }
     }
 }
 
 bool QSettingsPrivate::iniUnescapedStringList(const QByteArray &str, int from, int to,
-                                              QString &stringResult, QStringList &stringListResult,
-                                              QTextCodec *codec)
+                                              QString &stringResult, QStringList &stringListResult)
 {
     static const char escapeCodes[][2] =
     {
@@ -751,6 +747,7 @@ bool QSettingsPrivate::iniUnescapedStringList(const QByteArray &str, int from, i
     char16_t escapeVal = 0;
     int i = from;
     char ch;
+    QStringDecoder fromUtf8(QStringDecoder::Utf8);
 
 StSkipSpaces:
     while (i < to && ((ch = str.at(i)) == ' ' || ch == '\t'))
@@ -830,20 +827,7 @@ StNormal:
                 ++j;
             }
 
-#if !QT_CONFIG(textcodec)
-            Q_UNUSED(codec)
-#else
-            if (codec) {
-                stringResult += codec->toUnicode(str.constData() + i, j - i);
-            } else
-#endif
-            {
-                int n = stringResult.size();
-                stringResult.resize(n + (j - i));
-                QChar *resultData = stringResult.data() + n;
-                for (int k = i; k < j; ++k)
-                    *resultData++ = QLatin1Char(str.at(k));
-            }
+            stringResult += fromUtf8(str.constData() + i, j - i);
             i = j;
         }
         }
@@ -1675,16 +1659,10 @@ bool QConfFileSettingsPrivate::readIniFile(const QByteArray &data,
     int sectionPosition = 0;
     bool ok = true;
 
-    // detect utf8 BOM
+    // skip potential utf8 BOM
     const uchar *dd = (const uchar *)data.constData();
-    if (data.size() >= 3 && dd[0] == 0xef && dd[1] == 0xbb && dd[2] == 0xbf) {
-#if QT_CONFIG(textcodec)
-        iniCodec = QTextCodec::codecForName("UTF-8");
-#else
-        ok = false;
-#endif
+    if (data.size() >= 3 && dd[0] == 0xef && dd[1] == 0xbb && dd[2] == 0xbf)
         dataPos = 3;
-    }
 
     while (readIniLine(data, dataPos, lineStart, lineLen, equalsPos)) {
         char ch = data.at(lineStart);
@@ -1728,7 +1706,7 @@ bool QConfFileSettingsPrivate::readIniFile(const QByteArray &data,
 }
 
 bool QConfFileSettingsPrivate::readIniSection(const QSettingsKey &section, const QByteArray &data,
-                                              ParsedSettingsMap *settingsMap, QTextCodec *codec)
+                                              ParsedSettingsMap *settingsMap)
 {
     QStringList strListValue;
     bool sectionIsLowercase = (section == section.originalCaseKey());
@@ -1761,7 +1739,7 @@ bool QConfFileSettingsPrivate::readIniSection(const QSettingsKey &section, const
         QString strValue;
         strValue.reserve(lineLen - (valueStart - lineStart));
         bool isStringList = iniUnescapedStringList(data, valueStart, lineStart + lineLen,
-                                                   strValue, strListValue, codec);
+                                                   strValue, strListValue);
         QVariant variant;
         if (isStringList) {
             variant = stringListToVariantList(strListValue);
@@ -1894,9 +1872,9 @@ bool QConfFileSettingsPrivate::writeIniFile(QIODevice &device, const ParsedSetti
             */
             if (value.userType() == QMetaType::QStringList
                     || (value.userType() == QMetaType::QVariantList && value.toList().size() != 1)) {
-                iniEscapedStringList(variantListToStringList(value.toList()), block, iniCodec);
+                iniEscapedStringList(variantListToStringList(value.toList()), block);
             } else {
-                iniEscapedString(variantToString(value), block, iniCodec);
+                iniEscapedString(variantToString(value), block);
             }
             block += eol;
             if (device.write(block) == -1) {
@@ -1914,7 +1892,7 @@ void QConfFileSettingsPrivate::ensureAllSectionsParsed(QConfFile *confFile) cons
     const UnparsedSettingsMap::const_iterator end = confFile->unparsedIniSections.constEnd();
 
     for (; i != end; ++i) {
-        if (!QConfFileSettingsPrivate::readIniSection(i.key(), i.value(), &confFile->originalKeys, iniCodec))
+        if (!QConfFileSettingsPrivate::readIniSection(i.key(), i.value(), &confFile->originalKeys))
             setStatus(QSettings::FormatError);
     }
     confFile->unparsedIniSections.clear();
@@ -1942,7 +1920,7 @@ void QConfFileSettingsPrivate::ensureSectionParsed(QConfFile *confFile,
             return;
     }
 
-    if (!QConfFileSettingsPrivate::readIniSection(i.key(), i.value(), &confFile->originalKeys, iniCodec))
+    if (!QConfFileSettingsPrivate::readIniSection(i.key(), i.value(), &confFile->originalKeys))
         setStatus(QSettings::FormatError);
     confFile->unparsedIniSections.erase(i);
 }
@@ -2508,14 +2486,21 @@ void QConfFileSettingsPrivate::ensureSectionParsed(QConfFile *confFile,
         such as "General/someKey", the key will be located in the
         "%General" section, \e not in the "General" section.
 
-    \li  Following the philosophy that we should be liberal in what
-        we accept and conservative in what we generate, QSettings
-        will accept Latin-1 encoded INI files, but generate pure
-        ASCII files, where non-ASCII values are encoded using standard
-        INI escape sequences. To make the INI files more readable (but
-        potentially less compatible), call setIniCodec().
+    \li In line with most implementations today, QSettings will
+        assume the INI file is utf-8 encoded. This means that keys and values
+        will be decoded as utf-8 encoded entries and written back as utf-8.
+
     \endlist
 
+    \section2 Compatibility with older Qt versions
+
+    Please note that this behavior is different to how QSettings behaved
+    in versions of Qt prior to Qt 6. INI files written with Qt 5 or earlier aree
+    however fully readable by a Qt 6 based application (unless a ini codec
+    different from utf8 had been set). But INI files written with Qt 6
+    will only be readable by older Qt versions if you set the "iniCodec" to
+    a utf-8 textcodec.
+
     \sa registerFormat(), setPath()
 */
 
@@ -2872,61 +2857,6 @@ QString QSettings::applicationName() const
     return d->applicationName;
 }
 
-#if QT_CONFIG(textcodec)
-
-/*!
-    \since 4.5
-
-    Sets the codec for accessing INI files (including \c .conf files on Unix)
-    to \a codec. The codec is used for decoding any data that is read from
-    the INI file, and for encoding any data that is written to the file. By
-    default, no codec is used, and non-ASCII characters are encoded using
-    standard INI escape sequences.
-
-    \warning The codec must be set immediately after creating the QSettings
-    object, before accessing any data.
-
-    \sa iniCodec()
-*/
-void QSettings::setIniCodec(QTextCodec *codec)
-{
-    Q_D(QSettings);
-    d->iniCodec = codec;
-}
-
-/*!
-    \since 4.5
-    \overload
-
-    Sets the codec for accessing INI files (including \c .conf files on Unix)
-    to the QTextCodec for the encoding specified by \a codecName. Common
-    values for \c codecName include "ISO 8859-1", "UTF-8", and "UTF-16".
-    If the encoding isn't recognized, nothing happens.
-
-    \sa QTextCodec::codecForName()
-*/
-void QSettings::setIniCodec(const char *codecName)
-{
-    Q_D(QSettings);
-    if (QTextCodec *codec = QTextCodec::codecForName(codecName))
-        d->iniCodec = codec;
-}
-
-/*!
-    \since 4.5
-
-    Returns the codec that is used for accessing INI files. By default,
-    no codec is used, so \nullptr is returned.
-*/
-
-QTextCodec *QSettings::iniCodec() const
-{
-    Q_D(const QSettings);
-    return d->iniCodec;
-}
-
-#endif // textcodec
-
 /*!
     Returns a status code indicating the first error that was met by
     QSettings, or QSettings::NoError if no error occurred.
diff --git a/src/corelib/io/qsettings.h b/src/corelib/io/qsettings.h
index 07c746d043..0ca5c99432 100644
--- a/src/corelib/io/qsettings.h
+++ b/src/corelib/io/qsettings.h
@@ -179,12 +179,6 @@ public:
     QString organizationName() const;
     QString applicationName() const;
 
-#if QT_CONFIG(textcodec)
-    void setIniCodec(QTextCodec *codec);
-    void setIniCodec(const char *codecName);
-    QTextCodec *iniCodec() const;
-#endif
-
     static void setDefaultFormat(Format format);
     static Format defaultFormat();
 #if QT_DEPRECATED_SINCE(5, 13)
diff --git a/src/corelib/io/qsettings_p.h b/src/corelib/io/qsettings_p.h
index c30f099a72..795781cf77 100644
--- a/src/corelib/io/qsettings_p.h
+++ b/src/corelib/io/qsettings_p.h
@@ -235,18 +235,16 @@ public:
     static QVariant stringToVariant(const QString &s);
     static void iniEscapedKey(const QString &key, QByteArray &result);
     static bool iniUnescapedKey(const QByteArray &key, int from, int to, QString &result);
-    static void iniEscapedString(const QString &str, QByteArray &result, QTextCodec *codec);
-    static void iniEscapedStringList(const QStringList &strs, QByteArray &result, QTextCodec *codec);
+    static void iniEscapedString(const QString &str, QByteArray &result);
+    static void iniEscapedStringList(const QStringList &strs, QByteArray &result);
     static bool iniUnescapedStringList(const QByteArray &str, int from, int to,
-                                       QString &stringResult, QStringList &stringListResult,
-                                       QTextCodec *codec);
+                                       QString &stringResult, QStringList &stringListResult);
     static QStringList splitArgs(const QString &s, int idx);
 
     QSettings::Format format;
     QSettings::Scope scope;
     QString organizationName;
     QString applicationName;
-    QTextCodec *iniCodec;
 
 protected:
     QStack<QSettingsGroup> groupStack;
@@ -283,7 +281,7 @@ public:
 
     bool readIniFile(const QByteArray &data, UnparsedSettingsMap *unparsedIniSections);
     static bool readIniSection(const QSettingsKey &section, const QByteArray &data,
-                               ParsedSettingsMap *settingsMap, QTextCodec *codec);
+                               ParsedSettingsMap *settingsMap);
     static bool readIniLine(const QByteArray &data, int &dataPos, int &lineStart, int &lineLen,
                             int &equalsPos);
 
diff --git a/tests/auto/corelib/io/qsettings/tst_qsettings.cpp b/tests/auto/corelib/io/qsettings/tst_qsettings.cpp
index 427ce1a424..cef03c4591 100644
--- a/tests/auto/corelib/io/qsettings/tst_qsettings.cpp
+++ b/tests/auto/corelib/io/qsettings/tst_qsettings.cpp
@@ -170,7 +170,6 @@ private slots:
     void childGroups();
     void childKeys_data();
     void childKeys();
-    void setIniCodec();
     void testIniParsing_data();
     void testIniParsing();
     void testEscapes();
@@ -188,7 +187,6 @@ private slots:
     void testByteArray_data();
     void testByteArray();
     void testByteArrayNativeFormat();
-    void iniCodec();
     void bom();
     void embeddedZeroByte_data();
     void embeddedZeroByte();
@@ -695,28 +693,6 @@ void tst_QSettings::testByteArrayNativeFormat()
 #endif
 }
 
-void tst_QSettings::iniCodec()
-{
-    {
-        QSettings settings("QtProject", "tst_qsettings");
-        settings.setIniCodec("cp1251");
-        QByteArray ba;
-        ba.resize(256);
-        for (int i = 0; i < ba.size(); i++)
-            ba[i] = i;
-        settings.setValue("array",ba);
-    }
-    {
-        QSettings settings("QtProject", "tst_qsettings");
-        settings.setIniCodec("cp1251");
-        QByteArray ba = settings.value("array").toByteArray();
-        QCOMPARE(ba.size(), 256);
-        for (int i = 0; i < ba.size(); i++)
-            QCOMPARE((uchar)ba.at(i), (uchar)i);
-    }
-
-}
-
 void tst_QSettings::bom()
 {
     QSettings s(":/bom.ini", QSettings::IniFormat);
@@ -2416,71 +2392,6 @@ void tst_QSettings::fromFile()
     QDir::setCurrent(oldCur);
 }
 
-#ifdef QT_BUILD_INTERNAL
-void tst_QSettings::setIniCodec()
-{
-    QByteArray expeContents4, expeContents5;
-    QByteArray actualContents4, actualContents5;
-
-    {
-        QFile inFile(":/resourcefile4.ini");
-        inFile.open(QIODevice::ReadOnly);
-        expeContents4 = inFile.readAll();
-        inFile.close();
-    }
-
-    {
-        QFile inFile(":/resourcefile5.ini");
-        inFile.open(QIODevice::ReadOnly);
-        expeContents5 = inFile.readAll();
-        inFile.close();
-    }
-
-    {
-        QSettings settings4(QSettings::IniFormat, QSettings::UserScope, "software.org", "KillerAPP");
-        settings4.setIniCodec("UTF-8");
-        settings4.setValue(QLatin1String("Fa\xe7" "ade/QU\xc9" "BEC"), QLatin1String("Fa\xe7" "ade/QU\xc9" "BEC"));
-        settings4.sync();
-
-        QSettings settings5(QSettings::IniFormat, QSettings::UserScope, "other.software.org", "KillerAPP");
-        settings5.setIniCodec("ISO 8859-1");
-        settings5.setValue(QLatin1String("Fa\xe7" "ade/QU\xc9" "BEC"), QLatin1String("Fa\xe7" "ade/QU\xc9" "BEC"));
-        settings5.sync();
-
-        {
-            QFile inFile(settings4.fileName());
-            inFile.open(QIODevice::ReadOnly | QIODevice::Text);
-            actualContents4 = inFile.readAll();
-            inFile.close();
-        }
-
-        {
-            QFile inFile(settings5.fileName());
-            inFile.open(QIODevice::ReadOnly | QIODevice::Text);
-            actualContents5 = inFile.readAll();
-            inFile.close();
-        }
-    }
-
-    QConfFile::clearCache();
-
-    QCOMPARE(actualContents4, expeContents4);
-    QCOMPARE(actualContents5, expeContents5);
-
-    QSettings settings4(QSettings::IniFormat, QSettings::UserScope, "software.org", "KillerAPP");
-    settings4.setIniCodec("UTF-8");
-    QSettings settings5(QSettings::IniFormat, QSettings::UserScope, "other.software.org", "KillerAPP");
-    settings5.setIniCodec("Latin-1");
-
-    QCOMPARE(settings4.allKeys().count(), 1);
-    QCOMPARE(settings5.allKeys().count(), 1);
-
-    QCOMPARE(settings4.allKeys().first(), settings5.allKeys().first());
-    QCOMPARE(settings4.value(settings4.allKeys().first()).toString(),
-             settings5.value(settings5.allKeys().first()).toString());
-}
-#endif
-
 static bool containsSubList(QStringList mom, QStringList son)
 {
     for (int i = 0; i < son.size(); ++i) {
@@ -2782,7 +2693,7 @@ static QString iniUnescapedKey(const QByteArray &ba)
 static QByteArray iniEscapedStringList(const QStringList &strList)
 {
     QByteArray result;
-    QSettingsPrivate::iniEscapedStringList(strList, result, 0);
+    QSettingsPrivate::iniEscapedStringList(strList, result);
     return result;
 }
 
@@ -2790,23 +2701,9 @@ static QStringList iniUnescapedStringList(const QByteArray &ba)
 {
     QStringList result;
     QString str;
-#if QSETTINGS_P_H_VERSION >= 2
-    bool isStringList = QSettingsPrivate::iniUnescapedStringList(ba, 0, ba.size(), str, result
-#if QSETTINGS_P_H_VERSION >= 3
-                                                                 , 0
-#endif
-                                                                    );
+    bool isStringList = QSettingsPrivate::iniUnescapedStringList(ba, 0, ba.size(), str, result);
     if (!isStringList)
         result = QStringList(str);
-#else
-    QStringList *strList = QSettingsPrivate::iniUnescapedStringList(ba, 0, ba.size(), str);
-    if (strList) {
-        result = *strList;
-        delete strList;
-    } else {
-        result = QStringList(str);
-    }
-#endif
     return result;
 }
 #endif
@@ -2905,8 +2802,8 @@ void tst_QSettings::testEscapes()
     testEscapedStringList(QChar(0) + QString("0"), "\\0\\x30");
     testEscapedStringList("~!@#$%^&*()_+.-/\\=", "\"~!@#$%^&*()_+.-/\\\\=\"");
     testEscapedStringList("~!@#$%^&*()_+.-/\\", "~!@#$%^&*()_+.-/\\\\");
-    testEscapedStringList(QString("\x7F") + "12aFz", "\\x7f\\x31\\x32\\x61\\x46z");
-    testEscapedStringList(QString("   \t\n\\n") + QChar(0x123) + QChar(0x4567), "\"   \\t\\n\\\\n\\x123\\x4567\"");
+    testEscapedStringList(QString("\x7F") + "12aFz", QByteArray("\x7f") + "12aFz");
+    testEscapedStringList(QString("   \t\n\\n") + QChar(0x123) + QChar(0x4567), "\"   \\t\\n\\\\n\xC4\xA3\xE4\x95\xA7\"");
     testEscapedStringList(QString("\a\b\f\n\r\t\v'\"?\001\002\x03\x04"), "\\a\\b\\f\\n\\r\\t\\v'\\\"?\\x1\\x2\\x3\\x4");
     testEscapedStringList(QStringList() << "," << ";" << "a" << "ab,  \tc, d ", "\",\", \";\", a, \"ab,  \\tc, d \"");
 
@@ -2921,7 +2818,7 @@ void tst_QSettings::testEscapes()
                             QString() + QChar(0) + QChar(0) + QChar(0) + QChar(0) + QChar(1)
                             + QChar(0111) + QChar(011111) + QChar(0) + QChar(0xCDEF) + "GH"
                             + QChar(0x3456),
-                            "\\0\\0\\0\\0\\x1I\\x1249\\0\\xcdefGH\\x3456");
+                            "\\0\\0\\0\\0\\x1I\xE1\x89\x89\\0\xEC\xB7\xAFGH\xE3\x91\x96");
     testUnescapedStringList(QByteArray("\\c\\d\\e\\f\\g\\$\\*\\\0", 16), "\f", "\\f");
     testUnescapedStringList("\"a\",  \t\"bc \", \"  d\" , \"ef  \" ,,g,   hi  i,,, ,",
                             QStringList() << "a" << "bc " << "  d" << "ef  " << "" << "g" << "hi  i"