uic: Refactor string constant formatting

Refactor the fixString() helper, moving the code into a streamable
class using a helper which can be used for different encodings.

Task-number: PYSIDE-797
Change-Id: I0f82945b6b334da8524882dda2f104327eba79d4
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>

4 files changed, 226 insertions, 117 deletions

diff --git a/src/tools/uic/cpp/cppwriteinitialization.cpp b/src/tools/uic/cpp/cppwriteinitialization.cpp
index 881c48daea..938d47f62d 100644
--- a/src/tools/uic/cpp/cppwriteinitialization.cpp
+++ b/src/tools/uic/cpp/cppwriteinitialization.cpp
@@ -506,7 +506,8 @@ void WriteInitialization::acceptUI(DomUI *node)
             continue;
 
         const QString varConn = connection + QLatin1String("Connection");
-        m_output << m_indent << varConn << " = QSqlDatabase::database(" << fixString(connection, m_dindent) << ");\n";
+        m_output << m_indent << varConn << " = QSqlDatabase::database("
+            << language::charliteral(connection, m_dindent) << ");\n";
     }
 
     acceptWidget(node->elementWidget());
@@ -1108,7 +1109,7 @@ QString WriteInitialization::writeStringListProperty(const DomStringList *list)
             str << '\n' << m_indent << "    << " << trCall(values.at(i), comment);
     } else {
         for (int i = 0; i < values.size(); ++i)
-            str << " << QString::fromUtf8(" << fixString(values.at(i), m_dindent) << ')';
+            str << " << " << language::qstring(values.at(i), m_dindent);
     }
     return propertyValue;
 }
@@ -1140,8 +1141,8 @@ void WriteInitialization::writeProperties(const QString &varName,
     if (m_uic->customWidgetsInfo()->extends(className, QLatin1String("QAxWidget"))) {
         DomPropertyMap properties = propertyMap(lst);
         if (DomProperty *p = properties.value(QLatin1String("control"))) {
-            m_output << m_indent << varName << "->setControl(QString::fromUtf8("
-                     << fixString(toString(p->elementString()), m_dindent) << "));\n";
+            m_output << m_indent << varName << "->setControl("
+                << language::qstring(toString(p->elementString()), m_dindent) << ");\n";
         }
     }
 
@@ -1150,9 +1151,11 @@ void WriteInitialization::writeProperties(const QString &varName,
         indent = m_option.indent;
         m_output << m_indent << "if (" << varName << "->objectName().isEmpty())\n";
     }
-    if (!(flags & WritePropertyIgnoreObjectName))
+    if (!(flags & WritePropertyIgnoreObjectName)) {
         m_output << m_indent << indent << varName
-                 << "->setObjectName(QString::fromUtf8(" << fixString(varName, m_dindent) << "));\n";
+            << "->setObjectName(" << language::qstring(varName, m_dindent)
+            << ");\n";
+    }
 
     int leftMargin, topMargin, rightMargin, bottomMargin;
     leftMargin = topMargin = rightMargin = bottomMargin = -1;
@@ -1281,13 +1284,12 @@ void WriteInitialization::writeProperties(const QString &varName,
                 Buddy buddy = { varName, p->elementCstring() };
                 m_buddies.append(std::move(buddy));
             } else {
-                if (stdset)
-                    propertyValue = fixString(p->elementCstring(), m_dindent);
-                else {
-                    propertyValue = QLatin1String("QByteArray(")
-                                  + fixString(p->elementCstring(), m_dindent)
-                                  + QLatin1Char(')');
-                }
+                QTextStream str(&propertyValue);
+                if (!stdset)
+                    str << "QByteArray(";
+                str << language::charliteral(p->elementCstring(), m_dindent);
+                if (!stdset)
+                    str << ')';
             }
             break;
         case DomProperty::Cursor:
@@ -1456,8 +1458,8 @@ void WriteInitialization::writeProperties(const QString &varName,
 
         case DomProperty::Url: {
             const DomUrl* u = p->elementUrl();
-            propertyValue = QString::fromLatin1("QUrl(QString::fromUtf8(%1))")
-                            .arg(fixString(u->elementString()->text(), m_dindent));
+            QTextStream(&propertyValue) << "QUrl("
+                << language::qstring(u->elementString()->text(), m_dindent) << ")";
             break;
         }
         case DomProperty::Brush:
@@ -1549,8 +1551,8 @@ QString WriteInitialization::writeFontProperties(const DomFont *f)
 
     m_output << m_indent << "QFont " << fontName << ";\n";
     if (f->hasElementFamily() && !f->elementFamily().isEmpty()) {
-        m_output << m_indent << fontName << ".setFamily(QString::fromUtf8("
-                 << fixString(f->elementFamily(), m_dindent) << "));\n";
+        m_output << m_indent << fontName << ".setFamily("
+            << language::qstring(f->elementFamily(), m_dindent) << ");\n";
     }
     if (f->hasElementPointSize() && f->elementPointSize() > 0) {
          m_output << m_indent << fontName << ".setPointSize(" << f->elementPointSize()
@@ -1599,43 +1601,43 @@ static void writeResourceIcon(QTextStream &output,
                               const DomResourceIcon *i)
 {
     if (i->hasElementNormalOff()) {
-        output << indent << iconName << ".addFile(QString::fromUtf8("
-               << fixString(i->elementNormalOff()->text(), indent)
-               << "), QSize(), QIcon::Normal, QIcon::Off);\n";
+        output << indent << iconName << ".addFile("
+               << language::qstring(i->elementNormalOff()->text(), indent)
+               << ", QSize(), QIcon::Normal, QIcon::Off);\n";
     }
     if (i->hasElementNormalOn()) {
-        output << indent << iconName << ".addFile(QString::fromUtf8("
-               << fixString(i->elementNormalOn()->text(), indent)
-               << "), QSize(), QIcon::Normal, QIcon::On);\n";
+        output << indent << iconName << ".addFile("
+               << language::qstring(i->elementNormalOn()->text(), indent)
+               << ", QSize(), QIcon::Normal, QIcon::On);\n";
     }
     if (i->hasElementDisabledOff()) {
-        output << indent << iconName << ".addFile(QString::fromUtf8("
-               << fixString(i->elementDisabledOff()->text(), indent)
-               << "), QSize(), QIcon::Disabled, QIcon::Off);\n";
+        output << indent << iconName << ".addFile("
+               << language::qstring(i->elementDisabledOff()->text(), indent)
+               << ", QSize(), QIcon::Disabled, QIcon::Off);\n";
     }
     if (i->hasElementDisabledOn()) {
-        output << indent << iconName << ".addFile(QString::fromUtf8("
-               << fixString(i->elementDisabledOn()->text(), indent)
-               << "), QSize(), QIcon::Disabled, QIcon::On);\n";
+        output << indent << iconName << ".addFile("
+               << language::qstring(i->elementDisabledOn()->text(), indent)
+               << ", QSize(), QIcon::Disabled, QIcon::On);\n";
     }
     if (i->hasElementActiveOff()) {
-        output << indent << iconName << ".addFile(QString::fromUtf8("
-               << fixString(i->elementActiveOff()->text(), indent)
-               << "), QSize(), QIcon::Active, QIcon::Off);\n";
+        output << indent << iconName << ".addFile("
+               << language::qstring(i->elementActiveOff()->text(), indent)
+               << ", QSize(), QIcon::Active, QIcon::Off);\n";
     }
     if (i->hasElementActiveOn()) {
-        output << indent << iconName << ".addFile(QString::fromUtf8("
-               << fixString(i->elementActiveOn()->text(), indent)
-               << "), QSize(), QIcon::Active, QIcon::On);\n";
+        output << indent << iconName << ".addFile("
+               << language::qstring(i->elementActiveOn()->text(), indent)
+               << ", QSize(), QIcon::Active, QIcon::On);\n";
     }
     if (i->hasElementSelectedOff()) {
-        output << indent << iconName << ".addFile(QString::fromUtf8("
-               << fixString(i->elementSelectedOff()->text(), indent)
-               << "), QSize(), QIcon::Selected, QIcon::Off);\n";
+        output << indent << iconName << ".addFile("
+               << language::qstring(i->elementSelectedOff()->text(), indent)
+               << ", QSize(), QIcon::Selected, QIcon::Off);\n";
     }
     if (i->hasElementSelectedOn()) {
-        output << indent << iconName << ".addFile(QString::fromUtf8("
-               << fixString(i->elementSelectedOn()->text(), indent)
+        output << indent << iconName << ".addFile("
+               << language::qstring(i->elementSelectedOn()->text(), indent)
                << "), QSize(), QIcon::Selected, QIcon::On);\n";
     }
 }
@@ -1709,7 +1711,6 @@ QString WriteInitialization::writeIconProperties(const DomResourceIcon *i)
                 writePixmapFunctionIcon(m_output, iconName, m_indent, i);
         } else {
             // Theme: Generate code to check the theme and default to resource
-            const QString themeIconName = fixString(i->attributeTheme(), QString());
             if (iconHasStatePixmaps(i)) {
                 // Theme + default state pixmaps:
                 // Generate code to check the theme and default to state pixmaps
@@ -1721,8 +1722,8 @@ QString WriteInitialization::writeIconProperties(const DomResourceIcon *i)
                     m_output << "QString ";
                     m_firstThemeIcon = false;
                 }
-                m_output << themeNameStringVariableC << " = QString::fromUtf8("
-                         << themeIconName << ");\n";
+                m_output << themeNameStringVariableC << " = "
+                    << language::qstring(i->attributeTheme()) << ";\n";
                 m_output << m_indent << "if (QIcon::hasThemeIcon("
                          << themeNameStringVariableC
                          << ")) {\n"
@@ -1735,9 +1736,8 @@ QString WriteInitialization::writeIconProperties(const DomResourceIcon *i)
                 m_output << m_indent << "}\n";
             } else {
                 // Theme, but no state pixmaps: Construct from theme directly.
-                m_output << m_indent << "QIcon " << iconName
-                         << "(QIcon::fromTheme(QString::fromUtf8("
-                         << themeIconName << ")));\n";
+                m_output << m_indent << "QIcon " << iconName << "(QIcon::fromTheme("
+                    << language::qstring(i->attributeTheme()) << "));\n";
             } // Theme, but not state
         }     // >= 4.4
     } else {  // pre-4.4 legacy
@@ -1969,15 +1969,14 @@ QString WriteInitialization::pixCall(const QString &t, const QString &text) cons
         return type;
     }
 
+    QTextStream str(&type);
+    str << '(';
     QString pixFunc = m_uic->pixmapFunction();
     if (pixFunc.isEmpty())
-        pixFunc = QLatin1String("QString::fromUtf8");
-
-    type += QLatin1Char('(')
-          + pixFunc
-          + QLatin1Char('(')
-          + fixString(text, m_dindent)
-          + QLatin1String("))");
+        str << language::qstring(text, m_dindent);
+    else
+        str << pixFunc << '(' << language::charliteral(text, m_dindent) << ')';
+    str << ')';
     return type;
 }
 
@@ -2343,28 +2342,29 @@ QString WriteInitialization::trCall(const QString &str, const QString &commentHi
         return QLatin1String("QString()");
 
     QString result;
-    const QString comment = commentHint.isEmpty() ? QString(QLatin1String("nullptr")) : fixString(commentHint, m_dindent);
+    QTextStream ts(&result);
 
     const bool idBasedTranslations = m_driver->useIdBasedTranslations();
     if (m_option.translateFunction.isEmpty()) {
-        if (idBasedTranslations || m_option.idBased) {
-            result += QLatin1String("qtTrId(");
-        } else {
-            result += QLatin1String("QCoreApplication::translate(\"")
-                    + m_generatedClass
-                    + QLatin1String("\", ");
-        }
+        if (idBasedTranslations || m_option.idBased)
+            ts << "qtTrId(";
+        else
+            ts << "QCoreApplication::translate(\"" << m_generatedClass << "\", ";
     } else {
-        result += m_option.translateFunction + QLatin1Char('(');
+        ts << m_option.translateFunction << '(';
     }
 
-    result += fixString(idBasedTranslations ? id : str, m_dindent);
+    ts << language::charliteral(idBasedTranslations ? id : str, m_dindent);
 
     if (!idBasedTranslations && !m_option.idBased) {
-        result += QLatin1String(", ") + comment;
+        ts << ", ";
+        if (commentHint.isEmpty())
+            ts << "nullptr";
+        else
+            ts << language::charliteral(commentHint, m_dindent);
     }
 
-    result += QLatin1Char(')');
+    ts << ')';
     return result;
 }
 
@@ -2399,9 +2399,9 @@ QString WriteInitialization::noTrCall(DomString *str, const QString &defaultStri
         return QString();
     if (str)
         value = str->text();
-    QString ret = QLatin1String("QString::fromUtf8(");
-    ret += fixString(value, m_dindent);
-    ret += QLatin1Char(')');
+    QString ret;
+    QTextStream ts(&ret);
+    ts << language::qstring(value, m_dindent);
     return ret;
 }
 
diff --git a/src/tools/uic/shared/language.cpp b/src/tools/uic/shared/language.cpp
index 730e1562bc..d0491b74e3 100644
--- a/src/tools/uic/shared/language.cpp
+++ b/src/tools/uic/shared/language.cpp
@@ -32,6 +32,8 @@
 
 namespace language {
 
+static Encoding encoding = Encoding::Utf8;
+
 QTextStream &operator<<(QTextStream &str, const qtConfig &c)
 {
     str << "QT_CONFIG(" << c.parameter() << ')';
@@ -141,4 +143,129 @@ const char *paletteColorRole(int v)
     return lookupEnum(colorRoles, v);
 }
 
+// Helpers for formatting a character sequences
+
+// Format a special character like '\x0a'
+static int formatEscapedNumber(QTextStream &str, ushort value, int base, int width,
+                               char prefix = 0)
+{
+    int length = 1 + width;
+    str << '\\';
+    if (prefix) {
+        str << prefix;
+        ++length;
+    }
+    const auto oldPadChar = str.padChar();
+    const auto oldFieldWidth = str.fieldWidth();
+    const auto oldFieldAlignment = str.fieldAlignment();
+    const auto oldIntegerBase = str.integerBase();
+    str.setPadChar(QLatin1Char('0'));
+    str.setFieldWidth(width);
+    str.setFieldAlignment(QTextStream::AlignRight);
+    str.setIntegerBase(base);
+    str << value;
+    str.setIntegerBase(oldIntegerBase);
+    str.setFieldAlignment(oldFieldAlignment);
+    str.setFieldWidth(oldFieldWidth);
+    str.setPadChar(oldPadChar);
+    return length;
+}
+
+static int formatSpecialCharacter(QTextStream &str, ushort value)
+{
+    int length = 0;
+    switch (value) {
+    case '\\':
+        str << "\\\\";
+        length += 2;
+        break;
+    case '\"':
+        str << "\\\"";
+        length += 2;
+        break;
+    case '\n':
+        str << "\\n\"\n\"";
+        length += 5;
+        break;
+    default:
+        break;
+    }
+    return length;
+}
+
+// Format a sequence of characters for C++ with special characters numerically
+// escaped (non-raw string literals), wrappped at maxSegmentSize. FormattingTraits
+// are used to transform characters into (unsigned) codes, which can be used
+// for either normal escapes or Unicode code points as used in Unicode literals.
+
+enum : int { maxSegmentSize = 1024 };
+
+template <Encoding e>
+struct FormattingTraits
+{
+};
+
+template <>
+struct FormattingTraits<Encoding::Utf8>
+{
+    static ushort code(char c) { return uchar(c); }
+};
+
+template <>
+struct FormattingTraits<Encoding::Unicode>
+{
+    static ushort code(QChar c) { return c.unicode(); }
+};
+
+template <Encoding e, class Iterator>
+static void formatStringSequence(QTextStream &str, Iterator it, Iterator end,
+                                 const QString &indent,
+                                 int escapeIntegerBase, int escapeWidth,
+                                 char escapePrefix = 0)
+{
+    str << '"';
+    int length = 0;
+    while (it != end) {
+        const auto code = FormattingTraits<e>::code(*it);
+        if (code >= 0x80) {
+            length += formatEscapedNumber(str, code, escapeIntegerBase, escapeWidth, escapePrefix);
+        } else if (const int l = formatSpecialCharacter(str, code)) {
+            length += l;
+        } else if (code != '\r') {
+            str << *it;
+            ++length;
+        }
+        ++it;
+        if (it != end && length > maxSegmentSize) {
+            str << "\"\n" << indent << indent << '"';
+            length = 0;
+        }
+    }
+    str << '"';
+}
+
+void _formatString(QTextStream &str, const QString &value, const QString &indent,
+                   bool qString)
+{
+    switch (encoding) {
+    // Special characters as 3 digit octal escapes (u8"\303\234mlaut")
+    case Encoding::Utf8: {
+        if (qString)
+            str << "QString::fromUtf8(";
+        const QByteArray utf8 = value.toUtf8();
+        formatStringSequence<Encoding::Utf8>(str, utf8.cbegin(), utf8.cend(), indent,
+                                             8, 3);
+        if (qString)
+            str << ')';
+    }
+        break;
+    // Special characters as 4 digit hex Unicode points (u8"\u00dcmlaut")
+    case Encoding::Unicode:
+        str << 'u'; // Python Unicode literal (would be UTF-16 in C++)
+        formatStringSequence<Encoding::Unicode>(str, value.cbegin(), value.cend(), indent,
+                                                16, 4, 'u');
+        break;
+    }
+}
+
 } // namespace language
diff --git a/src/tools/uic/shared/language.h b/src/tools/uic/shared/language.h
index e7201b6529..04a3763e54 100644
--- a/src/tools/uic/shared/language.h
+++ b/src/tools/uic/shared/language.h
@@ -29,6 +29,7 @@
 #ifndef LANGUAGE_H
 #define LANGUAGE_H
 
+#include <QtCore/qstring.h>
 #include <QtCore/qstringview.h>
 
 QT_FORWARD_DECLARE_CLASS(QTextStream)
@@ -76,6 +77,36 @@ const char *sizePolicy(int v);
 const char *dockWidgetArea(int v);
 const char *paletteColorRole(int v);
 
+enum class Encoding { Utf8, Unicode };
+
+void _formatString(QTextStream &str, const QString &value, const QString &indent,
+                   bool qString);
+
+template <bool AsQString>
+class _string
+{
+public:
+    explicit _string(const QString &value, const QString &indent = QString())
+        : m_value(value), m_indent(indent) {}
+
+    void format(QTextStream &str) const
+    { _formatString(str, m_value, m_indent, AsQString); }
+
+private:
+    const QString &m_value;
+    const QString &m_indent;
+};
+
+template <bool AsQString>
+inline QTextStream &operator<<(QTextStream &str, const language::_string<AsQString> &s)
+{
+    s.format(str);
+    return str;
+}
+
+using charliteral = _string<false>;
+using qstring = _string<true>;
+
 } // namespace language
 
 #endif // LANGUAGE_H
diff --git a/src/tools/uic/utils.h b/src/tools/uic/utils.h
index 3f32a532ca..34c4ab23d4 100644
--- a/src/tools/uic/utils.h
+++ b/src/tools/uic/utils.h
@@ -42,55 +42,6 @@ inline bool toBool(const QString &str)
 inline QString toString(const DomString *str)
 { return str ? str->text() : QString(); }
 
-inline QString fixString(const QString &str, const QString &indent)
-{
-    QString cursegment;
-    QStringList result;
-    const QByteArray utf8 = str.toUtf8();
-    const int utf8Length = utf8.length();
-
-    for (int i = 0; i < utf8Length; ++i) {
-        const uchar cbyte = utf8.at(i);
-        if (cbyte >= 0x80) {
-            cursegment += QLatin1Char('\\');
-            cursegment += QString::number(cbyte, 8);
-        } else {
-            switch(cbyte) {
-            case '\\':
-                cursegment += QLatin1String("\\\\"); break;
-            case '\"':
-                cursegment += QLatin1String("\\\""); break;
-            case '\r':
-                break;
-            case '\n':
-                cursegment += QLatin1String("\\n\"\n\""); break;
-            default:
-                cursegment += QLatin1Char(cbyte);
-            }
-        }
-
-        if (cursegment.length() > 1024) {
-            result << cursegment;
-            cursegment.clear();
-        }
-    }
-
-    if (!cursegment.isEmpty())
-        result << cursegment;
-
-
-    QString joinstr = QLatin1String("\"\n");
-    joinstr += indent;
-    joinstr += indent;
-    joinstr += QLatin1Char('"');
-
-    QString rc(QLatin1Char('"'));
-    rc += result.join(joinstr);
-    rc += QLatin1Char('"');
-
-    return rc;
-}
-
 inline QHash<QString, DomProperty *> propertyMap(const QList<DomProperty *> &properties)
 {
     QHash<QString, DomProperty *> map;