Allow surrogate pairs for various "single character" locale data

Extract the character in its proper unicode form and encode it in a
new single_character_data table of locale data. Record each entry as
the range within that table that encodes it. Also added an assertion
in the generator script to check that the digits CLDR gives us are a
contiguous sequence in increasing order, as has been assumed by the
C++ code for some time. Lots of number-formatting code now has to take
account of how wide the digits are.

This leaves nowhere for updateSystemPrivate() to record values read
from sys_locale->query(), so we must always consult that function when
accessing these members of the systemData() object. Various internal
users of these single-character fields need the system-or-CLDR value
rather than the raw CLDR value, so move QLocalePrivate's methods to
supply them down to QLocaleData and ensure they check for system
values, where appropriate first.

This allows us to finally support the Chakma language and script, for
whose number system UTF-16 needs surrogate pairs.

Costs 10.8 kB in added data, much of it due to adding two new locales
that need surrogates to represent digits.

[ChangeLog][QtCore][QLocale] Various QLocale methods that returned
single QChar values now return QString values to accommodate those
locales which need a surrogate pair to represent the (single
character) return value.

Fixes: QTBUG-69324
Fixes: QTBUG-81053
Change-Id: I481722d6f5ee266164f09031679a851dfa6e7839
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

1 files changed, 69 insertions, 51 deletions

diff --git a/src/corelib/text/qlocale_tools.cpp b/src/corelib/text/qlocale_tools.cpp
index 0da769d694..9adb6e4a23 100644
--- a/src/corelib/text/qlocale_tools.cpp
+++ b/src/corelib/text/qlocale_tools.cpp
@@ -428,72 +428,85 @@ qstrtoll(const char * nptr, const char **endptr, int base, bool *ok)
     return result;
 }
 
-QString qulltoa(qulonglong l, int base, const QChar _zero)
+QString qulltoa(qulonglong number, int base, const QStringView zero)
 {
-    ushort buff[65]; // length of MAX_ULLONG in base 2
-    ushort *p = buff + 65;
-
-    if (base != 10 || _zero.unicode() == '0') {
-        while (l != 0) {
-            int c = l % base;
-
-            --p;
-
-            if (c < 10)
-                *p = '0' + c;
-            else
-                *p = c - 10 + 'a';
+    // Length of MAX_ULLONG in base 2 is 64; and we may need a surrogate pair
+    // per digit. We do not need a terminator.
+    const unsigned maxlen = 128;
+    Q_STATIC_ASSERT(CHAR_BIT * sizeof(number) <= maxlen);
+    ushort buff[maxlen];
+    ushort *const end = buff + maxlen, *p = end;
+
+    if (base != 10 || zero == u"0") {
+        while (number != 0) {
+            int c = number % base;
+            *--p = c < 10 ? '0' + c : c - 10 + 'a';
+            number /= base;
+        }
+    } else if (zero.size() && !zero.at(0).isSurrogate()) {
+        const ushort zeroUcs4 = zero.at(0).unicode();
+        while (number != 0) {
+            *(--p) = zeroUcs4 + number % base;
 
-            l /= base;
+            number /= base;
         }
-    }
-    else {
-        while (l != 0) {
-            int c = l % base;
+    } else if (zero.size() == 2 && zero.at(0).isHighSurrogate()) {
+        const uint zeroUcs4 = QChar::surrogateToUcs4(zero.at(0), zero.at(1));
+        while (number != 0) {
+            const uint digit = zeroUcs4 + number % base;
 
-            *(--p) = _zero.unicode() + c;
+            *(--p) = QChar::lowSurrogate(digit);
+            *(--p) = QChar::highSurrogate(digit);
 
-            l /= base;
+            number /= base;
         }
+    } else {
+        return QString();
     }
 
-    return QString(reinterpret_cast<QChar *>(p), 65 - (p - buff));
+    return QString(reinterpret_cast<QChar *>(p), end - p);
 }
 
-QString &decimalForm(QChar zero, QChar decimal, QChar group,
+QString &decimalForm(const QString &zero, const QString &decimal, const QString &group,
                      QString &digits, int decpt, int precision,
                      PrecisionMode pm,
                      bool always_show_decpt,
                      bool thousands_group)
 {
+    const auto digitWidth = zero.size();
+    Q_ASSERT(digitWidth == 1 || digitWidth == 2);
+    Q_ASSERT(digits.size() % digitWidth == 0);
+
     if (decpt < 0) {
         for (int i = 0; i < -decpt; ++i)
             digits.prepend(zero);
         decpt = 0;
-    }
-    else if (decpt > digits.length()) {
-        for (int i = digits.length(); i < decpt; ++i)
+    } else {
+        for (int i = digits.length() / digitWidth; i < decpt; ++i)
             digits.append(zero);
     }
 
-    if (pm == PMDecimalDigits) {
-        uint decimal_digits = digits.length() - decpt;
-        for (int i = decimal_digits; i < precision; ++i)
+    switch (pm) {
+    case PMDecimalDigits:
+        for (int i = digits.length() / digitWidth - decpt; i < precision; ++i)
             digits.append(zero);
-    }
-    else if (pm == PMSignificantDigits) {
-        for (int i = digits.length(); i < precision; ++i)
+        break;
+    case  PMSignificantDigits:
+        for (int i = digits.length() / digitWidth; i < precision; ++i)
             digits.append(zero);
-    }
-    else { // pm == PMChopTrailingZeros
+        break;
+    case PMChopTrailingZeros:
+        break;
     }
 
-    if (always_show_decpt || decpt < digits.length())
-        digits.insert(decpt, decimal);
+    if (always_show_decpt || decpt < digits.length() / digitWidth)
+        digits.insert(decpt * digitWidth, decimal);
 
+    // FIXME: they're not simply thousands separators !
+    // Need to mirror IndianNumberGrouping code in QLocaleData::longLongToString()
     if (thousands_group) {
         for (int i = decpt - 3; i > 0; i -= 3)
-            digits.insert(i, group);
+            digits.insert(i * digitWidth, group);
     }
 
     if (decpt == 0)
@@ -502,32 +515,37 @@ QString &decimalForm(QChar zero, QChar decimal, QChar group,
     return digits;
 }
 
-QString &exponentForm(QChar zero, QChar decimal, QChar exponential,
-                      QChar group, QChar plus, QChar minus,
+QString &exponentForm(const QString &zero, const QString &decimal, const QString &exponential,
+                      const QString &group, const QString &plus, const QString &minus,
                       QString &digits, int decpt, int precision,
                       PrecisionMode pm,
                       bool always_show_decpt,
                       bool leading_zero_in_exponent)
 {
-    int exp = decpt - 1;
+    const auto digitWidth = zero.size();
+    Q_ASSERT(digitWidth == 1 || digitWidth == 2);
+    Q_ASSERT(digits.size() % digitWidth == 0);
 
-    if (pm == PMDecimalDigits) {
-        for (int i = digits.length(); i < precision + 1; ++i)
+    switch (pm) {
+    case PMDecimalDigits:
+        for (int i = digits.length() / digitWidth; i < precision + 1; ++i)
             digits.append(zero);
-    }
-    else if (pm == PMSignificantDigits) {
-        for (int i = digits.length(); i < precision; ++i)
+        break;
+    case PMSignificantDigits:
+        for (int i = digits.length() / digitWidth; i < precision; ++i)
             digits.append(zero);
-    }
-    else { // pm == PMChopTrailingZeros
+        break;
+    case PMChopTrailingZeros:
+        break;
     }
 
-    if (always_show_decpt || digits.length() > 1)
-        digits.insert(1, decimal);
+    if (always_show_decpt || digits.length() > digitWidth)
+        digits.insert(digitWidth, decimal);
 
     digits.append(exponential);
-    digits.append(QLocaleData::longLongToString(zero, group, plus, minus,
-                   exp, leading_zero_in_exponent ? 2 : 1, 10, -1, QLocaleData::AlwaysShowSign));
+    digits.append(QLocaleData::longLongToString(zero, group, plus, minus, decpt - 1,
+                                                leading_zero_in_exponent ? 2 : 1,
+                                                10, -1, QLocaleData::AlwaysShowSign));
 
     return digits;
 }