Refactor the ICU code for QLocale

Clean up the ICU code and make it thread-safe. Add a
QIcuData structure to QLocalePrivate, that contains
ICU specific data.

Link against ICU directly, greatly simplifying the
code.

Also fix a bug in the locale specific case conversion
code that would cause it to fail and fall back to the
QString code if the output string was larger than the
input.

Change-Id: Ie67e5ea14fa204ebc5887d7aaeb1a4f3ecaf8697
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

6 files changed, 99 insertions, 238 deletions

diff --git a/src/corelib/tools/qlocale.cpp b/src/corelib/tools/qlocale.cpp
index 67e47e56bb..c809b559dd 100644
--- a/src/corelib/tools/qlocale.cpp
+++ b/src/corelib/tools/qlocale.cpp
@@ -82,12 +82,6 @@ static QLocaleData *system_data = 0;
 Q_GLOBAL_STATIC(QLocaleData, globalLocaleData)
 #endif
 
-#ifdef QT_USE_ICU
-extern bool qt_initIcu(const QString &localeName);
-extern bool qt_u_strToUpper(const QString &str, QString *out, const QLocale &locale);
-extern bool qt_u_strToLower(const QString &str, QString *out, const QLocale &locale);
-#endif
-
 /******************************************************************************
 ** Helpers for accessing Qt locale database
 */
@@ -515,12 +509,6 @@ void QLocalePrivate::updateSystemPrivate()
     res = sys_locale->query(QSystemLocale::PositiveSign, QVariant());
     if (!res.isNull())
         system_data->m_plus = res.toString().at(0).unicode();
-
-#ifdef QT_USE_ICU
-    if (!default_data)
-        qt_initIcu(sys_locale->fallbackUiLocale().bcp47Name());
-#endif
-
 }
 #endif
 
@@ -587,7 +575,7 @@ QDataStream &operator>>(QDataStream &ds, QLocale &l)
 
 static const int locale_data_size = sizeof(locale_data)/sizeof(QLocaleData) - 1;
 
-static const QLocaleData *dataPointerHelper(quint16 index)
+const QLocaleData *QLocalePrivate::dataPointerForIndex(quint16 index)
 {
 #ifndef QT_NO_SYSTEMLOCALE
     Q_ASSERT(index <= locale_data_size);
@@ -615,6 +603,14 @@ static quint16 localeDataIndex(const QLocaleData *p)
 }
 
 /*!
+ \internal
+*/
+QLocale::QLocale(QLocalePrivate &dd)
+    : d(&dd)
+{}
+
+
+/*!
     Constructs a QLocale object with the specified \a name,
     which has the format
     "language[_script][_country][.codeset][@modifier]" or "C", where:
@@ -644,11 +640,8 @@ static quint16 localeDataIndex(const QLocaleData *p)
 */
 
 QLocale::QLocale(const QString &name)
-    : d(new QLocalePrivate())
+    : d(new QLocalePrivate(localeDataIndex(findLocaleData(name))))
 {
-    d->m_numberOptions = 0;
-    d->m_index = localeDataIndex(findLocaleData(name));
-    d->m_data = dataPointerHelper(d->m_index);
 }
 
 /*!
@@ -660,11 +653,8 @@ QLocale::QLocale(const QString &name)
 */
 
 QLocale::QLocale()
-    : d(new QLocalePrivate())
+    : d(new QLocalePrivate(localeDataIndex(defaultData()), default_number_options))
 {
-    d->m_numberOptions = default_number_options;
-    d->m_index = localeDataIndex(defaultData());
-    d->m_data = dataPointerHelper(d->m_index);
 }
 
 /*!
@@ -687,19 +677,19 @@ QLocale::QLocale()
 */
 
 QLocale::QLocale(Language language, Country country)
-    : d(new QLocalePrivate())
 {
     const QLocaleData *data = QLocaleData::findLocaleData(language, QLocale::AnyScript, country);
+    int index;
+    int numberOptions = 0;
 
     // If not found, should default to system
     if (data->m_language_id == QLocale::C && language != QLocale::C) {
-        d->m_numberOptions = default_number_options;
-        d->m_index = localeDataIndex(defaultData());
+        numberOptions = default_number_options;
+        index = localeDataIndex(defaultData());
     } else {
-        d->m_numberOptions = 0;
-        d->m_index = localeDataIndex(data);
+        index = localeDataIndex(data);
     }
-    d->m_data = dataPointerHelper(d->m_index);
+    d = new QLocalePrivate(index, numberOptions);
 }
 \
 /*!
@@ -727,19 +717,19 @@ QLocale::QLocale(Language language, Country country)
 */
 
 QLocale::QLocale(Language language, Script script, Country country)
-    : d(new QLocalePrivate())
 {
     const QLocaleData *data = QLocaleData::findLocaleData(language, script, country);
+    int index;
+    int numberOptions = 0;
 
     // If not found, should default to system
     if (data->m_language_id == QLocale::C && language != QLocale::C) {
-        d->m_numberOptions = default_number_options;
-        d->m_index = localeDataIndex(defaultData());
+        numberOptions = default_number_options;
+        index = localeDataIndex(defaultData());
     } else {
-        d->m_numberOptions = 0;
-        d->m_index = localeDataIndex(data);
+        index = localeDataIndex(data);
     }
-    d->m_data = dataPointerHelper(d->m_index);
+    d = new QLocalePrivate(index, numberOptions);
 }
 
 /*!
@@ -897,10 +887,6 @@ void QLocale::setDefault(const QLocale &locale)
 {
     default_data = locale.d->m_data;
     default_number_options = locale.numberOptions();
-
-#ifdef QT_USE_ICU
-    qt_initIcu(locale.bcp47Name());
-#endif
 }
 
 /*!
@@ -1802,10 +1788,7 @@ QString QLocale::toString(double i, char f, int prec) const
 
 QLocale QLocale::system()
 {
-    QLocale result(C);
-    result.d->m_index = localeDataIndex(systemData());
-    result.d->m_data = dataPointerHelper(result.d->m_index);
-    return result;
+    return QLocale(*new QLocalePrivate(localeDataIndex(systemData())));
 }
 
 
@@ -1834,9 +1817,7 @@ QList<QLocale> QLocale::matchingLocales(QLocale::Language language,
         data += locale_index[language];
     while ( (data != locale_data + locale_data_size)
             && (language == QLocale::AnyLanguage || data->m_language_id == uint(language))) {
-        QLocale locale(QLocale::C);
-        locale.d->m_index = localeDataIndex(data);
-        locale.d->m_data = dataPointerHelper(locale.d->m_index);
+        QLocale locale(*new QLocalePrivate(localeDataIndex(data)));
         result.append(locale);
         ++data;
     }
@@ -2164,12 +2145,11 @@ Qt::LayoutDirection QLocale::textDirection() const
 QString QLocale::toUpper(const QString &str) const
 {
 #ifdef QT_USE_ICU
-    {
-        QString result;
-        if (qt_u_strToUpper(str, &result, *this))
-            return result;
-        // else fall through and use Qt's toUpper
-    }
+    bool ok = true;
+    QString result = QIcu::toUpper(d->m_localeID, str, &ok);
+    if (ok)
+        return result;
+    // else fall through and use Qt's toUpper
 #endif
     return str.toUpper();
 }
@@ -2182,12 +2162,11 @@ QString QLocale::toUpper(const QString &str) const
 QString QLocale::toLower(const QString &str) const
 {
 #ifdef QT_USE_ICU
-    {
-        QString result;
-        if (qt_u_strToLower(str, &result, *this))
-            return result;
-        // else fall through and use Qt's toUpper
-    }
+    bool ok = true;
+    QString result = QIcu::toLower(d->m_localeID, str, &ok);
+    if (ok)
+        return result;
+    // else fall through and use Qt's toUpper
 #endif
     return str.toLower();
 }
diff --git a/src/corelib/tools/qlocale.h b/src/corelib/tools/qlocale.h
index a014b6b7b5..c901211af4 100644
--- a/src/corelib/tools/qlocale.h
+++ b/src/corelib/tools/qlocale.h
@@ -703,6 +703,7 @@ public:
     QString createSeparatedList(const QStringList &strl) const;
 
 private:
+    QLocale(QLocalePrivate &dd);
     friend class QLocalePrivate;
     QSharedDataPointer<QLocalePrivate> d;
 };
diff --git a/src/corelib/tools/qlocale_icu.cpp b/src/corelib/tools/qlocale_icu.cpp
index 42651aa745..97ce82108b 100644
--- a/src/corelib/tools/qlocale_icu.cpp
+++ b/src/corelib/tools/qlocale_icu.cpp
@@ -42,199 +42,56 @@
 #include "qglobal.h"
 #include "qlibrary.h"
 #include "qdebug.h"
+#include "qlocale_p.h"
+#include "qmutex.h"
 
 #include "unicode/uversion.h"
 #include "unicode/ucol.h"
+#include "unicode/uloc.h"
+#include "unicode/ustring.h"
 
 QT_BEGIN_NAMESPACE
 
-typedef UCollator *(*Ptr_ucol_open)(const char *loc, UErrorCode *status);
-typedef void (*Ptr_ucol_close)(UCollator *coll);
-typedef UCollationResult (*Ptr_ucol_strcoll)(const UCollator *coll, const UChar *source, int32_t sourceLength, const UChar *target, int32_t targetLength);
 typedef int32_t (*Ptr_u_strToCase)(UChar *dest, int32_t destCapacity, const UChar *src, int32_t srcLength, const char *locale, UErrorCode *pErrorCode);
 
-static Ptr_ucol_open ptr_ucol_open = 0;
-static Ptr_ucol_strcoll ptr_ucol_strcoll = 0;
-static Ptr_ucol_close ptr_ucol_close = 0;
-static Ptr_u_strToCase ptr_u_strToUpper = 0;
-static Ptr_u_strToCase ptr_u_strToLower = 0;
-
-enum LibLoadStatus
-{
-    ErrorLoading = -1,
-    NotLoaded = 0,
-    Loaded = 1
-};
-
-static LibLoadStatus status = NotLoaded;
-
-static UCollator *icuCollator = 0;
-
-namespace {
-struct Libraries {
-    QLibrary libicui18n;
-    QLibrary libicuuc;
-    ~Libraries()
-    {
-        if (icuCollator) {
-            ptr_ucol_close(icuCollator);
-            icuCollator = 0;
-        }
-
-        libicui18n.unload();
-        libicuuc.unload();
-    }
-};
-}
-Q_GLOBAL_STATIC(Libraries, icuLibraries)
-
-static bool loadIcuLibrary(QLibrary &lib, const QString &name)
-{
-#ifdef Q_OS_WIN
-    // QLibrary on Windows does not use the version number, the libraries
-    // are named "icuin<version>.dll", though.
-    lib.setFileName(name);
-#else
-    // on Unix, we can use the version number
-    lib.setFileNameAndVersion(name, QStringLiteral(U_ICU_VERSION_SHORT));
-#endif
-
-    // the ICU libraries appear to allocate global statics and not free them
-    // set the PreventUnloadHint so that we can unload the QLibrary object and
-    // delete it, but the libraries themselves remain in memory
-    lib.setLoadHints(QLibrary::PreventUnloadHint);
-    return lib.load();
-}
-
-// this function is NOT THREAD-SAFE!
-bool qt_initIcu(const QString &localeString)
-{
-    if (status == ErrorLoading || !icuLibraries())
-        return false;
-
-    if (status == NotLoaded) {
-
-        // resolve libicui18n
-#ifdef Q_OS_WIN
-        // QLibrary on Windows does not use the version number, the libraries
-        // are named "icuin<version>.dll", though.
-        // QStringLiteral should work here and will work when MSVC fully supports C++11
-        // unfortunately, current versions have do not support proper string concatenation
-        QString libicui18nName = QLatin1String("icuin" U_ICU_VERSION_SHORT);
-        QString libicuucName = QLatin1String("icuuc" U_ICU_VERSION_SHORT);
-#else
-        QString libicui18nName = QStringLiteral("icui18n");
-        QString libicuucName = QStringLiteral("icuuc");
-#endif
-        QLibrary &lib = icuLibraries()->libicui18n;
-        if (!loadIcuLibrary(lib, libicui18nName)) {
-            qWarning("Unable to load library '%s' version " U_ICU_VERSION_SHORT ": %s",
-                     qPrintable(libicui18nName),
-                     qPrintable(lib.errorString()));
-            status = ErrorLoading;
-            return false;
-        }
-
-        ptr_ucol_open = (Ptr_ucol_open)lib.resolve("ucol_open");
-        ptr_ucol_close = (Ptr_ucol_close)lib.resolve("ucol_close");
-        ptr_ucol_strcoll = (Ptr_ucol_strcoll)lib.resolve("ucol_strcoll");
-
-        if (!ptr_ucol_open || !ptr_ucol_close || !ptr_ucol_strcoll) {
-            // try again with decorated symbol names
-            ptr_ucol_open = (Ptr_ucol_open)lib.resolve("ucol_open" QT_STRINGIFY(U_ICU_VERSION_SUFFIX));
-            ptr_ucol_close = (Ptr_ucol_close)lib.resolve("ucol_close" QT_STRINGIFY(U_ICU_VERSION_SUFFIX));
-            ptr_ucol_strcoll = (Ptr_ucol_strcoll)lib.resolve("ucol_strcoll" QT_STRINGIFY(U_ICU_VERSION_SUFFIX));
-        }
-
-        if (!ptr_ucol_open || !ptr_ucol_close || !ptr_ucol_strcoll) {
-            ptr_ucol_open = 0;
-            ptr_ucol_close = 0;
-            ptr_ucol_strcoll = 0;
-
-            qWarning("Unable to find symbols in '%s'.", qPrintable(libicui18nName));
-            status = ErrorLoading;
-            return false;
-        }
-
-        // resolve libicuuc
-        QLibrary &ucLib = icuLibraries()->libicuuc;
-        if (!loadIcuLibrary(ucLib, libicuucName)) {
-            qWarning("Unable to load library '%s' version " U_ICU_VERSION_SHORT ": %s",
-                     qPrintable(libicuucName),
-                     qPrintable(ucLib.errorString()));
-            status = ErrorLoading;
-            return false;
-        }
-
-        ptr_u_strToUpper = (Ptr_u_strToCase)ucLib.resolve("u_strToUpper");
-        ptr_u_strToLower = (Ptr_u_strToCase)ucLib.resolve("u_strToLower");
-
-        if (!ptr_u_strToUpper || !ptr_u_strToLower) {
-            ptr_u_strToUpper = (Ptr_u_strToCase)ucLib.resolve("u_strToUpper" QT_STRINGIFY(U_ICU_VERSION_SUFFIX));
-            ptr_u_strToLower = (Ptr_u_strToCase)ucLib.resolve("u_strToLower" QT_STRINGIFY(U_ICU_VERSION_SUFFIX));
-        }
-
-        if (!ptr_u_strToUpper || !ptr_u_strToLower) {
-            ptr_u_strToUpper = 0;
-            ptr_u_strToLower = 0;
-
-            qWarning("Unable to find symbols in '%s'", qPrintable(libicuucName));
-            status = ErrorLoading;
-            return false;
-        }
-
-        // success :)
-        status = Loaded;
-    }
-
-    if (icuCollator) {
-        ptr_ucol_close(icuCollator);
-        icuCollator = 0;
-    }
-
-    UErrorCode icuStatus = U_ZERO_ERROR;
-    icuCollator = ptr_ucol_open(localeString.toLatin1().constData(), &icuStatus);
-
-    if (!icuCollator) {
-        qWarning("Unable to open locale %s in ICU, error code %d", qPrintable(localeString), icuStatus);
-        return false;
-    }
-
-    return true;
-}
-
-bool qt_ucol_strcoll(const QChar *source, int sourceLength, const QChar *target, int targetLength, int *result)
+bool QIcu::strcoll(const QByteArray &localeID,
+                   const QChar *source, int sourceLength, const QChar *target, int targetLength, int *result)
 {
     Q_ASSERT(result);
     Q_ASSERT(source);
     Q_ASSERT(target);
 
-    if (!icuCollator)
+    UErrorCode icuStatus = U_ZERO_ERROR;
+    UCollator *collator = ucol_open(localeID, &icuStatus);
+
+    if (U_FAILURE((icuStatus)))
         return false;
 
-    *result = ptr_ucol_strcoll(icuCollator, reinterpret_cast<const UChar *>(source), int32_t(sourceLength),
-                               reinterpret_cast<const UChar *>(target), int32_t(targetLength));
+    *result = ucol_strcoll(collator,
+                           reinterpret_cast<const UChar *>(source), int32_t(sourceLength),
+                           reinterpret_cast<const UChar *>(target), int32_t(targetLength));
+
+    ucol_close(collator);
 
     return true;
 }
 
 // caseFunc can either be u_strToUpper or u_strToLower
-static bool qt_u_strToCase(const QString &str, QString *out, const QLocale &locale, Ptr_u_strToCase caseFunc)
+static bool qt_u_strToCase(const QString &str, QString *out, const char *localeID, Ptr_u_strToCase caseFunc)
 {
     Q_ASSERT(out);
 
-    if (!icuCollator)
-        return false;
-
-    QString result(str.size(), Qt::Uninitialized);
+    int32_t size = str.size();
+    size += size >> 2; // add 25% for possible expansions
+    QString result(size, Qt::Uninitialized);
 
     UErrorCode status = U_ZERO_ERROR;
 
-    int32_t size = caseFunc(reinterpret_cast<UChar *>(result.data()), result.size(),
+    size = caseFunc(reinterpret_cast<UChar *>(result.data()), result.size(),
             reinterpret_cast<const UChar *>(str.constData()), str.size(),
-            locale.bcp47Name().toLatin1().constData(), &status);
+            localeID, &status);
 
-    if (U_FAILURE(status))
+    if (U_FAILURE(status) && status != U_BUFFER_OVERFLOW_ERROR)
         return false;
 
     if (size < result.size()) {
@@ -246,7 +103,7 @@ static bool qt_u_strToCase(const QString &str, QString *out, const QLocale &loca
         status = U_ZERO_ERROR;
         size = caseFunc(reinterpret_cast<UChar *>(result.data()), result.size(),
             reinterpret_cast<const UChar *>(str.constData()), str.size(),
-            locale.bcp47Name().toLatin1().constData(), &status);
+            localeID, &status);
 
         if (U_FAILURE(status))
             return false;
@@ -260,14 +117,22 @@ static bool qt_u_strToCase(const QString &str, QString *out, const QLocale &loca
     return true;
 }
 
-bool qt_u_strToUpper(const QString &str, QString *out, const QLocale &locale)
+QString QIcu::toUpper(const QByteArray &localeID, const QString &str, bool *ok)
 {
-    return qt_u_strToCase(str, out, locale, ptr_u_strToUpper);
+    QString out;
+    bool err = qt_u_strToCase(str, &out, localeID, u_strToUpper);
+    if (ok)
+        *ok = err;
+    return out;
 }
 
-bool qt_u_strToLower(const QString &str, QString *out, const QLocale &locale)
+QString QIcu::toLower(const QByteArray &localeID, const QString &str, bool *ok)
 {
-    return qt_u_strToCase(str, out, locale, ptr_u_strToLower);
+    QString out;
+    bool err = qt_u_strToCase(str, &out, localeID, u_strToLower);
+    if (ok)
+        *ok = err;
+    return out;
 }
 
 QT_END_NAMESPACE
diff --git a/src/corelib/tools/qlocale_p.h b/src/corelib/tools/qlocale_p.h
index c2c1488966..1902e4e5af 100644
--- a/src/corelib/tools/qlocale_p.h
+++ b/src/corelib/tools/qlocale_p.h
@@ -126,6 +126,16 @@ private:
 };
 #endif
 
+#ifdef QT_USE_ICU
+namespace QIcu {
+    QString toUpper(const QByteArray &localeId, const QString &str, bool *ok);
+    QString toLower(const QByteArray &localeId, const QString &str, bool *ok);
+
+    bool strcoll(const QByteArray &localeID, const QChar *source, int sourceLength, const QChar *target, int targetLength, int *result);
+}
+#endif
+
+
 struct QLocaleData
 {
 public:
@@ -178,7 +188,17 @@ public:
 class Q_CORE_EXPORT QLocalePrivate : public QSharedData
 {
 public:
-    QLocalePrivate() : m_index(0), m_numberOptions(0), m_data(0) {}
+    QLocalePrivate(int index, int numberOptions = 0)
+        : m_index(index), m_numberOptions(numberOptions)
+    {
+        m_data = dataPointerForIndex(index);
+        m_localeID = bcp47Name().toLatin1();
+        m_localeID.replace('-','_');
+    }
+
+    ~QLocalePrivate()
+    {
+    }
 
     QChar decimal() const { return QChar(m_data->m_decimal); }
     QChar group() const { return QChar(m_data->m_group); }
@@ -203,6 +223,7 @@ public:
     static QLocale::Country codeToCountry(const QString &code);
     static void getLangAndCountry(const QString &name, QLocale::Language &lang,
                                   QLocale::Script &script, QLocale::Country &cntry);
+    static const QLocaleData *dataPointerForIndex(quint16 index);
 
     QLocale::MeasurementSystem measurementSystem() const;
 
@@ -285,11 +306,11 @@ public:
     QString dateTimeToString(const QString &format, const QDate *date, const QTime *time,
                              const QLocale *q) const;
 
-private:
     friend class QLocale;
     quint16 m_index;
     quint16 m_numberOptions;
     const QLocaleData *m_data;
+    QByteArray m_localeID;
 };
 
 inline char QLocalePrivate::digitToCLocale(QChar in) const
diff --git a/src/corelib/tools/qstring.cpp b/src/corelib/tools/qstring.cpp
index a2d91c561c..4f8d0e5614 100644
--- a/src/corelib/tools/qstring.cpp
+++ b/src/corelib/tools/qstring.cpp
@@ -100,12 +100,6 @@
 
 QT_BEGIN_NAMESPACE
 
-#ifdef QT_USE_ICU
-// qlocale_icu.cpp
-extern bool qt_ucol_strcoll(const QChar *source, int sourceLength, const QChar *target, int targetLength, int *result);
-#endif
-
-
 // internal
 int qFindString(const QChar *haystack, int haystackLen, int from,
     const QChar *needle, int needleLen, Qt::CaseSensitivity cs);
@@ -5011,12 +5005,11 @@ int QString::localeAwareCompare_helper(const QChar *data1, int length1,
     return result;
 #elif defined(Q_OS_UNIX)
 #  if defined(QT_USE_ICU)
+    QLocale locale;
     int res;
-    if (qt_ucol_strcoll(data1, length1, data2, length2, &res)) {
-        if (res == 0)
-            res = ucstrcmp(data1, length1, data2, length2);
+    if (QIcu::strcoll(locale.d.constData()->m_localeID, data1, length1, data2, length2, &res))
         return res;
-    } // else fall through
+    // else fall through
 #  endif
     // declared in <string.h>
     int delta = strcoll(toLocal8Bit_helper(data1, length1).constData(), toLocal8Bit_helper(data2, length2).constData());
diff --git a/src/corelib/tools/tools.pri b/src/corelib/tools/tools.pri
index 386db9665b..9db459ab68 100644
--- a/src/corelib/tools/tools.pri
+++ b/src/corelib/tools/tools.pri
@@ -112,6 +112,8 @@ else:include($$PWD/../../3rdparty/zlib_dependency.pri)
 contains(QT_CONFIG,icu) {
     SOURCES += tools/qlocale_icu.cpp
     DEFINES += QT_USE_ICU
+    win32:LIBS += -licuin -licuuc
+    else:LIBS += -licui18n -licuuc
 }
 
 pcre {