From 419f42903154489e2dace8d1eba183036275448b Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Fri, 22 Nov 2019 14:05:25 +0100 Subject: QCoreApplication: force the process locale codec to UTF-8 As discussed in the mailing list and in the Qt Contributor Summit 2019. Tested on Linux, macOS and FreeBSD, showing these fallbacks: OS environment setlocale() call FreeBSD empty "C.UTF-8" FreeBSD LC_ALL=C "C.UTF-8" Linux empty "C.UTF-8" Linux LC_ALL=C "C.UTF-8" Linux LANG=en_US "en_US.UTF-8" Linux LANG=de_DE@euro "de_DE.UTF-8" Linux LANG=en_GB.iso885915 "en_GB.UTF-8" Linux LANG=hy_AM.armscii8 "hy_AM.UTF-8" Linux LANG=ja_JP.sjis "ja_JP.UTF-8" Linux LANG=ru_RU.koi8r "ru_RU.UTF-8" macOS empty "UTF-8" macOS LC_ALL=C "UTF-8" Versions tested: FreeBSD 12.1, Linux w/ glibc 2.30, macOS 10.14.2. See * https://wiki.qt.io/Qt_Contributor_Summit_2019_-_QtCore * https://lists.qt-project.org/pipermail/development/2019-October/037791.html Change-Id: Ia2aa807ffa8a4c798425fffd15d97ddb4f35b0ae Reviewed-by: Thiago Macieira Reviewed-by: Lars Knoll --- src/corelib/kernel/qcoreapplication.cpp | 42 +++++++++++++++++++--- .../testlib/selftests/generate_expected_output.py | 4 +-- tests/auto/testlib/selftests/tst_selftests.cpp | 5 +++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/corelib/kernel/qcoreapplication.cpp b/src/corelib/kernel/qcoreapplication.cpp index fe3aa454ab..0c5e6cdb4a 100644 --- a/src/corelib/kernel/qcoreapplication.cpp +++ b/src/corelib/kernel/qcoreapplication.cpp @@ -112,6 +112,7 @@ #ifdef Q_OS_UNIX # include +# include # include # include #endif @@ -338,9 +339,6 @@ void Q_CORE_EXPORT qt_call_post_routines() } -// initialized in qcoreapplication and in qtextstream autotest when setlocale is called. -static bool qt_locale_initialized = false; - #ifndef QT_NO_QOBJECT // app starting up if false @@ -590,11 +588,45 @@ QString qAppName() void QCoreApplicationPrivate::initLocale() { +#if defined(Q_OS_UNIX) && !defined(QT_BOOTSTRAPPED) + static bool qt_locale_initialized = false; if (qt_locale_initialized) return; qt_locale_initialized = true; -#if defined(Q_OS_UNIX) && !defined(QT_BOOTSTRAPPED) - setlocale(LC_ALL, ""); + +#ifdef Q_OS_ANDROID + // Android's Bionic didn't get nl_langinfo until NDK 15 (Android 8.0), + // which is too new for Qt, so we just assume it's always UTF-8. + auto nl_langinfo = [](int) { return "UTF-8"; }; +#endif + + const char *locale = setlocale(LC_ALL, ""); + const char *codec = nl_langinfo(CODESET); + if (Q_UNLIKELY(strcmp(codec, "UTF-8") != 0 && strcmp(codec, "utf8") != 0)) { + QByteArray oldLocale = locale; + QByteArray newLocale = setlocale(LC_CTYPE, nullptr); + if (int dot = newLocale.indexOf('.'); dot != -1) + newLocale.truncate(dot); // remove encoding, if any + if (int at = newLocale.indexOf('@'); at != -1) + newLocale.truncate(at); // remove variant, as the old de_DE@euro + newLocale += ".UTF-8"; + newLocale = setlocale(LC_CTYPE, newLocale); + + // if locale doesn't exist, try some fallbacks +# ifdef Q_OS_DARWIN + if (newLocale.isEmpty()) + newLocale = setlocale(LC_CTYPE, "UTF-8"); +# endif + if (newLocale.isEmpty()) + newLocale = setlocale(LC_CTYPE, "C.UTF-8"); + if (newLocale.isEmpty()) + newLocale = setlocale(LC_CTYPE, "C.utf8"); + + qWarning("Detected system locale encoding (%s, locale \"%s\") is not UTF-8.\n" + "Qt shall use a UTF-8 locale (\"%s\") instead. If this causes problems,\n" + "reconfigure your locale. See the locale(1) manual for more information.", + codec, oldLocale.constData(), newLocale.constData()); + } #endif } diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py index 96fab44408..673c1ab3bd 100755 --- a/tests/auto/testlib/selftests/generate_expected_output.py +++ b/tests/auto/testlib/selftests/generate_expected_output.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ############################################################################# ## -## Copyright (C) 2016 The Qt Company Ltd. +## Copyright (C) 2020 The Qt Company Ltd. ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the release tools of the Qt Toolkit. @@ -263,7 +263,7 @@ def baseEnv(platname=None, keep += preserveLib cached = dict( - LC_ALL = 'C', # Use standard locale + LC_ALL = 'en-US.UTF-8', # Use standard locale # Avoid interference from any qtlogging.ini files, e.g. in # /etc/xdg/QtProject/, (must match tst_selftests.cpp's # processEnvironment()'s value): diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 84090c22ca..53c35dad65 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -632,6 +632,11 @@ static QProcessEnvironment processEnvironment() result.insert(QStringLiteral("QT_LOGGING_RULES"), // Must match generate_expected_output.py's main()'s value: QStringLiteral("*.debug=true;qt.*=false")); + +#if defined(Q_OS_UNIX) + // avoid the warning from QCoreApplication + result.insert(QStringLiteral("LC_ALL"), QStringLiteral("en_US.UTF-8")); +#endif } return result; } -- cgit v1.2.3