diff options
Diffstat (limited to 'util')
| -rw-r--r-- | util/corelib/qurl-generateTLDs/main.cpp | 163 | ||||
| -rw-r--r-- | util/glgen/codegenerator.cpp | 2 | ||||
| -rw-r--r-- | util/glgen/legacyspecparser.cpp | 2 | ||||
| -rw-r--r-- | util/lexgen/generator.cpp | 6 | ||||
| -rw-r--r-- | util/lexgen/nfa.cpp | 2 | ||||
| -rw-r--r-- | util/local_database/README | 1 | ||||
| -rwxr-xr-x | util/local_database/cldr2qtimezone.py | 431 | ||||
| -rw-r--r-- | util/locale_database/README | 5 | ||||
| -rwxr-xr-x | util/locale_database/cldr2qlocalexml.py (renamed from util/local_database/cldr2qlocalexml.py) | 56 | ||||
| -rwxr-xr-x | util/locale_database/cldr2qtimezone.py | 457 | ||||
| -rwxr-xr-x | util/locale_database/dateconverter.py (renamed from util/local_database/dateconverter.py) | 0 | ||||
| -rw-r--r-- | util/locale_database/enumdata.py (renamed from util/local_database/enumdata.py) | 2 | ||||
| -rw-r--r-- | util/locale_database/formattags.txt (renamed from util/local_database/formattags.txt) | 0 | ||||
| -rw-r--r-- | util/locale_database/localexml.py (renamed from util/local_database/localexml.py) | 104 | ||||
| -rwxr-xr-x | util/locale_database/qlocalexml2cpp.py (renamed from util/local_database/qlocalexml2cpp.py) | 140 | ||||
| -rw-r--r-- | util/locale_database/testlocales/localemodel.cpp (renamed from util/local_database/testlocales/localemodel.cpp) | 0 | ||||
| -rw-r--r-- | util/locale_database/testlocales/localemodel.h (renamed from util/local_database/testlocales/localemodel.h) | 0 | ||||
| -rw-r--r-- | util/locale_database/testlocales/localewidget.cpp (renamed from util/local_database/testlocales/localewidget.cpp) | 0 | ||||
| -rw-r--r-- | util/locale_database/testlocales/localewidget.h (renamed from util/local_database/testlocales/localewidget.h) | 0 | ||||
| -rw-r--r-- | util/locale_database/testlocales/main.cpp (renamed from util/local_database/testlocales/main.cpp) | 0 | ||||
| -rw-r--r-- | util/locale_database/testlocales/testlocales.pro (renamed from util/local_database/testlocales/testlocales.pro) | 0 | ||||
| -rw-r--r-- | util/locale_database/xpathlite.py (renamed from util/local_database/xpathlite.py) | 28 | ||||
| -rw-r--r-- | util/qfloat16-tables/gen_qfloat16_tables.cpp | 163 | ||||
| -rw-r--r-- | util/qfloat16-tables/qfloat16-tables.pro | 2 | ||||
| -rw-r--r-- | util/unicode/README | 2 | ||||
| -rw-r--r-- | util/unicode/codecs/big5/main.cpp | 4 | ||||
| -rw-r--r-- | util/unicode/main.cpp | 146 |
27 files changed, 1038 insertions, 678 deletions
diff --git a/util/corelib/qurl-generateTLDs/main.cpp b/util/corelib/qurl-generateTLDs/main.cpp index e1fe97646f..e458ea9d53 100644 --- a/util/corelib/qurl-generateTLDs/main.cpp +++ b/util/corelib/qurl-generateTLDs/main.cpp @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2019 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the utils of the Qt Toolkit. @@ -28,33 +28,66 @@ #include <QtCore> +const QString quadQuote = QStringLiteral("\"\""); // Closes one string, opens a new one. + static QString utf8encode(const QByteArray &array) // turns e.g. tranøy.no to tran\xc3\xb8y.no { QString result; result.reserve(array.length() + array.length() / 3); + bool wasHex = false; for (int i = 0; i < array.length(); ++i) { char c = array.at(i); // if char is non-ascii, escape it if (c < 0x20 || uchar(c) >= 0x7f) { result += "\\x" + QString::number(uchar(c), 16); + wasHex = true; } else { // if previous char was escaped, we need to make sure the next char is not // interpreted as part of the hex value, e.g. "äc.com" -> "\xabc.com"; this // should be "\xab""c.com" - QRegExp hexEscape("\\\\x[a-fA-F0-9][a-fA-F0-9]$"); bool isHexChar = ((c >= '0' && c <= '9') || - (c >= 'a' && c <= 'f') || - (c >= 'A' && c <= 'F')); - if (result.contains(hexEscape) && isHexChar) - result += "\"\""; + (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F')); + if (wasHex && isHexChar) + result += quadQuote; result += c; + wasHex = false; } } return result; } -int main(int argc, char **argv) { +/* + Digest public suffix data into efficiently-searchable form. + + Takes the public suffix list (see usage message), a list of DNS domains + whose child domains should not be presumed to trust one another, and + converts it to a form that lets qtbase/src/corelib/io/qtldurl.cpp's query + functions find entries efficiently. + + Each line of the suffix file (aside from comments and blanks) gives a suffix + (starting with a dot) with an optional prefix of '*' (to include every + immediate child) or of '!' (to exclude the suffix, e.g. from a '*' line for + a tail of it). A line with neither of these prefixes is an exact match. + + Each line is hashed and the hash is reduced modulo the number of lines + (tldCount); lines are grouped by reduced hash and separated by '\0' bytes + within each group. Conceptually, the groups are then emitted to a single + huge string, along with a table (tldIndices[tldCount]) of indices into that + string of the starts of the the various groups. + + However, that huge string would exceed the 64k limit at least one compiler + imposes on a single string literal, so we actually split up the huge string + into an array of chunks, each less than 64k in size. Each group is written + to a single chunk (so we start a new chunk if the next group would take the + present chunk over the limit). There are tldChunkCount chunks; their lengths + are saved in tldChunks[tldChunkCount]; the chunks themselves in + tldData[tldChunkCount]. See qtldurl.cpp's containsTLDEntry() for how to + search for a string in the resulting data. +*/ +int main(int argc, char **argv) +{ QCoreApplication app(argc, argv); if (argc < 3) { printf("\nUsage: ./%s inputFile outputFile\n\n", argv[0]); @@ -68,14 +101,20 @@ int main(int argc, char **argv) { return 1; } QFile file(argv[1]); - QFile outFile(argv[2]); - file.open(QIODevice::ReadOnly); - outFile.open(QIODevice::WriteOnly); + if (!file.open(QIODevice::ReadOnly)) { + fprintf("Failed to open input file (%s); see %s -usage", argv[1], argv[0]); + return 1; + } - QByteArray outIndicesBufferBA; - QBuffer outIndicesBuffer(&outIndicesBufferBA); - outIndicesBuffer.open(QIODevice::WriteOnly); + QFile outFile(argv[2]); + if (!outFile.open(QIODevice::WriteOnly)) { + file.close() + fprintf("Failed to open output file (%s); see %s -usage", argv[2], argv[0]); + return 1; + } + // Write tldData[] and tldIndices[] in one scan of the (input) file, but + // buffer tldData[] so we don'te interleave them in the outFile. QByteArray outDataBufferBA; QBuffer outDataBuffer(&outDataBufferBA); outDataBuffer.open(QIODevice::WriteOnly); @@ -85,75 +124,65 @@ int main(int argc, char **argv) { file.readLine(); lineCount++; } + outFile.write("static const quint16 tldCount = "); + outFile.write(QByteArray::number(lineCount)); + outFile.write(";\n"); + file.reset(); QVector<QString> strings(lineCount); while (!file.atEnd()) { - QString s = QString::fromUtf8(file.readLine()); - QString st = s.trimmed(); + QString st = QString::fromUtf8(file.readLine()).trimmed(); int num = qt_hash(st) % lineCount; + QString &entry = strings[num]; + st = utf8encode(st.toUtf8()); - QString utf8String = utf8encode(st.toUtf8()); - - // for domain 1.com, we could get something like - // a.com\01.com, which would be interpreted as octal 01, - // so we need to separate those strings with quotes - QRegExp regexpOctalEscape(QLatin1String("^[0-9]")); - if (!strings.at(num).isEmpty() && st.contains(regexpOctalEscape)) - strings[num].append("\"\""); + // For domain 1.com, we could get something like a.com\01.com, which + // would be misinterpreted as octal 01, so we need to separate such + // strings with quotes: + if (!entry.isEmpty() && st.at(0).isDigit()) + entry.append(quadQuote); - strings[num].append(utf8String); - strings[num].append("\\0"); + entry.append(st); + entry.append("\\0"); } - - outIndicesBuffer.write("static const quint16 tldCount = "); - outIndicesBuffer.write(QByteArray::number(lineCount)); - outIndicesBuffer.write(";\n"); - outIndicesBuffer.write("static const quint32 tldIndices["); -// outIndicesBuffer.write(QByteArray::number(lineCount+1)); // not needed - outIndicesBuffer.write("] = {\n"); + outFile.write("static const quint32 tldIndices[] = {\n"); + outDataBuffer.write("\nstatic const char *tldData[] = {"); int totalUtf8Size = 0; int chunkSize = 0; // strlen of the current chunk (sizeof is bigger by 1) - int stringUtf8Size = 0; QStringList chunks; for (int a = 0; a < lineCount; a++) { - bool lineIsEmpty = strings.at(a).isEmpty(); - if (!lineIsEmpty) { - strings[a].prepend("\""); - strings[a].append("\""); - } - int zeroCount = strings.at(a).count(QLatin1String("\\0")); - int utf8CharsCount = strings.at(a).count(QLatin1String("\\x")); - int quoteCount = strings.at(a).count('"'); - stringUtf8Size = strings.at(a).count() - (zeroCount + quoteCount + utf8CharsCount * 3); - chunkSize += stringUtf8Size; - // MSVC 2015 chokes if sizeof(a single string) > 0xffff - if (chunkSize >= 0xffff) { - static int chunkCount = 0; - qWarning() << "chunk" << ++chunkCount << "has length" << chunkSize - stringUtf8Size; - outDataBuffer.write(",\n\n"); - chunks.append(QByteArray::number(totalUtf8Size)); - chunkSize = 0; + outFile.write(QByteArray::number(totalUtf8Size)); + outFile.write(",\n"); + const QString &entry = strings.at(a); + if (!entry.isEmpty()) { + const int zeroCount = entry.count(QLatin1String("\\0")); + const int utf8CharsCount = entry.count(QLatin1String("\\x")); + const int quoteCount = entry.count('"'); + const int stringUtf8Size = entry.count() - (zeroCount + quoteCount + utf8CharsCount * 3); + chunkSize += stringUtf8Size; + // MSVC 2015 chokes if sizeof(a single string) > 0xffff + if (chunkSize >= 0xffff) { + static int chunkCount = 0; + qWarning() << "chunk" << ++chunkCount << "has length" << chunkSize - stringUtf8Size; + outDataBuffer.write(",\n"); + chunks.append(QString::number(totalUtf8Size)); + chunkSize = 0; + } + totalUtf8Size += stringUtf8Size; + + outDataBuffer.write("\n\""); + outDataBuffer.write(entry.toUtf8()); + outDataBuffer.write("\""); } - outDataBuffer.write(strings.at(a).toUtf8()); - if (!lineIsEmpty) - outDataBuffer.write("\n"); - outIndicesBuffer.write(QByteArray::number(totalUtf8Size)); - outIndicesBuffer.write(",\n"); - totalUtf8Size += stringUtf8Size; } - chunks.append(QByteArray::number(totalUtf8Size)); - outIndicesBuffer.write(QByteArray::number(totalUtf8Size)); - outIndicesBuffer.write("};\n"); - outIndicesBuffer.close(); - outFile.write(outIndicesBufferBA); + chunks.append(QString::number(totalUtf8Size)); + outFile.write(QByteArray::number(totalUtf8Size)); + outFile.write("\n};\n"); + outDataBuffer.write("\n};\n"); outDataBuffer.close(); - outFile.write("\nstatic const char *tldData["); -// outFile.write(QByteArray::number(charSize)); // not needed - outFile.write("] = {\n"); outFile.write(outDataBufferBA); - outFile.write("};\n"); // write chunk information outFile.write("\nstatic const quint16 tldChunkCount = "); @@ -162,6 +191,6 @@ int main(int argc, char **argv) { outFile.write(chunks.join(", ").toLatin1()); outFile.write("};\n"); outFile.close(); - printf("data generated to %s . Now copy the data from this file to src/corelib/io/qurltlds_p.h in your Qt repo\n", argv[2]); - exit(0); + printf("Data generated to %s - now revise qtbase/src/corelib/io/qurltlds_p.h to use this data.\n", argv[2]); + return 0; } diff --git a/util/glgen/codegenerator.cpp b/util/glgen/codegenerator.cpp index 4627daa48b..08327a62f5 100644 --- a/util/glgen/codegenerator.cpp +++ b/util/glgen/codegenerator.cpp @@ -184,7 +184,7 @@ void CodeGenerator::writeCoreFactoryImplementation(const QString &fileName) cons // Get the set of version functions classes we need to create QList<Version> versions = m_parser->versions(); - qSort(versions.begin(), versions.end(), qGreater<Version>()); + std::sort(m_versions.begin(), m_versions.end(), std::greater<Version>()); // Outout the #include statements stream << QStringLiteral("#if !defined(QT_OPENGL_ES_2)") << endl; diff --git a/util/glgen/legacyspecparser.cpp b/util/glgen/legacyspecparser.cpp index 0f4d085bba..ab2c9495e7 100644 --- a/util/glgen/legacyspecparser.cpp +++ b/util/glgen/legacyspecparser.cpp @@ -291,7 +291,7 @@ void LegacySpecParser::parseFunctions(QTextStream &stream) } m_versions = versions.toList(); - qSort(m_versions); + std::sort(m_versions.begin(), m_versions.end()); } bool LegacySpecParser::inDeprecationException(const QString &functionName) const diff --git a/util/lexgen/generator.cpp b/util/lexgen/generator.cpp index 481d586e73..3b966e025b 100644 --- a/util/lexgen/generator.cpp +++ b/util/lexgen/generator.cpp @@ -183,7 +183,7 @@ Generator::Generator(const DFA &_dfa, const Config &config) : dfa(_dfa), cfg(config) { QList<InputType> lst = cfg.maxInputSet.toList(); - qSort(lst); + std::sort(lst.begin(), lst.end()); minInput = lst.first(); maxInput = lst.last(); @@ -230,7 +230,7 @@ static QVector<Generator::TransitionSequence> convertToSequences(const Transitio return sequences; QList<InputType> keys = transitions.keys(); - qSort(keys); + std::sort(keys.begin(), keys.end()); int i = 0; Generator::TransitionSequence sequence; sequence.first = keys.at(0); @@ -359,7 +359,7 @@ void Generator::generateTransitions(CodeBlock &body, const TransitionMap &transi } } else { QList<InputType> keys = transitions.keys(); - qSort(keys); + std::sort(keys.begin(), keys.end()); body << "switch (ch.unicode()) {"; body.indent(); diff --git a/util/lexgen/nfa.cpp b/util/lexgen/nfa.cpp index 9047a17fde..f6e3a5c355 100644 --- a/util/lexgen/nfa.cpp +++ b/util/lexgen/nfa.cpp @@ -384,7 +384,7 @@ QSet<int> NFA::epsilonClosure(const QSet<int> &initialClosure) const QStack<int> stateStack; stateStack.resize(closure.count()); - qCopy(closure.constBegin(), closure.constEnd(), stateStack.begin()); + std::copy(closure.constBegin(), closure.constEnd(), stateStack.begin()); while (!stateStack.isEmpty()) { int t = stateStack.pop(); diff --git a/util/local_database/README b/util/local_database/README deleted file mode 100644 index 23b6a33ad8..0000000000 --- a/util/local_database/README +++ /dev/null @@ -1 +0,0 @@ -local_database is used to generate qlocale data from the Common Locale Data Repository (The database for localized names (like date formats, country names etc)). diff --git a/util/local_database/cldr2qtimezone.py b/util/local_database/cldr2qtimezone.py deleted file mode 100755 index 7c10b1dfd2..0000000000 --- a/util/local_database/cldr2qtimezone.py +++ /dev/null @@ -1,431 +0,0 @@ -#!/usr/bin/env python2 -############################################################################# -## -## Copyright (C) 2016 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the test suite of the Qt Toolkit. -## -## $QT_BEGIN_LICENSE:GPL-EXCEPT$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3 as published by the Free Software -## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# -"""Parse CLDR data for QTimeZone use with MS-Windows - -Script to parse the CLDR supplemental/windowsZones.xml file and encode -for use in QTimeZone. See ``./cldr2qlocalexml.py`` for where to get -the CLDR data. Pass its common/ directory as first parameter to this -script and the qtbase root directory as second parameter. It shall -update qtbase's src/corelib/tools/qtimezoneprivate_data_p.h ready for -use. - -The XML structure is as follows: - - <supplementalData> - <version number="$Revision: 7825 $"/> - <generation date="$Date: 2012-10-10 14:45:31 -0700 (Wed, 10 Oct 2012) $"/> - <windowsZones> - <mapTimezones otherVersion="7dc0101" typeVersion="2012f"> - <!-- (UTC-08:00) Pacific Time (US & Canada) --> - <mapZone other="Pacific Standard Time" territory="001" type="America/Los_Angeles"/> - <mapZone other="Pacific Standard Time" territory="CA" type="America/Vancouver America/Dawson America/Whitehorse"/> - <mapZone other="Pacific Standard Time" territory="MX" type="America/Tijuana"/> - <mapZone other="Pacific Standard Time" territory="US" type="America/Los_Angeles"/> - <mapZone other="Pacific Standard Time" territory="ZZ" type="PST8PDT"/> - </mapTimezones> - </windowsZones> - </supplementalData> -""" - -import os -import sys -import datetime -import tempfile -import enumdata -import xpathlite -from xpathlite import DraftResolution -import re -import qlocalexml2cpp - -findAlias = xpathlite.findAlias -findEntry = xpathlite.findEntry -findEntryInFile = xpathlite._findEntryInFile -findTagsInFile = xpathlite.findTagsInFile -unicode2hex = qlocalexml2cpp.unicode2hex -wrap_list = qlocalexml2cpp.wrap_list - -class ByteArrayData: - def __init__(self): - self.data = [] - self.hash = {} - def append(self, s): - s = s + '\0' - if s in self.hash: - return self.hash[s] - - lst = unicode2hex(s) - index = len(self.data) - if index > 65535: - print "\n\n\n#error Data index is too big!" - sys.stderr.write ("\n\n\nERROR: index exceeds the uint16 range! index = %d\n" % index) - sys.exit(1) - self.hash[s] = index - self.data += lst - return index - -# List of currently known Windows IDs. If script fails on missing ID plase add it here -# Not public so may be safely changed. -# Windows Key : [ Windows Id, Offset Seconds ] -windowsIdList = { - 1 : [ u'Afghanistan Standard Time', 16200 ], - 2 : [ u'Alaskan Standard Time', -32400 ], - 3 : [ u'Arab Standard Time', 10800 ], - 4 : [ u'Arabian Standard Time', 14400 ], - 5 : [ u'Arabic Standard Time', 10800 ], - 6 : [ u'Argentina Standard Time', -10800 ], - 7 : [ u'Atlantic Standard Time', -14400 ], - 8 : [ u'AUS Central Standard Time', 34200 ], - 9 : [ u'AUS Eastern Standard Time', 36000 ], - 10 : [ u'Azerbaijan Standard Time', 14400 ], - 11 : [ u'Azores Standard Time', -3600 ], - 12 : [ u'Bahia Standard Time', -10800 ], - 13 : [ u'Bangladesh Standard Time', 21600 ], - 14 : [ u'Belarus Standard Time', 10800 ], - 15 : [ u'Canada Central Standard Time', -21600 ], - 16 : [ u'Cape Verde Standard Time', -3600 ], - 17 : [ u'Caucasus Standard Time', 14400 ], - 18 : [ u'Cen. Australia Standard Time', 34200 ], - 19 : [ u'Central America Standard Time', -21600 ], - 20 : [ u'Central Asia Standard Time', 21600 ], - 21 : [ u'Central Brazilian Standard Time', -14400 ], - 22 : [ u'Central Europe Standard Time', 3600 ], - 23 : [ u'Central European Standard Time', 3600 ], - 24 : [ u'Central Pacific Standard Time', 39600 ], - 25 : [ u'Central Standard Time (Mexico)', -21600 ], - 26 : [ u'Central Standard Time', -21600 ], - 27 : [ u'China Standard Time', 28800 ], - 28 : [ u'Dateline Standard Time', -43200 ], - 29 : [ u'E. Africa Standard Time', 10800 ], - 30 : [ u'E. Australia Standard Time', 36000 ], - 31 : [ u'E. South America Standard Time', -10800 ], - 32 : [ u'Eastern Standard Time', -18000 ], - 33 : [ u'Eastern Standard Time (Mexico)', -18000 ], - 34 : [ u'Egypt Standard Time', 7200 ], - 35 : [ u'Ekaterinburg Standard Time', 18000 ], - 36 : [ u'Fiji Standard Time', 43200 ], - 37 : [ u'FLE Standard Time', 7200 ], - 38 : [ u'Georgian Standard Time', 14400 ], - 39 : [ u'GMT Standard Time', 0 ], - 40 : [ u'Greenland Standard Time', -10800 ], - 41 : [ u'Greenwich Standard Time', 0 ], - 42 : [ u'GTB Standard Time', 7200 ], - 43 : [ u'Hawaiian Standard Time', -36000 ], - 44 : [ u'India Standard Time', 19800 ], - 45 : [ u'Iran Standard Time', 12600 ], - 46 : [ u'Israel Standard Time', 7200 ], - 47 : [ u'Jordan Standard Time', 7200 ], - 48 : [ u'Kaliningrad Standard Time', 7200 ], - 49 : [ u'Korea Standard Time', 32400 ], - 50 : [ u'Libya Standard Time', 7200 ], - 51 : [ u'Line Islands Standard Time', 50400 ], - 52 : [ u'Magadan Standard Time', 36000 ], - 53 : [ u'Mauritius Standard Time', 14400 ], - 54 : [ u'Middle East Standard Time', 7200 ], - 55 : [ u'Montevideo Standard Time', -10800 ], - 56 : [ u'Morocco Standard Time', 0 ], - 57 : [ u'Mountain Standard Time (Mexico)', -25200 ], - 58 : [ u'Mountain Standard Time', -25200 ], - 59 : [ u'Myanmar Standard Time', 23400 ], - 60 : [ u'N. Central Asia Standard Time', 21600 ], - 61 : [ u'Namibia Standard Time', 3600 ], - 62 : [ u'Nepal Standard Time', 20700 ], - 63 : [ u'New Zealand Standard Time', 43200 ], - 64 : [ u'Newfoundland Standard Time', -12600 ], - 65 : [ u'North Asia East Standard Time', 28800 ], - 66 : [ u'North Asia Standard Time', 25200 ], - 67 : [ u'Pacific SA Standard Time', -10800 ], - 68 : [ u'E. Europe Standard Time', 7200 ], - 69 : [ u'Pacific Standard Time', -28800 ], - 70 : [ u'Pakistan Standard Time', 18000 ], - 71 : [ u'Paraguay Standard Time', -14400 ], - 72 : [ u'Romance Standard Time', 3600 ], - 73 : [ u'Russia Time Zone 3', 14400 ], - 74 : [ u'Russia Time Zone 10', 39600 ], - 75 : [ u'Russia Time Zone 11', 43200 ], - 76 : [ u'Russian Standard Time', 10800 ], - 77 : [ u'SA Eastern Standard Time', -10800 ], - 78 : [ u'SA Pacific Standard Time', -18000 ], - 79 : [ u'SA Western Standard Time', -14400 ], - 80 : [ u'Samoa Standard Time', 46800 ], - 81 : [ u'SE Asia Standard Time', 25200 ], - 82 : [ u'Singapore Standard Time', 28800 ], - 83 : [ u'South Africa Standard Time', 7200 ], - 84 : [ u'Sri Lanka Standard Time', 19800 ], - 85 : [ u'Syria Standard Time', 7200 ], - 86 : [ u'Taipei Standard Time', 28800 ], - 87 : [ u'Tasmania Standard Time', 36000 ], - 88 : [ u'Tokyo Standard Time', 32400 ], - 89 : [ u'Tonga Standard Time', 46800 ], - 90 : [ u'Turkey Standard Time', 7200 ], - 91 : [ u'Ulaanbaatar Standard Time', 28800 ], - 92 : [ u'US Eastern Standard Time', -18000 ], - 93 : [ u'US Mountain Standard Time', -25200 ], - 94 : [ u'UTC-02', -7200 ], - 95 : [ u'UTC-11', -39600 ], - 96 : [ u'UTC', 0 ], - 97 : [ u'UTC+12', 43200 ], - 98 : [ u'Venezuela Standard Time', -16200 ], - 99 : [ u'Vladivostok Standard Time', 36000 ], - 100: [ u'W. Australia Standard Time', 28800 ], - 101: [ u'W. Central Africa Standard Time', 3600 ], - 102: [ u'W. Europe Standard Time', 3600 ], - 103: [ u'West Asia Standard Time', 18000 ], - 104: [ u'West Pacific Standard Time', 36000 ], - 105: [ u'Yakutsk Standard Time', 32400 ], - 106: [ u'North Korea Standard Time', 30600 ] -} - -def windowsIdToKey(windowsId): - for windowsKey in windowsIdList: - if windowsIdList[windowsKey][0] == windowsId: - return windowsKey - return 0 - -# List of standard UTC IDs to use. Not public so may be safely changed. -# Do not remove ID's as is part of API/behavior guarantee -# Key : [ UTC Id, Offset Seconds ] -utcIdList = { - 0 : [ u'UTC', 0 ], # Goes first so is default - 1 : [ u'UTC-14:00', -50400 ], - 2 : [ u'UTC-13:00', -46800 ], - 3 : [ u'UTC-12:00', -43200 ], - 4 : [ u'UTC-11:00', -39600 ], - 5 : [ u'UTC-10:00', -36000 ], - 6 : [ u'UTC-09:00', -32400 ], - 7 : [ u'UTC-08:00', -28800 ], - 8 : [ u'UTC-07:00', -25200 ], - 9 : [ u'UTC-06:00', -21600 ], - 10 : [ u'UTC-05:00', -18000 ], - 11 : [ u'UTC-04:30', -16200 ], - 12 : [ u'UTC-04:00', -14400 ], - 13 : [ u'UTC-03:30', -12600 ], - 14 : [ u'UTC-03:00', -10800 ], - 15 : [ u'UTC-02:00', -7200 ], - 16 : [ u'UTC-01:00', -3600 ], - 17 : [ u'UTC-00:00', 0 ], - 18 : [ u'UTC+00:00', 0 ], - 19 : [ u'UTC+01:00', 3600 ], - 20 : [ u'UTC+02:00', 7200 ], - 21 : [ u'UTC+03:00', 10800 ], - 22 : [ u'UTC+03:30', 12600 ], - 23 : [ u'UTC+04:00', 14400 ], - 24 : [ u'UTC+04:30', 16200 ], - 25 : [ u'UTC+05:00', 18000 ], - 26 : [ u'UTC+05:30', 19800 ], - 27 : [ u'UTC+05:45', 20700 ], - 28 : [ u'UTC+06:00', 21600 ], - 29 : [ u'UTC+06:30', 23400 ], - 30 : [ u'UTC+07:00', 25200 ], - 31 : [ u'UTC+08:00', 28800 ], - 32 : [ u'UTC+09:00', 32400 ], - 33 : [ u'UTC+09:30', 34200 ], - 34 : [ u'UTC+10:00', 36000 ], - 35 : [ u'UTC+11:00', 39600 ], - 36 : [ u'UTC+12:00', 43200 ], - 37 : [ u'UTC+13:00', 46800 ], - 38 : [ u'UTC+14:00', 50400 ], - 39 : [ u'UTC+08:30', 30600 ] -} - -def usage(): - print "Usage: cldr2qtimezone.py <path to cldr core/common> <path to qtbase>" - sys.exit() - -if len(sys.argv) != 3: - usage() - -cldrPath = sys.argv[1] -qtPath = sys.argv[2] - -if not os.path.isdir(cldrPath) or not os.path.isdir(qtPath): - usage() - -windowsZonesPath = cldrPath + "/supplemental/windowsZones.xml" -tempFileDir = qtPath -dataFilePath = qtPath + "/src/corelib/tools/qtimezoneprivate_data_p.h" - -if not os.path.isfile(windowsZonesPath): - usage() - -if not os.path.isfile(dataFilePath): - usage() - -cldr_version = 'unknown' -ldml = open(cldrPath + "/dtd/ldml.dtd", "r") -for line in ldml: - if 'version cldrVersion CDATA #FIXED' in line: - cldr_version = line.split('"')[1] - -# [[u'version', [(u'number', u'$Revision: 7825 $')]]] -versionNumber = findTagsInFile(windowsZonesPath, "version")[0][1][0][1] - -mapTimezones = findTagsInFile(windowsZonesPath, "windowsZones/mapTimezones") - -defaultDict = {} -windowsIdDict = {} - -if mapTimezones: - for mapZone in mapTimezones: - # [u'mapZone', [(u'territory', u'MH'), (u'other', u'UTC+12'), (u'type', u'Pacific/Majuro Pacific/Kwajalein')]] - if mapZone[0] == u'mapZone': - data = {} - for attribute in mapZone[1]: - if attribute[0] == u'other': - data['windowsId'] = attribute[1] - if attribute[0] == u'territory': - data['countryCode'] = attribute[1] - if attribute[0] == u'type': - data['ianaList'] = attribute[1] - - data['windowsKey'] = windowsIdToKey(data['windowsId']) - if data['windowsKey'] <= 0: - raise xpathlite.Error("Unknown Windows ID, please add \"%s\"" % data['windowsId']) - - countryId = 0 - if data['countryCode'] == u'001': - defaultDict[data['windowsKey']] = data['ianaList'] - else: - data['countryId'] = enumdata.countryCodeToId(data['countryCode']) - if data['countryId'] < 0: - raise xpathlite.Error("Unknown Country Code \"%s\"" % data['countryCode']) - data['country'] = enumdata.country_list[data['countryId']][0] - windowsIdDict[data['windowsKey'], data['countryId']] = data - -print "Input file parsed, now writing data" - -GENERATED_BLOCK_START = "// GENERATED PART STARTS HERE\n" -GENERATED_BLOCK_END = "// GENERATED PART ENDS HERE\n" - -# Create a temp file to write the new data into -(newTempFile, newTempFilePath) = tempfile.mkstemp("qtimezone_data_p", dir=tempFileDir) -newTempFile = os.fdopen(newTempFile, "w") - -# Open the old file and copy over the first non-generated section to the new file -oldDataFile = open(dataFilePath, "r") -s = oldDataFile.readline() -while s and s != GENERATED_BLOCK_START: - newTempFile.write(s) - s = oldDataFile.readline() - -# Write out generated block start tag and warning -newTempFile.write(GENERATED_BLOCK_START) -newTempFile.write(""" -/* - This part of the file was generated on %s from the - Common Locale Data Repository v%s supplemental/windowsZones.xml file %s - - http://www.unicode.org/cldr/ - - Do not edit this code: run cldr2qtimezone.py on updated (or - edited) CLDR data; see qtbase/util/local_database/. -*/ - -""" % (str(datetime.date.today()), cldr_version, versionNumber) ) - -windowsIdData = ByteArrayData() -ianaIdData = ByteArrayData() - -# Write Windows/IANA table -newTempFile.write("// Windows ID Key, Country Enum, IANA ID Index\n") -newTempFile.write("static const QZoneData zoneDataTable[] = {\n") -for index in windowsIdDict: - data = windowsIdDict[index] - newTempFile.write(" { %6d,%6d,%6d }, // %s / %s\n" - % (data['windowsKey'], - data['countryId'], - ianaIdData.append(data['ianaList']), - data['windowsId'], - data['country'])) -newTempFile.write(" { 0, 0, 0 } // Trailing zeroes\n") -newTempFile.write("};\n\n") - -print "Done Zone Data" - -# Write Windows ID key table -newTempFile.write("// Windows ID Key, Windows ID Index, IANA ID Index, UTC Offset\n") -newTempFile.write("static const QWindowsData windowsDataTable[] = {\n") -for windowsKey in windowsIdList: - newTempFile.write(" { %6d,%6d,%6d,%6d }, // %s\n" - % (windowsKey, - windowsIdData.append(windowsIdList[windowsKey][0]), - ianaIdData.append(defaultDict[windowsKey]), - windowsIdList[windowsKey][1], - windowsIdList[windowsKey][0])) -newTempFile.write(" { 0, 0, 0, 0 } // Trailing zeroes\n") -newTempFile.write("};\n\n") - -print "Done Windows Data Table" - -# Write UTC ID key table -newTempFile.write("// IANA ID Index, UTC Offset\n") -newTempFile.write("static const QUtcData utcDataTable[] = {\n") -for index in utcIdList: - data = utcIdList[index] - newTempFile.write(" { %6d,%6d }, // %s\n" - % (ianaIdData.append(data[0]), - data[1], - data[0])) -newTempFile.write(" { 0, 0 } // Trailing zeroes\n") -newTempFile.write("};\n\n") - -print "Done UTC Data Table" - -# Write out Windows ID's data -newTempFile.write("static const char windowsIdData[] = {\n") -newTempFile.write(wrap_list(windowsIdData.data)) -newTempFile.write("\n};\n\n") - -# Write out IANA ID's data -newTempFile.write("static const char ianaIdData[] = {\n") -newTempFile.write(wrap_list(ianaIdData.data)) -newTempFile.write("\n};\n") - -print "Done ID Data Table" - -# Write out the end of generated block tag -newTempFile.write(GENERATED_BLOCK_END) -s = oldDataFile.readline() - -# Skip through the old generated data in the old file -while s and s != GENERATED_BLOCK_END: - s = oldDataFile.readline() - -# Now copy the rest of the original file into the new file -s = oldDataFile.readline() -while s: - newTempFile.write(s) - s = oldDataFile.readline() - -# Now close the old and new file, delete the old file and copy the new file in its place -newTempFile.close() -oldDataFile.close() -os.remove(dataFilePath) -os.rename(newTempFilePath, dataFilePath) - -print "Data generation completed, please check the new file at " + dataFilePath diff --git a/util/locale_database/README b/util/locale_database/README new file mode 100644 index 0000000000..8654968d66 --- /dev/null +++ b/util/locale_database/README @@ -0,0 +1,5 @@ +locale_database is used to generate qlocale data from CLDR. + +CLDR is the Common Locale Data Repository, a database for localized +data (like date formats, country names etc). It is provided by the +Unicode consortium. diff --git a/util/local_database/cldr2qlocalexml.py b/util/locale_database/cldr2qlocalexml.py index 4ce0a6e3b1..072ea9e4ed 100755 --- a/util/local_database/cldr2qlocalexml.py +++ b/util/locale_database/cldr2qlocalexml.py @@ -1,7 +1,8 @@ #!/usr/bin/env python2 +# coding=utf8 ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2018 The Qt Company Ltd. ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the test suite of the Qt Toolkit. @@ -38,13 +39,16 @@ command-line argument. Save its standard output (but not error) to a file for later processing by ``./qlocalexml2cpp.py`` When you update the CLDR data, be sure to also update -src/corelib/tools/qt_attribution.json's entry for unicode-cldr. Check +src/corelib/text/qt_attribution.json's entry for unicode-cldr. Check this script's output for unknown language, country or script messages; if any can be resolved, use their entry in common/main/en.xml to append new entries to enumdata.py's lists and update documentation in -src/corelib/tools/qlocale.qdoc, adding the new entries in alphabetic +src/corelib/text/qlocale.qdoc, adding the new entries in alphabetic order. +While updating the locale data, check also for updates to MS-Win's +time zone names; see cldr2qtimezone.py for details. + .. _CLDR: ftp://unicode.org/Public/cldr/ """ @@ -59,6 +63,8 @@ from xpathlite import DraftResolution, findAlias, findEntry, findTagsInFile from dateconverter import convert_date from localexml import Locale +# TODO: make calendars a command-line option +calendars = ['gregorian', 'persian', 'islamic'] # 'hebrew' findEntryInFile = xpathlite._findEntryInFile def wrappedwarn(prefix, tokens): return sys.stderr.write( @@ -95,6 +101,34 @@ def parse_number_format(patterns, data): result.append(pattern) return result +def raiseUnknownCode(code, form, cache={}): + """Check whether an unknown code could be supported. + + We declare a language, script or country code unknown if it's not + known to enumdata.py; however, if it's present in main/en.xml's + mapping of codes to names, we have the option of adding support. + This caches the necessary look-up (so we only read main/en.xml + once) and returns the name we should use if we do add support. + + First parameter, code, is the unknown code. Second parameter, + form, is one of 'language', 'script' or 'country' to select the + type of code to look up. Do not pass further parameters (the next + will deprive you of the cache). + + Raises xpathlite.Error with a suitable message, that includes the + unknown code's full name if found. + + Relies on global cldr_dir being set before it's called; see tail + of this file. + """ + if not cache: + cache.update(xpathlite.codeMapsFromFile(os.path.join(cldr_dir, 'en.xml'))) + name = cache[form].get(code) + msg = 'unknown %s code "%s"' % (form, code) + if name: + msg += ' - could use "%s"' % name + raise xpathlite.Error(msg) + def parse_list_pattern_part_format(pattern): # This is a very limited parsing of the format for list pattern part only. return pattern.replace("{0}", "%1").replace("{1}", "%2").replace("{2}", "%3") @@ -193,18 +227,18 @@ def _generateLocaleInfo(path, language_code, script_code, country_code, variant_ language_id = enumdata.languageCodeToId(language_code) if language_id <= 0: - raise xpathlite.Error('unknown language code "%s"' % language_code) + raiseUnknownCode(language_code, 'language') script_id = enumdata.scriptCodeToId(script_code) if script_id == -1: - raise xpathlite.Error('unknown script code "%s"' % script_code) + raiseUnknownCode(script_code, 'script') # we should handle fully qualified names with the territory if not country_code: return {} country_id = enumdata.countryCodeToId(country_code) if country_id <= 0: - raise xpathlite.Error('unknown country code "%s"' % country_code) + raiseUnknownCode(country_code, 'country') # So we say we accept only those values that have "contributed" or # "approved" resolution. see http://www.unicode.org/cldr/process.html @@ -364,12 +398,12 @@ def _generateLocaleInfo(path, language_code, script_code, country_code, variant_ ('narrow', 'format', 'narrow'), ) - # Month data: - for cal in ('gregorian',): # We shall want to add to this + # Month names for 12-month calendars: + for cal in calendars: stem = 'dates/calendars/calendar[' + cal + ']/months/' for (key, mode, size) in namings: prop = 'monthContext[' + mode + ']/monthWidth[' + size + ']/' - result[key + 'Months'] = ';'.join( + result[key + 'Months_' + cal] = ';'.join( findEntry(path, stem + prop + "month[%d]" % i) for i in range(1, 13)) + ';' @@ -655,9 +689,9 @@ if skips: wrappedwarn('skipping likelySubtags (for unknown language codes): ', skips) print " <localeList>" -Locale.C().toXml() +Locale.C(calendars).toXml(calendars) for key in locale_keys: - locale_database[key].toXml() + locale_database[key].toXml(calendars) print " </localeList>" print "</localeDatabase>" diff --git a/util/locale_database/cldr2qtimezone.py b/util/locale_database/cldr2qtimezone.py new file mode 100755 index 0000000000..c240d0d190 --- /dev/null +++ b/util/locale_database/cldr2qtimezone.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python2 +############################################################################# +## +## Copyright (C) 2016 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the test suite of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# +"""Parse CLDR data for QTimeZone use with MS-Windows + +Script to parse the CLDR supplemental/windowsZones.xml file and encode +for use in QTimeZone. See ``./cldr2qlocalexml.py`` for where to get +the CLDR data. Pass its common/ directory as first parameter to this +script and the qtbase root directory as second parameter. It shall +update qtbase's src/corelib/time/qtimezoneprivate_data_p.h ready for +use. + +The XML structure is as follows: + + <supplementalData> + <version number="$Revision:...$"/> + <generation date="$Date:...$"/> + <windowsZones> + <mapTimezones otherVersion="..." typeVersion="..."> + <!-- (UTC-08:00) Pacific Time (US & Canada) --> + <mapZone other="Pacific Standard Time" territory="001" type="America/Los_Angeles"/> + <mapZone other="Pacific Standard Time" territory="CA" type="America/Vancouver America/Dawson America/Whitehorse"/> + <mapZone other="Pacific Standard Time" territory="US" type="America/Los_Angeles America/Metlakatla"/> + <mapZone other="Pacific Standard Time" territory="ZZ" type="PST8PDT"/> + </mapTimezones> + </windowsZones> + </supplementalData> +""" + +import os +import sys +import datetime +import tempfile +import enumdata +import xpathlite +from xpathlite import DraftResolution +import re +import qlocalexml2cpp + +findAlias = xpathlite.findAlias +findEntry = xpathlite.findEntry +findEntryInFile = xpathlite._findEntryInFile +findTagsInFile = xpathlite.findTagsInFile +unicode2hex = qlocalexml2cpp.unicode2hex +wrap_list = qlocalexml2cpp.wrap_list + +class ByteArrayData: + def __init__(self): + self.data = [] + self.hash = {} + def append(self, s): + s = s + '\0' + if s in self.hash: + return self.hash[s] + + lst = unicode2hex(s) + index = len(self.data) + if index > 65535: + print "\n\n\n#error Data index is too big!" + sys.stderr.write ("\n\n\nERROR: index exceeds the uint16 range! index = %d\n" % index) + sys.exit(1) + self.hash[s] = index + self.data += lst + return index + +# List of currently known Windows IDs. +# If this script reports missing IDs, please add them here. +# Look up the offset using (google and) timeanddate.com. +# Not public so may safely be changed. Please keep in alphabetic order by ID. +# ( Windows Id, Offset Seconds ) +windowsIdList = ( + (u'Afghanistan Standard Time', 16200), + (u'Alaskan Standard Time', -32400), + (u'Aleutian Standard Time', -36000), + (u'Altai Standard Time', 25200), + (u'Arab Standard Time', 10800), + (u'Arabian Standard Time', 14400), + (u'Arabic Standard Time', 10800), + (u'Argentina Standard Time', -10800), + (u'Astrakhan Standard Time', 14400), + (u'Atlantic Standard Time', -14400), + (u'AUS Central Standard Time', 34200), + (u'Aus Central W. Standard Time', 31500), + (u'AUS Eastern Standard Time', 36000), + (u'Azerbaijan Standard Time', 14400), + (u'Azores Standard Time', -3600), + (u'Bahia Standard Time', -10800), + (u'Bangladesh Standard Time', 21600), + (u'Belarus Standard Time', 10800), + (u'Bougainville Standard Time', 39600), + (u'Canada Central Standard Time', -21600), + (u'Cape Verde Standard Time', -3600), + (u'Caucasus Standard Time', 14400), + (u'Cen. Australia Standard Time', 34200), + (u'Central America Standard Time', -21600), + (u'Central Asia Standard Time', 21600), + (u'Central Brazilian Standard Time', -14400), + (u'Central Europe Standard Time', 3600), + (u'Central European Standard Time', 3600), + (u'Central Pacific Standard Time', 39600), + (u'Central Standard Time (Mexico)', -21600), + (u'Central Standard Time', -21600), + (u'China Standard Time', 28800), + (u'Chatham Islands Standard Time', 45900), + (u'Cuba Standard Time', -18000), + (u'Dateline Standard Time', -43200), + (u'E. Africa Standard Time', 10800), + (u'E. Australia Standard Time', 36000), + (u'E. Europe Standard Time', 7200), + (u'E. South America Standard Time', -10800), + (u'Easter Island Standard Time', -21600), + (u'Eastern Standard Time', -18000), + (u'Eastern Standard Time (Mexico)', -18000), + (u'Egypt Standard Time', 7200), + (u'Ekaterinburg Standard Time', 18000), + (u'Fiji Standard Time', 43200), + (u'FLE Standard Time', 7200), + (u'Georgian Standard Time', 14400), + (u'GMT Standard Time', 0), + (u'Greenland Standard Time', -10800), + (u'Greenwich Standard Time', 0), + (u'GTB Standard Time', 7200), + (u'Haiti Standard Time', -18000), + (u'Hawaiian Standard Time', -36000), + (u'India Standard Time', 19800), + (u'Iran Standard Time', 12600), + (u'Israel Standard Time', 7200), + (u'Jordan Standard Time', 7200), + (u'Kaliningrad Standard Time', 7200), + (u'Korea Standard Time', 32400), + (u'Libya Standard Time', 7200), + (u'Line Islands Standard Time', 50400), + (u'Lord Howe Standard Time', 37800), + (u'Magadan Standard Time', 36000), + (u'Magallanes Standard Time', -10800), # permanent DST + (u'Marquesas Standard Time', -34200), + (u'Mauritius Standard Time', 14400), + (u'Middle East Standard Time', 7200), + (u'Montevideo Standard Time', -10800), + (u'Morocco Standard Time', 0), + (u'Mountain Standard Time (Mexico)', -25200), + (u'Mountain Standard Time', -25200), + (u'Myanmar Standard Time', 23400), + (u'N. Central Asia Standard Time', 21600), + (u'Namibia Standard Time', 3600), + (u'Nepal Standard Time', 20700), + (u'New Zealand Standard Time', 43200), + (u'Newfoundland Standard Time', -12600), + (u'Norfolk Standard Time', 39600), + (u'North Asia East Standard Time', 28800), + (u'North Asia Standard Time', 25200), + (u'North Korea Standard Time', 30600), + (u'Omsk Standard Time', 21600), + (u'Pacific SA Standard Time', -10800), + (u'Pacific Standard Time', -28800), + (u'Pacific Standard Time (Mexico)', -28800), + (u'Pakistan Standard Time', 18000), + (u'Paraguay Standard Time', -14400), + (u'Romance Standard Time', 3600), + (u'Russia Time Zone 3', 14400), + (u'Russia Time Zone 10', 39600), + (u'Russia Time Zone 11', 43200), + (u'Russian Standard Time', 10800), + (u'SA Eastern Standard Time', -10800), + (u'SA Pacific Standard Time', -18000), + (u'SA Western Standard Time', -14400), + (u'Saint Pierre Standard Time', -10800), # New France + (u'Sakhalin Standard Time', 39600), + (u'Samoa Standard Time', 46800), + (u'Sao Tome Standard Time', 0), + (u'Saratov Standard Time', 14400), + (u'SE Asia Standard Time', 25200), + (u'Singapore Standard Time', 28800), + (u'South Africa Standard Time', 7200), + (u'Sri Lanka Standard Time', 19800), + (u'Sudan Standard Time', 7200), # unless they mean South Sudan, +03:00 + (u'Syria Standard Time', 7200), + (u'Taipei Standard Time', 28800), + (u'Tasmania Standard Time', 36000), + (u'Tocantins Standard Time', -10800), + (u'Tokyo Standard Time', 32400), + (u'Tomsk Standard Time', 25200), + (u'Tonga Standard Time', 46800), + (u'Transbaikal Standard Time', 32400), # Yakutsk + (u'Turkey Standard Time', 7200), + (u'Turks And Caicos Standard Time', -14400), + (u'Ulaanbaatar Standard Time', 28800), + (u'US Eastern Standard Time', -18000), + (u'US Mountain Standard Time', -25200), + (u'UTC-11', -39600), + (u'UTC-09', -32400), + (u'UTC-08', -28800), + (u'UTC-02', -7200), + (u'UTC', 0), + (u'UTC+12', 43200), + (u'UTC+13', 46800), + (u'Venezuela Standard Time', -16200), + (u'Vladivostok Standard Time', 36000), + (u'W. Australia Standard Time', 28800), + (u'W. Central Africa Standard Time', 3600), + (u'W. Europe Standard Time', 3600), + (u'W. Mongolia Standard Time', 25200), # Hovd + (u'West Asia Standard Time', 18000), + (u'West Bank Standard Time', 7200), + (u'West Pacific Standard Time', 36000), + (u'Yakutsk Standard Time', 32400), +) + +def windowsIdToKey(windowsId): + for index, pair in enumerate(windowsIdList): + if pair[0] == windowsId: + return index + 1 + return 0 + +# List of standard UTC IDs to use. Not public so may be safely changed. +# Do not remove IDs, as each entry is part of the API/behavior guarantee. +# ( UTC Id, Offset Seconds ) +utcIdList = ( + (u'UTC', 0), # Goes first so is default + (u'UTC-14:00', -50400), + (u'UTC-13:00', -46800), + (u'UTC-12:00', -43200), + (u'UTC-11:00', -39600), + (u'UTC-10:00', -36000), + (u'UTC-09:00', -32400), + (u'UTC-08:00', -28800), + (u'UTC-07:00', -25200), + (u'UTC-06:00', -21600), + (u'UTC-05:00', -18000), + (u'UTC-04:30', -16200), + (u'UTC-04:00', -14400), + (u'UTC-03:30', -12600), + (u'UTC-03:00', -10800), + (u'UTC-02:00', -7200), + (u'UTC-01:00', -3600), + (u'UTC-00:00', 0), + (u'UTC+00:00', 0), + (u'UTC+01:00', 3600), + (u'UTC+02:00', 7200), + (u'UTC+03:00', 10800), + (u'UTC+03:30', 12600), + (u'UTC+04:00', 14400), + (u'UTC+04:30', 16200), + (u'UTC+05:00', 18000), + (u'UTC+05:30', 19800), + (u'UTC+05:45', 20700), + (u'UTC+06:00', 21600), + (u'UTC+06:30', 23400), + (u'UTC+07:00', 25200), + (u'UTC+08:00', 28800), + (u'UTC+08:30', 30600), + (u'UTC+09:00', 32400), + (u'UTC+09:30', 34200), + (u'UTC+10:00', 36000), + (u'UTC+11:00', 39600), + (u'UTC+12:00', 43200), + (u'UTC+13:00', 46800), + (u'UTC+14:00', 50400), +) + +def usage(): + print "Usage: cldr2qtimezone.py <path to cldr core/common> <path to qtbase>" + sys.exit() + +if len(sys.argv) != 3: + usage() + +cldrPath = sys.argv[1] +qtPath = sys.argv[2] + +if not os.path.isdir(cldrPath) or not os.path.isdir(qtPath): + usage() + +windowsZonesPath = cldrPath + "/supplemental/windowsZones.xml" +tempFileDir = qtPath +dataFilePath = qtPath + "/src/corelib/time/qtimezoneprivate_data_p.h" + +if not (os.path.isfile(windowsZonesPath) and os.path.isfile(dataFilePath)): + usage() + +cldr_version = 'unknown' +ldml = open(cldrPath + "/dtd/ldml.dtd", "r") +for line in ldml: + if 'version cldrVersion CDATA #FIXED' in line: + cldr_version = line.split('"')[1] + +# [[u'version', [(u'number', u'$Revision: 7825 $')]]] +versionNumber = findTagsInFile(windowsZonesPath, "version")[0][1][0][1] + +mapTimezones = findTagsInFile(windowsZonesPath, "windowsZones/mapTimezones") + +defaultDict = {} +windowsIdDict = {} + +if mapTimezones: + badZones = set() + for mapZone in mapTimezones: + # [u'mapZone', [(u'territory', u'MH'), (u'other', u'UTC+12'), (u'type', u'Pacific/Majuro Pacific/Kwajalein')]] + if mapZone[0] == u'mapZone': + data = {} + for attribute in mapZone[1]: + if attribute[0] == u'other': + data['windowsId'] = attribute[1] + if attribute[0] == u'territory': + data['countryCode'] = attribute[1] + if attribute[0] == u'type': + data['ianaList'] = attribute[1] + + data['windowsKey'] = windowsIdToKey(data['windowsId']) + if data['windowsKey'] <= 0: + badZones.add(data['windowsId']) + + countryId = 0 + if data['countryCode'] == u'001': + defaultDict[data['windowsKey']] = data['ianaList'] + else: + data['countryId'] = enumdata.countryCodeToId(data['countryCode']) + if data['countryId'] < 0: + raise xpathlite.Error("Unknown Country Code \"%s\"" % data['countryCode']) + data['country'] = enumdata.country_list[data['countryId']][0] + windowsIdDict[data['windowsKey'], data['countryId']] = data + if badZones: + sys.stderr.write('\n\t'.join(["\nUnknown Windows ID, please add:"] + sorted(badZones)) + + "\nto the windowIdList in cldr2qtimezone.py\n\n") + raise xpathlite.Error("Unknown Windows IDs") + +print "Input file parsed, now writing data" + +GENERATED_BLOCK_START = "// GENERATED PART STARTS HERE\n" +GENERATED_BLOCK_END = "// GENERATED PART ENDS HERE\n" + +# Create a temp file to write the new data into +(newTempFile, newTempFilePath) = tempfile.mkstemp("qtimezone_data_p", dir=tempFileDir) +newTempFile = os.fdopen(newTempFile, "w") + +# Open the old file and copy over the first non-generated section to the new file +oldDataFile = open(dataFilePath, "r") +s = oldDataFile.readline() +while s and s != GENERATED_BLOCK_START: + newTempFile.write(s) + s = oldDataFile.readline() + +# Write out generated block start tag and warning +newTempFile.write(GENERATED_BLOCK_START) +newTempFile.write(""" +/* + This part of the file was generated on %s from the + Common Locale Data Repository v%s supplemental/windowsZones.xml file %s + + http://www.unicode.org/cldr/ + + Do not edit this code: run cldr2qtimezone.py on updated (or + edited) CLDR data; see qtbase/util/locale_database/. +*/ + +""" % (str(datetime.date.today()), cldr_version, versionNumber) ) + +windowsIdData = ByteArrayData() +ianaIdData = ByteArrayData() + +# Write Windows/IANA table +newTempFile.write("// Windows ID Key, Country Enum, IANA ID Index\n") +newTempFile.write("static const QZoneData zoneDataTable[] = {\n") +for index in windowsIdDict: + data = windowsIdDict[index] + newTempFile.write(" { %6d,%6d,%6d }, // %s / %s\n" + % (data['windowsKey'], + data['countryId'], + ianaIdData.append(data['ianaList']), + data['windowsId'], + data['country'])) +newTempFile.write(" { 0, 0, 0 } // Trailing zeroes\n") +newTempFile.write("};\n\n") + +print "Done Zone Data" + +# Write Windows ID key table +newTempFile.write("// Windows ID Key, Windows ID Index, IANA ID Index, UTC Offset\n") +newTempFile.write("static const QWindowsData windowsDataTable[] = {\n") +for index, pair in enumerate(windowsIdList): + newTempFile.write(" { %6d,%6d,%6d,%6d }, // %s\n" + % (index + 1, windowsIdData.append(pair[0]), + ianaIdData.append(defaultDict[index + 1]), pair[1], pair[0])) +newTempFile.write(" { 0, 0, 0, 0 } // Trailing zeroes\n") +newTempFile.write("};\n\n") + +print "Done Windows Data Table" + +# Write UTC ID key table +newTempFile.write("// IANA ID Index, UTC Offset\n") +newTempFile.write("static const QUtcData utcDataTable[] = {\n") +for pair in utcIdList: + newTempFile.write(" { %6d,%6d }, // %s\n" + % (ianaIdData.append(pair[0]), pair[1], pair[0])) +newTempFile.write(" { 0, 0 } // Trailing zeroes\n") +newTempFile.write("};\n\n") + +print "Done UTC Data Table" + +# Write out Windows ID's data +newTempFile.write("static const char windowsIdData[] = {\n") +newTempFile.write(wrap_list(windowsIdData.data)) +newTempFile.write("\n};\n\n") + +# Write out IANA ID's data +newTempFile.write("static const char ianaIdData[] = {\n") +newTempFile.write(wrap_list(ianaIdData.data)) +newTempFile.write("\n};\n") + +print "Done ID Data Table" + +# Write out the end of generated block tag +newTempFile.write(GENERATED_BLOCK_END) +s = oldDataFile.readline() + +# Skip through the old generated data in the old file +while s and s != GENERATED_BLOCK_END: + s = oldDataFile.readline() + +# Now copy the rest of the original file into the new file +s = oldDataFile.readline() +while s: + newTempFile.write(s) + s = oldDataFile.readline() + +# Now close the old and new file, delete the old file and copy the new file in its place +newTempFile.close() +oldDataFile.close() +os.remove(dataFilePath) +os.rename(newTempFilePath, dataFilePath) + +print "Data generation completed, please check the new file at " + dataFilePath diff --git a/util/local_database/dateconverter.py b/util/locale_database/dateconverter.py index 1990fe0c61..1990fe0c61 100755 --- a/util/local_database/dateconverter.py +++ b/util/locale_database/dateconverter.py diff --git a/util/local_database/enumdata.py b/util/locale_database/enumdata.py index 26bb74d1fe..0e40d8a9ee 100644 --- a/util/local_database/enumdata.py +++ b/util/locale_database/enumdata.py @@ -402,6 +402,8 @@ language_list = { 362: ["Sicilian", "scn"], 363: ["Southern Kurdish", "sdh"], 364: ["Western Balochi", "bgn"], + 365: ["Cebuano", "ceb"], + 366: ["Erzya", "myv"], } language_aliases = { diff --git a/util/local_database/formattags.txt b/util/locale_database/formattags.txt index 5138c37a81..5138c37a81 100644 --- a/util/local_database/formattags.txt +++ b/util/locale_database/formattags.txt diff --git a/util/local_database/localexml.py b/util/locale_database/localexml.py index e95b3aebcc..9b353f5122 100644 --- a/util/local_database/localexml.py +++ b/util/locale_database/localexml.py @@ -1,6 +1,7 @@ +# coding=utf8 ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2018 The Qt Company Ltd. ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the test suite of the Qt Toolkit. @@ -113,12 +114,11 @@ def convertFormat(format): return result class Locale: - # Tool used during class body (see del below), not method: - def propsMonthDay(lengths=('long', 'short', 'narrow'), scale=('months', 'days')): + @staticmethod + def propsMonthDay(scale, lengths=('long', 'short', 'narrow')): for L in lengths: - for S in scale: - yield camelCase((L, S)) - yield camelCase(('standalone', L, S)) + yield camelCase((L, scale)) + yield camelCase(('standalone', L, scale)) # Expected to be numbers, read with int(): __asint = ("decimal", "group", "zero", @@ -137,15 +137,13 @@ class Locale: "listPatternPartEnd", "listPatternPartTwo", "am", "pm", 'byte_unit', 'byte_si_quantified', 'byte_iec_quantified', "currencyIsoCode", "currencySymbol", "currencyDisplayName", - "currencyFormat", "currencyNegativeFormat" - ) + tuple(propsMonthDay()) - del propsMonthDay + "currencyFormat", "currencyNegativeFormat") # Day-of-Week numbering used by Qt: __qDoW = {"mon": 1, "tue": 2, "wed": 3, "thu": 4, "fri": 5, "sat": 6, "sun": 7} @classmethod - def fromXmlData(cls, lookup): + def fromXmlData(cls, lookup, calendars=('gregorian',)): """Constructor from the contents of XML elements. Single parameter, lookup, is called with the names of XML @@ -170,12 +168,15 @@ class Locale: for k in cls.__asfmt: data[k] = convertFormat(lookup(k)) - for k in cls.__astxt: + for k in cls.__astxt + tuple(cls.propsMonthDay('days')): data[k] = lookup(k) + for k in cls.propsMonthDay('months'): + data[k] = dict((cal, lookup('_'.join((k, cal)))) for cal in calendars) + return cls(data) - def toXml(self, indent=' ', tab=' '): + def toXml(self, calendars=('gregorian',), indent=' ', tab=' '): print indent + '<locale>' inner = indent + tab get = lambda k: getattr(self, k) @@ -199,13 +200,14 @@ class Locale: 'weekendStart', 'weekendEnd', 'longDateFormat', 'shortDateFormat', 'longTimeFormat', 'shortTimeFormat', - 'standaloneLongMonths', 'standaloneShortMonths', - 'standaloneNarrowMonths', - 'longMonths', 'shortMonths', 'narrowMonths', 'longDays', 'shortDays', 'narrowDays', 'standaloneLongDays', 'standaloneShortDays', 'standaloneNarrowDays', 'currencyIsoCode', 'currencySymbol', 'currencyDisplayName', - 'currencyFormat', 'currencyNegativeFormat'): + 'currencyFormat', 'currencyNegativeFormat' + ) + tuple(self.propsMonthDay('days')) + tuple( + '_'.join((k, cal)) + for k in self.propsMonthDay('months') + for cal in calendars): ent = camelCase(key.split('_')) if key.endswith('_endonym') else key print inner + "<%s>%s</%s>" % (ent, escape(get(key)).encode('utf-8'), ent) @@ -218,16 +220,70 @@ class Locale: if data: self.__dict__.update(data) if kw: self.__dict__.update(kw) + # Tools used by __monthNames: + def fullName(i, name): return name + def firstThree(i, name): return name[:3] + def initial(i, name): return name[:1] + def number(i, name): return str(i + 1) + def islamicShort(i, name): + if not name: return name + if name == 'Shawwal': return 'Shaw.' + words = name.split() + if words[0].startswith('Dhu'): + words[0] = words[0][:7] + '.' + elif len(words[0]) > 3: + words[0] = words[0][:3] + '.' + return ' '.join(words) + @staticmethod + def __monthNames(calendars, + known={ # Map calendar to (names, extractors...): + 'gregorian': (('January', 'February', 'March', 'April', 'May', 'June', 'July', + 'August', 'September', 'October', 'November', 'December'), + # Extractor pairs, (plain, standalone) + (fullName, fullName), # long + (firstThree, firstThree), # short + (number, initial)), # narrow + 'persian': (('Farvardin', 'Ordibehesht', 'Khordad', 'Tir', 'Mordad', + 'Shahrivar', 'Mehr', 'Aban', 'Azar', 'Dey', 'Bahman', 'Esfand'), + (fullName, fullName), + (firstThree, firstThree), + (number, initial)), + 'islamic': ((u'Muharram', u'Safar', u'Rabiʻ I', u'Rabiʻ II', u'Jumada I', + u'Jumada II', u'Rajab', u'Shaʻban', u'Ramadan', u'Shawwal', + u'Dhuʻl-Qiʻdah', u'Dhuʻl-Hijjah'), + (fullName, fullName), + (islamicShort, islamicShort), + (number, number)), + 'hebrew': (('Tishri', 'Heshvan', 'Kislev', 'Tevet', 'Shevat', 'Adar I', + 'Adar', 'Nisan', 'Iyar', 'Sivan', 'Tamuz', 'Av'), + (fullName, fullName), + (fullName, fullName), + (number, number)), + }, + sizes=('long', 'short', 'narrow')): + for cal in calendars: + try: + data = known[cal] + except KeyError: # Need to add an entry to known, above. + print 'Unsupported calendar:', cal + raise + names, get = data[0] + ('',), data[1:] + for n, size in enumerate(sizes): + yield ('_'.join((camelCase((size, 'months')), cal)), + ';'.join(get[n][0](i, x) for i, x in enumerate(names))) + yield ('_'.join((camelCase(('standalone', size, 'months')), cal)), + ';'.join(get[n][1](i, x) for i, x in enumerate(names))) + del fullName, firstThree, initial, number, islamicShort + @classmethod - def C(cls, - # Empty entries at end to ensure final separator when join()ed: - months = ('January', 'February', 'March', 'April', 'May', 'June', 'July', - 'August', 'September', 'October', 'November', 'December', ''), + def C(cls, calendars=('gregorian',), + # Empty entry at end to ensure final separator when join()ed: days = ('Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', ''), quantifiers=('k', 'M', 'G', 'T', 'P', 'E')): """Returns an object representing the C locale.""" - return cls(language='C', language_code='0', language_endonym='', + return cls(dict(cls.__monthNames(calendars)), + language='C', language_code='0', language_endonym='', script='AnyScript', script_code='0', country='AnyCountry', country_code='0', country_endonym='', decimal='.', group=',', list=';', percent='%', @@ -245,12 +301,6 @@ class Locale: weekendStart='sat', weekendEnd='sun', longDateFormat='EEEE, d MMMM yyyy', shortDateFormat='d MMM yyyy', longTimeFormat='HH:mm:ss z', shortTimeFormat='HH:mm:ss', - longMonths=';'.join(months), - shortMonths=';'.join(m[:3] for m in months), - narrowMonths='1;2;3;4;5;6;7;8;9;10;11;12;', - standaloneLongMonths=';'.join(months), - standaloneShortMonths=';'.join(m[:3] for m in months), - standaloneNarrowMonths=';'.join(m[:1] for m in months), longDays=';'.join(days), shortDays=';'.join(d[:3] for d in days), narrowDays='7;1;2;3;4;5;6;', diff --git a/util/local_database/qlocalexml2cpp.py b/util/locale_database/qlocalexml2cpp.py index fb5ae5ba54..a5ff7ebbf4 100755 --- a/util/local_database/qlocalexml2cpp.py +++ b/util/locale_database/qlocalexml2cpp.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2018 The Qt Company Ltd. ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the test suite of the Qt Toolkit. @@ -42,6 +42,24 @@ from enumdata import language_aliases, country_aliases, script_aliases from localexml import Locale +# TODO: Make calendars a command-line parameter +# map { CLDR name: Qt file name } +calendars = {'gregorian': 'roman', 'persian': 'jalali', 'islamic': 'hijri',} # 'hebrew': 'hebrew', + +generated_template = """ +/* + This part of the file was generated on %s from the + Common Locale Data Repository v%s + + http://www.unicode.org/cldr/ + + Do not edit this section: instead regenerate it using + cldr2qlocalexml.py and qlocalexml2cpp.py on updated (or + edited) CLDR data; see qtbase/util/locale_database/. +*/ + +""" + class Error: def __init__(self, msg): self.msg = msg @@ -151,7 +169,7 @@ def loadLocaleMap(doc, language_map, script_map, country_map, likely_subtags_map result = {} for locale_elt in eachEltInGroup(doc.documentElement, "localeList", "locale"): - locale = Locale.fromXmlData(lambda k: firstChildText(locale_elt, k)) + locale = Locale.fromXmlData(lambda k: firstChildText(locale_elt, k), calendars.keys()) language_id = languageNameToId(locale.language, language_map) if language_id == -1: sys.stderr.write("Cannot find a language id for '%s'\n" % locale.language) @@ -254,6 +272,7 @@ class StringData: self.data = [] self.hash = {} self.name = name + def append(self, s): if s in self.hash: return self.hash[s] @@ -279,6 +298,11 @@ class StringData: self.data += lst return token + def write(self, fd): + fd.write("\nstatic const ushort %s[] = {\n" % self.name) + fd.write(wrap_list(self.data)) + fd.write("\n};\n") + def escapedString(s): result = "" i = 0 @@ -339,7 +363,7 @@ def main(): (data_temp_file, data_temp_file_path) = tempfile.mkstemp("qlocale_data_p", dir=qtsrcdir) data_temp_file = os.fdopen(data_temp_file, "w") - qlocaledata_file = open(qtsrcdir + "/src/corelib/tools/qlocale_data_p.h", "r") + qlocaledata_file = open(qtsrcdir + "/src/corelib/text/qlocale_data_p.h", "r") s = qlocaledata_file.readline() while s and s != GENERATED_BLOCK_START: data_temp_file.write(s) @@ -360,20 +384,7 @@ def main(): dupes = findDupes(language_map, country_map) cldr_version = firstChildText(doc.documentElement, "version") - - data_temp_file.write(""" -/* - This part of the file was generated on %s from the - Common Locale Data Repository v%s - - http://www.unicode.org/cldr/ - - Do not edit this section: instead regenerate it using - cldr2qlocalexml.py and qlocalexml2cpp.py on updated (or - edited) CLDR data; see qtbase/util/local_database/. -*/ - -""" % (str(datetime.date.today()), cldr_version) ) + data_temp_file.write(generated_template % (datetime.date.today(), cldr_version)) # Likely subtags map data_temp_file.write("static const QLocaleId likely_subtags[] = {\n") @@ -442,7 +453,6 @@ def main(): list_pattern_part_data = StringData('list_pattern_part_data') date_format_data = StringData('date_format_data') time_format_data = StringData('time_format_data') - months_data = StringData('months_data') days_data = StringData('days_data') am_data = StringData('am_data') pm_data = StringData('pm_data') @@ -482,12 +492,6 @@ def main(): + ' lDtFmt ' + ' sTmFmt ' # Time format + ' lTmFmt ' - + ' ssMonth ' # Months - + ' slMonth ' - + ' snMonth ' - + ' sMonth ' - + ' lMonth ' - + ' nMonth ' + ' ssDays ' # Days + ' slDays ' + ' snDays ' @@ -532,7 +536,7 @@ def main(): # Quotation marks: + '%8d,' * 4 # List patterns, date/time formats, month/day names, am/pm: - + '%11s,' * 22 + + '%11s,' * 16 # SI/IEC byte-unit abbreviations: + '%8s,' * 3 # Currency ISO code: @@ -568,12 +572,6 @@ def main(): date_format_data.append(l.longDateFormat), time_format_data.append(l.shortTimeFormat), time_format_data.append(l.longTimeFormat), - months_data.append(l.standaloneShortMonths), - months_data.append(l.standaloneLongMonths), - months_data.append(l.standaloneNarrowMonths), - months_data.append(l.shortMonths), - months_data.append(l.longMonths), - months_data.append(l.narrowMonths), days_data.append(l.standaloneShortDays), days_data.append(l.standaloneLongDays), days_data.append(l.standaloneNarrowDays), @@ -599,7 +597,7 @@ def main(): l.weekendEnd) + ", // %s/%s/%s\n" % (l.language, l.script, l.country)) data_temp_file.write(line_format # All zeros, matching the format: - % ( (0,) * (3 + 8 + 4) + ("0,0",) * (22 + 3) + % ( (0,) * (3 + 8 + 4) + ("0,0",) * (16 + 3) + (currencyIsoCodeData(0),) + ("0,0",) * 6 + (0,) * (2 + 3)) + " // trailing 0s\n") @@ -607,13 +605,11 @@ def main(): # StringData tables: for data in (list_pattern_part_data, date_format_data, - time_format_data, months_data, days_data, + time_format_data, days_data, byte_unit_data, am_data, pm_data, currency_symbol_data, currency_display_name_data, currency_format_data, endonyms_data): - data_temp_file.write("\nstatic const ushort %s[] = {\n" % data.name) - data_temp_file.write(wrap_list(data.data)) - data_temp_file.write("\n};\n") + data.write(data_temp_file) data_temp_file.write("\n") @@ -735,14 +731,70 @@ def main(): data_temp_file.close() qlocaledata_file.close() - os.remove(qtsrcdir + "/src/corelib/tools/qlocale_data_p.h") - os.rename(data_temp_file_path, qtsrcdir + "/src/corelib/tools/qlocale_data_p.h") + os.remove(qtsrcdir + "/src/corelib/text/qlocale_data_p.h") + os.rename(data_temp_file_path, qtsrcdir + "/src/corelib/text/qlocale_data_p.h") + + # Generate calendar data + calendar_format = ' {%6d,%6d,%6d,{%5s},{%5s},{%5s},{%5s},{%5s},{%5s}}, ' + for calendar, stem in calendars.items(): + months_data = StringData('months_data') + calendar_data_file = "q%scalendar_data_p.h" % stem + calendar_template_file = open(os.path.join(qtsrcdir, 'src', 'corelib', 'time', + calendar_data_file), "r") + (calendar_temp_file, calendar_temp_file_path) = tempfile.mkstemp(calendar_data_file, dir=qtsrcdir) + calendar_temp_file = os.fdopen(calendar_temp_file, "w") + s = calendar_template_file.readline() + while s and s != GENERATED_BLOCK_START: + calendar_temp_file.write(s) + s = calendar_template_file.readline() + calendar_temp_file.write(GENERATED_BLOCK_START) + calendar_temp_file.write(generated_template % (datetime.date.today(), cldr_version)) + calendar_temp_file.write("static const QCalendarLocale locale_data[] = {\n") + calendar_temp_file.write(' // ' + # IDs, width 7 (6 + comma) + + ' lang ' + + ' script' + + ' terr ' + # Month-name start-end pairs, width 8 (5 plus '{},'): + + ' sShort ' + + ' sLong ' + + ' sNarrow' + + ' short ' + + ' long ' + + ' narrow' + # No trailing space on last; be sure + # to pad before adding later entries. + + '\n') + for key in locale_keys: + l = locale_map[key] + calendar_temp_file.write( + calendar_format + % (key[0], key[1], key[2], + months_data.append(l.standaloneShortMonths[calendar]), + months_data.append(l.standaloneLongMonths[calendar]), + months_data.append(l.standaloneNarrowMonths[calendar]), + months_data.append(l.shortMonths[calendar]), + months_data.append(l.longMonths[calendar]), + months_data.append(l.narrowMonths[calendar])) + + "// %s/%s/%s\n " % (l.language, l.script, l.country)) + calendar_temp_file.write(calendar_format % ( (0,) * 3 + ('0,0',) * 6 ) + + '// trailing zeros\n') + calendar_temp_file.write("};\n") + months_data.write(calendar_temp_file) + s = calendar_template_file.readline() + while s and s != GENERATED_BLOCK_END: + s = calendar_template_file.readline() + while s: + calendar_temp_file.write(s) + s = calendar_template_file.readline() + os.rename(calendar_temp_file_path, + os.path.join(qtsrcdir, 'src', 'corelib', 'time', calendar_data_file)) # qlocale.h (qlocaleh_temp_file, qlocaleh_temp_file_path) = tempfile.mkstemp("qlocale.h", dir=qtsrcdir) qlocaleh_temp_file = os.fdopen(qlocaleh_temp_file, "w") - qlocaleh_file = open(qtsrcdir + "/src/corelib/tools/qlocale.h", "r") + qlocaleh_file = open(qtsrcdir + "/src/corelib/text/qlocale.h", "r") s = qlocaleh_file.readline() while s and s != GENERATED_BLOCK_START: qlocaleh_temp_file.write(s) @@ -808,14 +860,14 @@ def main(): qlocaleh_temp_file.close() qlocaleh_file.close() - os.remove(qtsrcdir + "/src/corelib/tools/qlocale.h") - os.rename(qlocaleh_temp_file_path, qtsrcdir + "/src/corelib/tools/qlocale.h") + os.remove(qtsrcdir + "/src/corelib/text/qlocale.h") + os.rename(qlocaleh_temp_file_path, qtsrcdir + "/src/corelib/text/qlocale.h") # qlocale.qdoc (qlocaleqdoc_temp_file, qlocaleqdoc_temp_file_path) = tempfile.mkstemp("qlocale.qdoc", dir=qtsrcdir) qlocaleqdoc_temp_file = os.fdopen(qlocaleqdoc_temp_file, "w") - qlocaleqdoc_file = open(qtsrcdir + "/src/corelib/tools/qlocale.qdoc", "r") + qlocaleqdoc_file = open(qtsrcdir + "/src/corelib/text/qlocale.qdoc", "r") s = qlocaleqdoc_file.readline() DOCSTRING = " QLocale's data is based on Common Locale Data Repository " while s: @@ -827,8 +879,8 @@ def main(): qlocaleqdoc_temp_file.close() qlocaleqdoc_file.close() - os.remove(qtsrcdir + "/src/corelib/tools/qlocale.qdoc") - os.rename(qlocaleqdoc_temp_file_path, qtsrcdir + "/src/corelib/tools/qlocale.qdoc") + os.remove(qtsrcdir + "/src/corelib/text/qlocale.qdoc") + os.rename(qlocaleqdoc_temp_file_path, qtsrcdir + "/src/corelib/text/qlocale.qdoc") if __name__ == "__main__": main() diff --git a/util/local_database/testlocales/localemodel.cpp b/util/locale_database/testlocales/localemodel.cpp index d380d01e09..d380d01e09 100644 --- a/util/local_database/testlocales/localemodel.cpp +++ b/util/locale_database/testlocales/localemodel.cpp diff --git a/util/local_database/testlocales/localemodel.h b/util/locale_database/testlocales/localemodel.h index b24fc5f4c6..b24fc5f4c6 100644 --- a/util/local_database/testlocales/localemodel.h +++ b/util/locale_database/testlocales/localemodel.h diff --git a/util/local_database/testlocales/localewidget.cpp b/util/locale_database/testlocales/localewidget.cpp index 3ff7f73a98..3ff7f73a98 100644 --- a/util/local_database/testlocales/localewidget.cpp +++ b/util/locale_database/testlocales/localewidget.cpp diff --git a/util/local_database/testlocales/localewidget.h b/util/locale_database/testlocales/localewidget.h index 896a6e5229..896a6e5229 100644 --- a/util/local_database/testlocales/localewidget.h +++ b/util/locale_database/testlocales/localewidget.h diff --git a/util/local_database/testlocales/main.cpp b/util/locale_database/testlocales/main.cpp index 0c3c45f989..0c3c45f989 100644 --- a/util/local_database/testlocales/main.cpp +++ b/util/locale_database/testlocales/main.cpp diff --git a/util/local_database/testlocales/testlocales.pro b/util/locale_database/testlocales/testlocales.pro index a9a6247f96..a9a6247f96 100644 --- a/util/local_database/testlocales/testlocales.pro +++ b/util/locale_database/testlocales/testlocales.pro diff --git a/util/local_database/xpathlite.py b/util/locale_database/xpathlite.py index 218135d7a7..97efaaab41 100644 --- a/util/local_database/xpathlite.py +++ b/util/locale_database/xpathlite.py @@ -78,14 +78,38 @@ def findChild(parent, tag_name, arg_name=None, arg_value=None, draft=None): return node return False +def codeMapsFromFile(file): + """Extract mappings of language, script and country codes to names. + + The file shall typically be common/main/en.xml, which contains a + localeDisplayNames element with children languages, scripts and + territories; each element in each of these has a code as its type + attribute and its name as element content. This returns a mapping + withe keys 'language', 'script' and 'country', each of which + has, as value, a mapping of the relevant codes to names. + """ + parent = findChild(findChild(parseDoc(file), 'ldml'), 'localeDisplayNames') + keys, result = {'languages': 'language', 'scripts': 'script', 'territories': 'country'}, {} + for src, dst in keys.items(): + child = findChild(parent, src) + data = result[dst] = {} + for elt in child.childNodes: + if elt.attributes and elt.attributes.has_key('type'): + key, value = elt.attributes['type'].value, elt.childNodes[0].wholeText + # Don't over-write previously-read data for an alt form: + if elt.attributes.has_key('alt') and data.has_key(key): + continue + data[key] = value + + return result + def findTagsInFile(file, path): doc = parseDoc(file) elt = doc.documentElement tag_spec_list = path.split("/") last_entry = None - for i in range(len(tag_spec_list)): - tag_spec = tag_spec_list[i] + for tag_spec in tag_spec_list: tag_name = tag_spec arg_name = 'type' arg_value = '' diff --git a/util/qfloat16-tables/gen_qfloat16_tables.cpp b/util/qfloat16-tables/gen_qfloat16_tables.cpp new file mode 100644 index 0000000000..faedb5073c --- /dev/null +++ b/util/qfloat16-tables/gen_qfloat16_tables.cpp @@ -0,0 +1,163 @@ +/**************************************************************************** +** +** Copyright (C) 2016 by Southwest Research Institute (R) +** Copyright (C) 2019 Intel Corporation. +** Contact: http://www.qt-project.org/legal +** +** This file is part of the QtCore module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include <stdint.h> +#include <stdio.h> + +/* + * This tool generates the tables used by qfloat16 to implement a + * software-emulated version of IEEE 754 binary16. qfloat16 automatically uses + * CPU instructions to convert to and from float (IEEE 754 binary32), but if + * the CPU is not guaranteed to have those instructions available at compile + * time, then qfloat16 needs the tables to perform the conversion with + * reasonable performance. + * + * Because Qt requires float to be IEEE 754 binary32, these tables are + * platform-independent and will never change. + */ + +uint32_t convertmantissa(int32_t i) +{ + uint32_t m = i << 13; // Zero pad mantissa bits + uint32_t e = 0; // Zero exponent + + while (!(m & 0x00800000)) { // While not normalized + e -= 0x00800000; // Decrement exponent (1<<23) + m <<= 1; // Shift mantissa + } + m &= ~0x00800000; // Clear leading 1 bit + e += 0x38800000; // Adjust bias ((127-14)<<23) + return m | e; // Return combined number +} + +// we first build these tables up and then print them out as a separate step in order +// to more closely map the implementation given in the paper. +uint32_t basetable[512]; +uint32_t shifttable[512]; + +int main() +{ + uint32_t i; + + printf("/* This file was generated by util/qfloat16-tables/gen_qfloat16_tables.cpp */\n\n"); + printf("#include <QtCore/qfloat16.h>\n\n"); + + printf("QT_BEGIN_NAMESPACE\n\n"); + printf("#if !defined(__F16C__) && !defined(__ARM_FP16_FORMAT_IEEE)\n\n"); + + printf("const quint32 qfloat16::mantissatable[2048] = {\n"); + printf("0,\n"); + for (i = 1; i < 1024; i++) + printf("0x%XU,\n", convertmantissa(i)); + for (i = 1024; i < 2048; i++) + printf("0x%XU,\n", 0x38000000U + ((i - 1024) << 13)); + printf("};\n\n"); + + printf("const quint32 qfloat16::exponenttable[64] = {\n"); + printf("0,\n"); + for (i = 1; i < 31; i++) + printf("0x%XU,\n", i << 23); + printf("0x47800000U,\n"); // 31 + printf("0x80000000U,\n"); // 32 + for (i = 33; i < 63; i++) + printf("0x%XU,\n", 0x80000000U + ((i - 32) << 23)); + printf("0xC7800000U,\n"); // 63 + printf("};\n\n"); + + printf("const quint32 qfloat16::offsettable[64] = {\n"); + printf("0,\n"); + for (i = 1; i < 32; i++) + printf("1024U,\n"); + printf("0,\n"); + for (i = 33; i < 64; i++) + printf("1024U,\n"); + printf("};\n\n"); + + int32_t e; + for (i = 0; i < 256; ++i) { + e = i - 127; + if (e < -24) { // Very small numbers map to zero + basetable[i | 0x000] = 0x0000; + basetable[i | 0x100] = 0x8000; + shifttable[i | 0x000] = 24; + shifttable[i | 0x100] = 24; + + } else if (e < -14) { // Small numbers map to denorms + basetable[i | 0x000] = (0x0400 >> (-e - 14)); + basetable[i | 0x100] = (0x0400 >> (-e - 14)) | 0x8000; + shifttable[i | 0x000] = -e - 1; + shifttable[i | 0x100] = -e - 1; + + } else if (e <= 15) { // Normal numbers just lose precision + basetable[i | 0x000] = ((e + 15) << 10); + basetable[i | 0x100] = ((e + 15) << 10) | 0x8000; + shifttable[i | 0x000] = 13; + shifttable[i | 0x100] = 13; + + } else if (e < 128) { // Large numbers map to Infinity + basetable[i | 0x000] = 0x7C00; + basetable[i | 0x100] = 0xFC00; + shifttable[i | 0x000] = 24; + shifttable[i | 0x100] = 24; + + } else { // Infinity and NaN's stay Infinity and NaN's + basetable[i | 0x000] = 0x7C00; + basetable[i | 0x100] = 0xFC00; + shifttable[i | 0x000] = 13; + shifttable[i | 0x100] = 13; + } + } + + printf("const quint32 qfloat16::basetable[512] = {\n"); + for (i = 0; i < 512; i++) + printf("0x%XU,\n", basetable[i]); + + printf("};\n\n"); + + printf("const quint32 qfloat16::shifttable[512] = {\n"); + for (i = 0; i < 512; i++) + printf("0x%XU,\n", shifttable[i]); + + printf("};\n\n"); + + printf("#endif // !__F16C__ && !__ARM_FP16_FORMAT_IEEE\n\n"); + printf("QT_END_NAMESPACE\n"); + return 0; +} diff --git a/util/qfloat16-tables/qfloat16-tables.pro b/util/qfloat16-tables/qfloat16-tables.pro new file mode 100644 index 0000000000..3c9a1e61a3 --- /dev/null +++ b/util/qfloat16-tables/qfloat16-tables.pro @@ -0,0 +1,2 @@ +QT = core +SOURCES += gen_qfloat16_tables.cpp diff --git a/util/unicode/README b/util/unicode/README index e52f26175a..87f055d42d 100644 --- a/util/unicode/README +++ b/util/unicode/README @@ -19,7 +19,7 @@ To update: need to be expanded, for example. In some cases QChar may need additions to some of its enums. * Build with the modified code, fix any compilation issues. -* That may have updated qtbase/src/corelib/tools/qunicodetables.cpp; +* That may have updated qtbase/src/corelib/text/qunicodetables.cpp; if so the update matters; be sure to commit the changes to data/ at the same time and update tools/qt_attribution.json to match; use the UCD Revision number, rather than the Unicode standard number, as the diff --git a/util/unicode/codecs/big5/main.cpp b/util/unicode/codecs/big5/main.cpp index 142db74ffd..54d3839755 100644 --- a/util/unicode/codecs/big5/main.cpp +++ b/util/unicode/codecs/big5/main.cpp @@ -128,7 +128,7 @@ int main(int argc, char **argv) list += QByteArray(" { 0x" + QByteArray::number(m.b5, 16) + ", 0x" + QByteArray::number(m.uc, 16) + " }\n");; } QByteArray ba; - qSort(list); + std::sort(list.begin(), list.end()); foreach(QByteArray a, list) ba += a; qDebug() << "struct B5Map b5_to_uc_map = {\n" << ba + "\n};"; @@ -138,7 +138,7 @@ int main(int argc, char **argv) if (!b5_ok.contains(m.uc)) list += QByteArray(" { 0x" + QByteArray::number(m.uc, 16) + ", 0x" + QByteArray::number(m.b5, 16) + " }\n");; ba = QByteArray(); - qSort(list); + std::sort(list.begin(), list.end());; foreach(QByteArray a, list) ba += a; qDebug() << "struct B5Map uc_to_b5_map = {\n" << ba + "\n};"; diff --git a/util/unicode/main.cpp b/util/unicode/main.cpp index 22a4405ca9..26cdab87d6 100644 --- a/util/unicode/main.cpp +++ b/util/unicode/main.cpp @@ -789,6 +789,15 @@ static void initScriptMap() // Keep this one in sync with the code in createPropertyInfo static const char *property_string = + "enum Case {\n" + " LowerCase,\n" + " UpperCase,\n" + " TitleCase,\n" + " CaseFold,\n" + "\n" + " NumCases\n" + "};\n" + "\n" "struct Properties {\n" " ushort category : 8; /* 5 used */\n" " ushort direction : 8; /* 5 used */\n" @@ -796,84 +805,51 @@ static const char *property_string = " ushort joining : 3;\n" " signed short digitValue : 5;\n" " signed short mirrorDiff : 16;\n" - " ushort lowerCaseSpecial : 1;\n" - " signed short lowerCaseDiff : 15;\n" + " ushort unicodeVersion : 8; /* 5 used */\n" + " ushort nfQuickCheck : 8;\n" // could be narrowed "#ifdef Q_OS_WASM\n" " unsigned char : 0; //wasm 64 packing trick\n" "#endif\n" - " ushort upperCaseSpecial : 1;\n" - " signed short upperCaseDiff : 15;\n" - " ushort titleCaseSpecial : 1;\n" - " signed short titleCaseDiff : 15;\n" - " ushort caseFoldSpecial : 1;\n" - " signed short caseFoldDiff : 15;\n" - " ushort unicodeVersion : 8; /* 5 used */\n" - " ushort nfQuickCheck : 8;\n" // could be narrowed + " struct {\n" + " ushort special : 1;\n" + " signed short diff : 15;\n" + " } cases[NumCases];\n" "#ifdef Q_OS_WASM\n" " unsigned char : 0; //wasm 64 packing trick\n" "#endif\n" " ushort graphemeBreakClass : 5; /* 5 used */\n" " ushort wordBreakClass : 5; /* 5 used */\n" - " ushort sentenceBreakClass : 8; /* 4 used */\n" " ushort lineBreakClass : 6; /* 6 used */\n" + " ushort sentenceBreakClass : 8; /* 4 used */\n" " ushort script : 8;\n" "};\n\n" - "Q_CORE_EXPORT const Properties * QT_FASTCALL properties(uint ucs4) Q_DECL_NOTHROW;\n" - "Q_CORE_EXPORT const Properties * QT_FASTCALL properties(ushort ucs2) Q_DECL_NOTHROW;\n" - "\n" - "struct LowercaseTraits\n" - "{\n" - " static inline signed short caseDiff(const Properties *prop)\n" - " { return prop->lowerCaseDiff; }\n" - " static inline bool caseSpecial(const Properties *prop)\n" - " { return prop->lowerCaseSpecial; }\n" - "};\n" - "\n" - "struct UppercaseTraits\n" - "{\n" - " static inline signed short caseDiff(const Properties *prop)\n" - " { return prop->upperCaseDiff; }\n" - " static inline bool caseSpecial(const Properties *prop)\n" - " { return prop->upperCaseSpecial; }\n" - "};\n" - "\n" - "struct TitlecaseTraits\n" - "{\n" - " static inline signed short caseDiff(const Properties *prop)\n" - " { return prop->titleCaseDiff; }\n" - " static inline bool caseSpecial(const Properties *prop)\n" - " { return prop->titleCaseSpecial; }\n" - "};\n" - "\n" - "struct CasefoldTraits\n" - "{\n" - " static inline signed short caseDiff(const Properties *prop)\n" - " { return prop->caseFoldDiff; }\n" - " static inline bool caseSpecial(const Properties *prop)\n" - " { return prop->caseFoldSpecial; }\n" - "};\n" + "Q_CORE_EXPORT const Properties * QT_FASTCALL properties(uint ucs4) noexcept;\n" + "Q_CORE_EXPORT const Properties * QT_FASTCALL properties(ushort ucs2) noexcept;\n" "\n"; static const char *methods = - "Q_CORE_EXPORT GraphemeBreakClass QT_FASTCALL graphemeBreakClass(uint ucs4) Q_DECL_NOTHROW;\n" - "inline GraphemeBreakClass graphemeBreakClass(QChar ch) Q_DECL_NOTHROW\n" + "Q_CORE_EXPORT GraphemeBreakClass QT_FASTCALL graphemeBreakClass(uint ucs4) noexcept;\n" + "inline GraphemeBreakClass graphemeBreakClass(QChar ch) noexcept\n" "{ return graphemeBreakClass(ch.unicode()); }\n" "\n" - "Q_CORE_EXPORT WordBreakClass QT_FASTCALL wordBreakClass(uint ucs4) Q_DECL_NOTHROW;\n" - "inline WordBreakClass wordBreakClass(QChar ch) Q_DECL_NOTHROW\n" + "Q_CORE_EXPORT WordBreakClass QT_FASTCALL wordBreakClass(uint ucs4) noexcept;\n" + "inline WordBreakClass wordBreakClass(QChar ch) noexcept\n" "{ return wordBreakClass(ch.unicode()); }\n" "\n" - "Q_CORE_EXPORT SentenceBreakClass QT_FASTCALL sentenceBreakClass(uint ucs4) Q_DECL_NOTHROW;\n" - "inline SentenceBreakClass sentenceBreakClass(QChar ch) Q_DECL_NOTHROW\n" + "Q_CORE_EXPORT SentenceBreakClass QT_FASTCALL sentenceBreakClass(uint ucs4) noexcept;\n" + "inline SentenceBreakClass sentenceBreakClass(QChar ch) noexcept\n" "{ return sentenceBreakClass(ch.unicode()); }\n" "\n" - "Q_CORE_EXPORT LineBreakClass QT_FASTCALL lineBreakClass(uint ucs4) Q_DECL_NOTHROW;\n" - "inline LineBreakClass lineBreakClass(QChar ch) Q_DECL_NOTHROW\n" + "Q_CORE_EXPORT LineBreakClass QT_FASTCALL lineBreakClass(uint ucs4) noexcept;\n" + "inline LineBreakClass lineBreakClass(QChar ch) noexcept\n" "{ return lineBreakClass(ch.unicode()); }\n" "\n"; static const int SizeOfPropertiesStruct = 20; +static const QByteArray sizeOfPropertiesStructCheck = + "Q_STATIC_ASSERT(sizeof(Properties) == " + QByteArray::number(SizeOfPropertiesStruct) + ");\n\n"; + struct PropertyFlags { bool operator==(const PropertyFlags &o) const { return (combiningClass == o.combiningClass @@ -2470,48 +2446,45 @@ static QByteArray createPropertyInfo() // " signed short mirrorDiff : 16;\n" out += QByteArray::number( p.mirrorDiff ); out += ", "; -// " ushort lowerCaseSpecial : 1;\n" -// " signed short lowerCaseDiff : 15;\n" +// " ushort unicodeVersion : 8; /* 5 used */\n" + out += QByteArray::number( p.age ); + out += ", "; +// " ushort nfQuickCheck : 8;\n" + out += QByteArray::number( p.nfQuickCheck ); + out += ", "; +// " struct {\n" +// " ushort special : 1;\n" +// " signed short diff : 15;\n" +// " } cases[NumCases];\n" + out += " { {"; out += QByteArray::number( p.lowerCaseSpecial ); out += ", "; out += QByteArray::number( p.lowerCaseDiff ); - out += ", "; -// " ushort upperCaseSpecial : 1;\n" -// " signed short upperCaseDiff : 15;\n" + out += "}, {"; out += QByteArray::number( p.upperCaseSpecial ); out += ", "; out += QByteArray::number( p.upperCaseDiff ); - out += ", "; -// " ushort titleCaseSpecial : 1;\n" -// " signed short titleCaseDiff : 15;\n" + out += "}, {"; out += QByteArray::number( p.titleCaseSpecial ); out += ", "; out += QByteArray::number( p.titleCaseDiff ); - out += ", "; -// " ushort caseFoldSpecial : 1;\n" -// " signed short caseFoldDiff : 15;\n" + out += "}, {"; out += QByteArray::number( p.caseFoldSpecial ); out += ", "; out += QByteArray::number( p.caseFoldDiff ); - out += ", "; -// " ushort unicodeVersion : 8; /* 5 used */\n" - out += QByteArray::number( p.age ); - out += ", "; -// " ushort nfQuickCheck : 8;\n" - out += QByteArray::number( p.nfQuickCheck ); - out += ", "; + out += "} }, "; // " ushort graphemeBreakClass : 5; /* 5 used */\n" // " ushort wordBreakClass : 5; /* 5 used */\n" -// " ushort sentenceBreakClass : 8; /* 4 used */\n" // " ushort lineBreakClass : 6; /* 6 used */\n" out += QByteArray::number( p.graphemeBreakClass ); out += ", "; out += QByteArray::number( p.wordBreakClass ); out += ", "; - out += QByteArray::number( p.sentenceBreakClass ); - out += ", "; out += QByteArray::number( p.lineBreakClass ); out += ", "; +// " ushort sentenceBreakClass : 8; /* 4 used */\n" + out += QByteArray::number( p.sentenceBreakClass ); + out += ", "; // " ushort script : 8;\n" out += QByteArray::number( p.script ); out += " },"; @@ -2521,42 +2494,42 @@ static QByteArray createPropertyInfo() out += "\n};\n\n"; - out += "Q_DECL_CONST_FUNCTION static inline const Properties *qGetProp(uint ucs4) Q_DECL_NOTHROW\n" + out += "Q_DECL_CONST_FUNCTION static inline const Properties *qGetProp(uint ucs4) noexcept\n" "{\n" " return uc_properties + GET_PROP_INDEX(ucs4);\n" "}\n" "\n" - "Q_DECL_CONST_FUNCTION static inline const Properties *qGetProp(ushort ucs2) Q_DECL_NOTHROW\n" + "Q_DECL_CONST_FUNCTION static inline const Properties *qGetProp(ushort ucs2) noexcept\n" "{\n" " return uc_properties + GET_PROP_INDEX_UCS2(ucs2);\n" "}\n" "\n" - "Q_DECL_CONST_FUNCTION Q_CORE_EXPORT const Properties * QT_FASTCALL properties(uint ucs4) Q_DECL_NOTHROW\n" + "Q_DECL_CONST_FUNCTION Q_CORE_EXPORT const Properties * QT_FASTCALL properties(uint ucs4) noexcept\n" "{\n" " return qGetProp(ucs4);\n" "}\n" "\n" - "Q_DECL_CONST_FUNCTION Q_CORE_EXPORT const Properties * QT_FASTCALL properties(ushort ucs2) Q_DECL_NOTHROW\n" + "Q_DECL_CONST_FUNCTION Q_CORE_EXPORT const Properties * QT_FASTCALL properties(ushort ucs2) noexcept\n" "{\n" " return qGetProp(ucs2);\n" "}\n\n"; - out += "Q_CORE_EXPORT GraphemeBreakClass QT_FASTCALL graphemeBreakClass(uint ucs4) Q_DECL_NOTHROW\n" + out += "Q_CORE_EXPORT GraphemeBreakClass QT_FASTCALL graphemeBreakClass(uint ucs4) noexcept\n" "{\n" " return static_cast<GraphemeBreakClass>(qGetProp(ucs4)->graphemeBreakClass);\n" "}\n" "\n" - "Q_CORE_EXPORT WordBreakClass QT_FASTCALL wordBreakClass(uint ucs4) Q_DECL_NOTHROW\n" + "Q_CORE_EXPORT WordBreakClass QT_FASTCALL wordBreakClass(uint ucs4) noexcept\n" "{\n" " return static_cast<WordBreakClass>(qGetProp(ucs4)->wordBreakClass);\n" "}\n" "\n" - "Q_CORE_EXPORT SentenceBreakClass QT_FASTCALL sentenceBreakClass(uint ucs4) Q_DECL_NOTHROW\n" + "Q_CORE_EXPORT SentenceBreakClass QT_FASTCALL sentenceBreakClass(uint ucs4) noexcept\n" "{\n" " return static_cast<SentenceBreakClass>(qGetProp(ucs4)->sentenceBreakClass);\n" "}\n" "\n" - "Q_CORE_EXPORT LineBreakClass QT_FASTCALL lineBreakClass(uint ucs4) Q_DECL_NOTHROW\n" + "Q_CORE_EXPORT LineBreakClass QT_FASTCALL lineBreakClass(uint ucs4) noexcept\n" "{\n" " return static_cast<LineBreakClass>(qGetProp(ucs4)->lineBreakClass);\n" "}\n" @@ -2832,7 +2805,7 @@ static QByteArray createLigatureInfo() QList<Ligature> l = ligatureHashes.value(uc); if (!l.isEmpty()) { Q_ASSERT(!QChar::requiresSurrogates(uc)); - qSort(l); // needed for bsearch in ligatureHelper code + std::sort(l.begin(), l.end()); // needed for bsearch in ligatureHelper code ligatures.append(l.size()); for (int j = 0; j < l.size(); ++j) { @@ -2864,7 +2837,7 @@ static QByteArray createLigatureInfo() QList<Ligature> l = ligatureHashes.value(uc); if (!l.isEmpty()) { Q_ASSERT(QChar::requiresSurrogates(uc)); - qSort(l); // needed for bsearch in ligatureHelper code + std::sort(l.begin(), l.end()); // needed for bsearch in ligatureHelper code ligatures.append(l.size()); for (int j = 0; j < l.size(); ++j) { @@ -3096,7 +3069,7 @@ int main(int, char **) "// We mean it.\n" "//\n\n"; - QFile f("../../src/corelib/tools/qunicodetables.cpp"); + QFile f("../../src/corelib/text/qunicodetables.cpp"); f.open(QFile::WriteOnly|QFile::Truncate); f.write(header); f.write(note); @@ -3116,7 +3089,7 @@ int main(int, char **) f.write("QT_END_NAMESPACE\n"); f.close(); - f.setFileName("../../src/corelib/tools/qunicodetables_p.h"); + f.setFileName("../../src/corelib/text/qunicodetables_p.h"); f.open(QFile::WriteOnly | QFile::Truncate); f.write(header); f.write(note); @@ -3129,6 +3102,7 @@ int main(int, char **) f.write("#define UNICODE_DATA_VERSION " DATA_VERSION_STR "\n\n"); f.write("namespace QUnicodeTables {\n\n"); f.write(property_string); + f.write(sizeOfPropertiesStructCheck); f.write(grapheme_break_class_string); f.write(word_break_class_string); f.write(sentence_break_class_string); |