From dec451cf676d1175e6913abf7600d45fa3f8c699 Mon Sep 17 00:00:00 2001 From: Konstantin Ritt Date: Tue, 24 Mar 2015 01:01:06 +0400 Subject: Update Unicode data up to v7.0 * Two newly adopted currency symbols: the Azerbaijan manat and the Russia ruble * Pictographic symbols (including many emoji), geometric symbols, arrows, and ornaments originating from the Wingdings and Webdings sets * Twenty-three new lesser-used and historic scripts extending support for written languages of North America, China, India, other Asian countries, and Africa * Letters used in Teuthonista and other transcriptional systems, and a new notational set, Duployan For more details, see http://www.unicode.org/versions/Unicode7.0.0/ The Properties struct's .*Diff members were narrowed down to signed 15 bits and the unicodeVersion has been expanded to 8 bits. [ChangeLog][QtCore] Unicode data updated to v.7.0 Change-Id: I93ab6f79fa3b05f61abc7279f1d046834c1c1a0b Reviewed-by: Lars Knoll --- util/unicode/main.cpp | 104 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 37 deletions(-) (limited to 'util') diff --git a/util/unicode/main.cpp b/util/unicode/main.cpp index af193c3bc2..2f6e28bd9f 100644 --- a/util/unicode/main.cpp +++ b/util/unicode/main.cpp @@ -43,8 +43,8 @@ #include #endif -#define DATA_VERSION_S "6.3" -#define DATA_VERSION_STR "QChar::Unicode_6_3" +#define DATA_VERSION_S "7.0" +#define DATA_VERSION_STR "QChar::Unicode_7_0" static QHash age_map; @@ -70,6 +70,7 @@ static void initAgeMap() { QChar::Unicode_6_1, "6.1" }, { QChar::Unicode_6_2, "6.2" }, { QChar::Unicode_6_3, "6.3" }, + { QChar::Unicode_7_0, "7.0" }, { QChar::Unicode_Unassigned, 0 } }; AgeMap *d = ageMap; @@ -694,6 +695,30 @@ static void initScriptMap() { QChar::Script_Sharada, "Sharada" }, { QChar::Script_SoraSompeng, "SoraSompeng" }, { QChar::Script_Takri, "Takri" }, + // 7.0 + { QChar::Script_CaucasianAlbanian, "CaucasianAlbanian" }, + { QChar::Script_BassaVah, "BassaVah" }, + { QChar::Script_Duployan, "Duployan" }, + { QChar::Script_Elbasan, "Elbasan" }, + { QChar::Script_Grantha, "Grantha" }, + { QChar::Script_PahawhHmong, "PahawhHmong" }, + { QChar::Script_Khojki, "Khojki" }, + { QChar::Script_LinearA, "LinearA" }, + { QChar::Script_Mahajani, "Mahajani" }, + { QChar::Script_Manichaean, "Manichaean" }, + { QChar::Script_MendeKikakui, "MendeKikakui" }, + { QChar::Script_Modi, "Modi" }, + { QChar::Script_Mro, "Mro" }, + { QChar::Script_OldNorthArabian, "OldNorthArabian" }, + { QChar::Script_Nabataean, "Nabataean" }, + { QChar::Script_Palmyrene, "Palmyrene" }, + { QChar::Script_PauCinHau, "PauCinHau" }, + { QChar::Script_OldPermic, "OldPermic" }, + { QChar::Script_PsalterPahlavi, "PsalterPahlavi" }, + { QChar::Script_Siddham, "Siddham" }, + { QChar::Script_Khudawadi, "Khudawadi" }, + { QChar::Script_Tirhuta, "Tirhuta" }, + { QChar::Script_WarangCiti, "WarangCiti" }, // unhandled { QChar::Script_Unknown, 0 } }; @@ -712,23 +737,23 @@ static const char *property_string = " ushort direction : 8; /* 5 used */\n" " ushort combiningClass : 8;\n" " ushort joining : 3;\n" - " signed short digitValue : 5; /* 5 used */\n" + " signed short digitValue : 5;\n" " signed short mirrorDiff : 16;\n" - " signed short lowerCaseDiff : 16;\n" - " signed short upperCaseDiff : 16;\n" - " signed short titleCaseDiff : 16;\n" - " signed short caseFoldDiff : 16;\n" " ushort lowerCaseSpecial : 1;\n" + " signed short lowerCaseDiff : 15;\n" " ushort upperCaseSpecial : 1;\n" + " signed short upperCaseDiff : 15;\n" " ushort titleCaseSpecial : 1;\n" + " signed short titleCaseDiff : 15;\n" " ushort caseFoldSpecial : 1;\n" - " ushort unicodeVersion : 4;\n" + " signed short caseFoldDiff : 15;\n" + " ushort unicodeVersion : 8; /* 5 used */\n" " ushort nfQuickCheck : 8;\n" // could be narrowed " ushort graphemeBreakClass : 4; /* 4 used */\n" " ushort wordBreakClass : 4; /* 4 used */\n" " ushort sentenceBreakClass : 8; /* 4 used */\n" " ushort lineBreakClass : 8; /* 6 used */\n" - " ushort script : 8; /* 7 used */\n" + " ushort script : 8;\n" "};\n\n" "Q_CORE_EXPORT const Properties * QT_FASTCALL properties(uint ucs4) Q_DECL_NOTHROW;\n" "Q_CORE_EXPORT const Properties * QT_FASTCALL properties(ushort ucs2) Q_DECL_NOTHROW;\n" @@ -786,7 +811,7 @@ struct PropertyFlags { // from ArabicShaping.txt QChar::JoiningType joining : 3; // from DerivedAge.txt - QChar::UnicodeVersion age : 4; + QChar::UnicodeVersion age : 5; int digitValue; int mirrorDiff : 16; @@ -849,6 +874,7 @@ static int appendToSpecialCaseMap(const QList &map) return pos; } +// DerivedCoreProperties.txt static inline bool isDefaultIgnorable(uint ucs4) { // Default_Ignorable_Code_Point: @@ -860,9 +886,11 @@ static inline bool isDefaultIgnorable(uint ucs4) return ucs4 == 0xad; return ucs4 == 0x034f + || ucs4 == 0x061c || (ucs4 >= 0x115f && ucs4 <= 0x1160) || (ucs4 >= 0x17b4 && ucs4 <= 0x17b5) || (ucs4 >= 0x180b && ucs4 <= 0x180d) + || ucs4 == 0x180e || (ucs4 >= 0x200b && ucs4 <= 0x200f) || (ucs4 >= 0x202a && ucs4 <= 0x202e) || (ucs4 >= 0x2060 && ucs4 <= 0x206f) @@ -871,7 +899,9 @@ static inline bool isDefaultIgnorable(uint ucs4) || ucs4 == 0xfeff || ucs4 == 0xffa0 || (ucs4 >= 0xfff0 && ucs4 <= 0xfff8) - || (ucs4 >= 0x1d173 && ucs4 <= 0xe0fff && (ucs4 <= 0x1d17a || ucs4 >= 0xe0000)); + || (ucs4 >= 0x1bca0 && ucs4 <= 0x1bca3) + || (ucs4 >= 0x1d173 && ucs4 <= 0x1d17a) + || (ucs4 >= 0xe0000 && ucs4 <= 0xe0fff); } struct UnicodeData { @@ -1092,7 +1122,7 @@ static void readUnicodeData() int upperCase = properties[UD_UpperCase].toInt(&ok, 16); Q_ASSERT(ok); int diff = upperCase - codepoint; - if (qAbs(diff) >= (1<<14)) { + if (qAbs(diff) >= (1<<13)) { qWarning() << "upperCaseDiff exceeded (" << hex << codepoint << "->" << upperCase << "); map it for special case"; // if the condition below doesn't hold anymore we need to modify our special upper casing code in qchar.cpp Q_ASSERT(!QChar::requiresSurrogates(codepoint) && !QChar::requiresSurrogates(upperCase)); @@ -1112,7 +1142,7 @@ static void readUnicodeData() int lowerCase = properties[UD_LowerCase].toInt(&ok, 16); Q_ASSERT(ok); int diff = lowerCase - codepoint; - if (qAbs(diff) >= (1<<14)) { + if (qAbs(diff) >= (1<<13)) { qWarning() << "lowerCaseDiff exceeded (" << hex << codepoint << "->" << lowerCase << "); map it for special case"; // if the condition below doesn't hold anymore we need to modify our special lower casing code in qchar.cpp Q_ASSERT(!QChar::requiresSurrogates(codepoint) && !QChar::requiresSurrogates(lowerCase)); @@ -1135,7 +1165,7 @@ static void readUnicodeData() int titleCase = properties[UD_TitleCase].toInt(&ok, 16); Q_ASSERT(ok); int diff = titleCase - codepoint; - if (qAbs(diff) >= (1<<14)) { + if (qAbs(diff) >= (1<<13)) { qWarning() << "titleCaseDiff exceeded (" << hex << codepoint << "->" << titleCase << "); map it for special case"; // if the condition below doesn't hold anymore we need to modify our special title casing code in qchar.cpp Q_ASSERT(!QChar::requiresSurrogates(codepoint) && !QChar::requiresSurrogates(titleCase)); @@ -1701,7 +1731,7 @@ static void readCaseFolding() if (foldMap.size() == 1) { int caseFolded = foldMap.at(0); int diff = caseFolded - codepoint; - if (qAbs(diff) >= (1<<14)) { + if (qAbs(diff) >= (1<<13)) { qWarning() << "caseFoldDiff exceeded (" << hex << codepoint << "->" << caseFolded << "); map it for special case"; // if the condition below doesn't hold anymore we need to modify our special case folding code in qchar.cpp Q_ASSERT(!QChar::requiresSurrogates(codepoint) && !QChar::requiresSurrogates(caseFolded)); @@ -2182,10 +2212,10 @@ static QByteArray createPropertyInfo() // we reserve one bit more than in the assert below for the sign Q_ASSERT(maxMirroredDiff < (1<<12)); - Q_ASSERT(maxLowerCaseDiff < (1<<14)); - Q_ASSERT(maxUpperCaseDiff < (1<<14)); - Q_ASSERT(maxTitleCaseDiff < (1<<14)); - Q_ASSERT(maxCaseFoldDiff < (1<<14)); + Q_ASSERT(maxLowerCaseDiff < (1<<13)); + Q_ASSERT(maxUpperCaseDiff < (1<<13)); + Q_ASSERT(maxTitleCaseDiff < (1<<13)); + Q_ASSERT(maxCaseFoldDiff < (1<<13)); const int BMP_BLOCKSIZE = 32; const int BMP_SHIFT = 5; @@ -2337,37 +2367,37 @@ static QByteArray createPropertyInfo() // " ushort joining : 3;\n" out += QByteArray::number( p.joining ); out += ", "; -// " signed short digitValue : 5; /* 5 used */\n" +// " signed short digitValue : 5;\n" out += QByteArray::number( p.digitValue ); out += ", "; // " signed short mirrorDiff : 16;\n" -// " signed short lowerCaseDiff : 16;\n" -// " signed short upperCaseDiff : 16;\n" -// " signed short titleCaseDiff : 16;\n" -// " signed short caseFoldDiff : 16;\n" out += QByteArray::number( p.mirrorDiff ); out += ", "; - out += QByteArray::number( p.lowerCaseDiff ); - out += ", "; - out += QByteArray::number( p.upperCaseDiff ); - out += ", "; - out += QByteArray::number( p.titleCaseDiff ); - out += ", "; - out += QByteArray::number( p.caseFoldDiff ); - out += ", "; // " ushort lowerCaseSpecial : 1;\n" -// " ushort upperCaseSpecial : 1;\n" -// " ushort titleCaseSpecial : 1;\n" -// " ushort caseFoldSpecial : 1;\n" +// " signed short lowerCaseDiff : 15;\n" out += QByteArray::number( p.lowerCaseSpecial ); out += ", "; + out += QByteArray::number( p.lowerCaseDiff ); + out += ", "; +// " ushort upperCaseSpecial : 1;\n" +// " signed short upperCaseDiff : 15;\n" out += QByteArray::number( p.upperCaseSpecial ); out += ", "; + out += QByteArray::number( p.upperCaseDiff ); + out += ", "; +// " ushort titleCaseSpecial : 1;\n" +// " signed short titleCaseDiff : 15;\n" out += QByteArray::number( p.titleCaseSpecial ); out += ", "; + out += QByteArray::number( p.titleCaseDiff ); + out += ", "; +// " ushort caseFoldSpecial : 1;\n" +// " signed short caseFoldDiff : 15;\n" out += QByteArray::number( p.caseFoldSpecial ); out += ", "; -// " ushort unicodeVersion : 4;\n" + out += QByteArray::number( p.caseFoldDiff ); + out += ", "; +// " ushort unicodeVersion : 8; /* 5 used */\n" out += QByteArray::number( p.age ); out += ", "; // " ushort nfQuickCheck : 8;\n" @@ -2385,7 +2415,7 @@ static QByteArray createPropertyInfo() out += ", "; out += QByteArray::number( p.lineBreakClass ); out += ", "; -// " ushort script : 8; /* 7 used */\n" +// " ushort script : 8;\n" out += QByteArray::number( p.script ); out += " },"; } -- cgit v1.2.3