Fix a bug in the generation of the Unicode joining property data

The code that was supposed to initialize the joining property of
characters to the correct defaults was actually applied after reading
in the data from the Unicode file, and was in a couple of cases
overwriting explicitly specified data in ArabicShaping.txt

Task-number: QTBUG-63191
Change-Id: Ie35261039b2211a827322ca11afacd9555ccefc7
Reviewed-by: Konstantin Ritt <ritt.ks@gmail.com>

3 files changed, 28 insertions, 26 deletions

diff --git a/src/corelib/tools/qunicodetables.cpp b/src/corelib/tools/qunicodetables.cpp
index ecab750833..01fa8b2102 100644
--- a/src/corelib/tools/qunicodetables.cpp
+++ b/src/corelib/tools/qunicodetables.cpp
@@ -6083,9 +6083,9 @@ static const Properties uc_properties[] = {
     { 18, 1, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 8, 13, 7 },
     { 25, 1, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 8, 12, 7 },
     { 25, 1, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 12, 7 },
-    { 10, 5, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 4, 4, 12, 8 },
-    { 10, 5, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 4, 4, 12, 8 },
-    { 10, 5, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3, 4, 4, 12, 2 },
+    { 10, 5, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 4, 4, 12, 8 },
+    { 10, 5, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 4, 4, 12, 8 },
+    { 10, 5, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3, 4, 4, 12, 2 },
     { 26, 10, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 12, 8 },
     { 26, 13, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 12, 8 },
     { 25, 4, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 10, 8 },
@@ -6146,7 +6146,7 @@ static const Properties uc_properties[] = {
     { 18, 13, 0, 3, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8, 8, 12, 8 },
     { 25, 13, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 12, 6, 8 },
     { 0, 17, 230, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4, 4, 21, 8 },
-    { 10, 5, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4, 4, 12, 2 },
+    { 10, 5, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4, 4, 12, 2 },
     { 29, 10, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 12, 8 },
     { 0, 17, 220, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4, 4, 21, 8 },
     { 17, 13, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 8, 12, 8 },
@@ -6749,7 +6749,7 @@ static const Properties uc_properties[] = {
     { 25, 10, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 12, 6, 33 },
     { 25, 10, 0, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 12, 33 },
     { 0, 17, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 4, 4, 21, 33 },
-    { 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 4, 4, 4, 33 },
+    { 10, 18, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 4, 4, 4, 33 },
     { 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 14, 9, 11, 33 },
     { 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 14, 9, 11, 33 },
     { 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 14, 9, 11, 33 },
@@ -7058,7 +7058,7 @@ static const Properties uc_properties[] = {
     { 6, 9, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 80, 0, 0, 5, 17, 2 },
     { 6, 9, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 80, 0, 0, 5, 4, 2 },
     { 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4, 20, 2 },
-    { 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4, 4, 21, 1 },
+    { 10, 18, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4, 4, 21, 1 },
     { 10, 18, 0, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4, 4, 21, 1 },
     { 10, 0, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4, 4, 21, 2 },
     { 10, 1, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4, 4, 21, 2 },
@@ -7108,10 +7108,10 @@ static const Properties uc_properties[] = {
     { 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 4, 4, 12, 2 },
     { 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3, 4, 4, 12, 2 },
     { 13, 18, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 12, 0 },
-    { 10, 19, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
-    { 10, 20, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
-    { 10, 21, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
-    { 10, 22, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
+    { 10, 19, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
+    { 10, 20, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
+    { 10, 21, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
+    { 10, 22, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
     { 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4, 4, 21, 2 },
     { 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 80, 0, 0, 0, 12, 2 },
     { 17, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 80, 0, 8, 6, 12, 3 },
diff --git a/src/corelib/tools/qunicodetables_p.h b/src/corelib/tools/qunicodetables_p.h
index 5a422ea4eb..be2f44f0c3 100644
--- a/src/corelib/tools/qunicodetables_p.h
+++ b/src/corelib/tools/qunicodetables_p.h
@@ -50,11 +50,11 @@
 // We mean it.
 //
 
-#include <QtCore/private/qglobal_p.h>
-
 #ifndef QUNICODETABLES_P_H
 #define QUNICODETABLES_P_H
 
+#include <QtCore/private/qglobal_p.h>
+
 #include <QtCore/qchar.h>
 
 QT_BEGIN_NAMESPACE
diff --git a/util/unicode/main.cpp b/util/unicode/main.cpp
index c995a40343..fe6d4fbca1 100644
--- a/util/unicode/main.cpp
+++ b/util/unicode/main.cpp
@@ -1293,6 +1293,18 @@ static void readArabicShaping()
 {
     qDebug("Reading ArabicShaping.txt");
 
+    // Initialize defaults:
+    // Code points that are not explicitly listed in ArabicShaping.txt are either of joining type T or U:
+    // - Those that not explicitly listed that are of General Category Mn, Me, or Cf have joining type T.
+    // - All others not explicitly listed have joining type U.
+    for (int codepoint = 0; codepoint <= QChar::LastValidCodePoint; ++codepoint) {
+        UnicodeData &d = UnicodeData::valueRef(codepoint);
+        if (d.p.joining == QChar::Joining_None) {
+            if (d.p.category == QChar::Mark_NonSpacing || d.p.category == QChar::Mark_Enclosing || d.p.category == QChar::Other_Format)
+                d.p.joining = QChar::Joining_Transparent;
+        }
+    }
+
     QFile f("data/ArabicShaping.txt");
     if (!f.exists())
         qFatal("Couldn't find ArabicShaping.txt");
@@ -1338,17 +1350,6 @@ static void readArabicShaping()
             break;
         }
     }
-
-    // Code points that are not explicitly listed in ArabicShaping.txt are either of joining type T or U:
-    // - Those that not explicitly listed that are of General Category Mn, Me, or Cf have joining type T.
-    // - All others not explicitly listed have joining type U.
-    for (int codepoint = 0; codepoint <= QChar::LastValidCodePoint; ++codepoint) {
-        UnicodeData &d = UnicodeData::valueRef(codepoint);
-        if (d.p.joining == QChar::Joining_None) {
-            if (d.p.category == QChar::Mark_NonSpacing || d.p.category == QChar::Mark_Enclosing || d.p.category == QChar::Other_Format)
-                d.p.joining = QChar::Joining_Transparent;
-        }
-    }
 }
 
 static void readDerivedAge()
@@ -2525,7 +2526,7 @@ static QByteArray createSpecialCaseMap()
     out.chop(1);
     out += "\n};\n\n";
 
-    qDebug("    memory usage: %d bytes", specialCaseMap.size()*sizeof(unsigned short));
+    qDebug("    memory usage: %ld bytes", specialCaseMap.size()*sizeof(unsigned short));
 
     return out;
 }
@@ -3021,7 +3022,7 @@ int main(int, char **)
         "****************************************************************************/\n\n";
 
     QByteArray note =
-        "/* This file is autogenerated from the Unicode "DATA_VERSION_S" database. Do not edit */\n\n";
+        "/* This file is autogenerated from the Unicode " DATA_VERSION_S " database. Do not edit */\n\n";
 
     QByteArray warning =
         "//\n"
@@ -3062,9 +3063,10 @@ int main(int, char **)
     f.write(warning);
     f.write("#ifndef QUNICODETABLES_P_H\n"
             "#define QUNICODETABLES_P_H\n\n"
+            "#include <QtCore/private/qglobal_p.h>\n\n"
             "#include <QtCore/qchar.h>\n\n"
             "QT_BEGIN_NAMESPACE\n\n");
-    f.write("#define UNICODE_DATA_VERSION "DATA_VERSION_STR"\n\n");
+    f.write("#define UNICODE_DATA_VERSION " DATA_VERSION_STR "\n\n");
     f.write("namespace QUnicodeTables {\n\n");
     f.write(property_string);
     f.write(grapheme_break_class_string);