Merge remote-tracking branch 'origin/wip/qt6' into wip/cmake

Change-Id: I3a1d7673c3c20019ab12a2ea0a60f1619920a34c

3 files changed, 173 insertions, 68 deletions

diff --git a/util/locale_database/cldr2qlocalexml.py b/util/locale_database/cldr2qlocalexml.py
index 1982e3ba27..072ea9e4ed 100755
--- a/util/locale_database/cldr2qlocalexml.py
+++ b/util/locale_database/cldr2qlocalexml.py
@@ -1,7 +1,8 @@
 #!/usr/bin/env python2
+# coding=utf8
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of the test suite of the Qt Toolkit.
@@ -62,6 +63,8 @@ from xpathlite import DraftResolution, findAlias, findEntry, findTagsInFile
 from dateconverter import convert_date
 from localexml import Locale
 
+# TODO: make calendars a command-line option
+calendars = ['gregorian', 'persian', 'islamic'] # 'hebrew'
 findEntryInFile = xpathlite._findEntryInFile
 def wrappedwarn(prefix, tokens):
     return sys.stderr.write(
@@ -395,12 +398,12 @@ def _generateLocaleInfo(path, language_code, script_code, country_code, variant_
         ('narrow', 'format', 'narrow'),
         )
 
-    # Month data:
-    for cal in ('gregorian',): # We shall want to add to this
+    # Month names for 12-month calendars:
+    for cal in calendars:
         stem = 'dates/calendars/calendar[' + cal + ']/months/'
         for (key, mode, size) in namings:
             prop = 'monthContext[' + mode + ']/monthWidth[' + size + ']/'
-            result[key + 'Months'] = ';'.join(
+            result[key + 'Months_' + cal] = ';'.join(
                 findEntry(path, stem + prop + "month[%d]" % i)
                 for i in range(1, 13)) + ';'
 
@@ -686,9 +689,9 @@ if skips:
     wrappedwarn('skipping likelySubtags (for unknown language codes): ', skips)
 print "    <localeList>"
 
-Locale.C().toXml()
+Locale.C(calendars).toXml(calendars)
 for key in locale_keys:
-    locale_database[key].toXml()
+    locale_database[key].toXml(calendars)
 
 print "    </localeList>"
 print "</localeDatabase>"
diff --git a/util/locale_database/localexml.py b/util/locale_database/localexml.py
index e95b3aebcc..9b353f5122 100644
--- a/util/locale_database/localexml.py
+++ b/util/locale_database/localexml.py
@@ -1,6 +1,7 @@
+# coding=utf8
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of the test suite of the Qt Toolkit.
@@ -113,12 +114,11 @@ def convertFormat(format):
     return result
 
 class Locale:
-    # Tool used during class body (see del below), not method:
-    def propsMonthDay(lengths=('long', 'short', 'narrow'), scale=('months', 'days')):
+    @staticmethod
+    def propsMonthDay(scale, lengths=('long', 'short', 'narrow')):
         for L in lengths:
-            for S in scale:
-                yield camelCase((L, S))
-                yield camelCase(('standalone', L, S))
+            yield camelCase((L, scale))
+            yield camelCase(('standalone', L, scale))
 
     # Expected to be numbers, read with int():
     __asint = ("decimal", "group", "zero",
@@ -137,15 +137,13 @@ class Locale:
                "listPatternPartEnd", "listPatternPartTwo", "am", "pm",
                'byte_unit', 'byte_si_quantified', 'byte_iec_quantified',
                "currencyIsoCode", "currencySymbol", "currencyDisplayName",
-               "currencyFormat", "currencyNegativeFormat"
-               ) + tuple(propsMonthDay())
-    del propsMonthDay
+               "currencyFormat", "currencyNegativeFormat")
 
     # Day-of-Week numbering used by Qt:
     __qDoW = {"mon": 1, "tue": 2, "wed": 3, "thu": 4, "fri": 5, "sat": 6, "sun": 7}
 
     @classmethod
-    def fromXmlData(cls, lookup):
+    def fromXmlData(cls, lookup, calendars=('gregorian',)):
         """Constructor from the contents of XML elements.
 
         Single parameter, lookup, is called with the names of XML
@@ -170,12 +168,15 @@ class Locale:
         for k in cls.__asfmt:
             data[k] = convertFormat(lookup(k))
 
-        for k in cls.__astxt:
+        for k in cls.__astxt + tuple(cls.propsMonthDay('days')):
             data[k] = lookup(k)
 
+        for k in cls.propsMonthDay('months'):
+            data[k] = dict((cal, lookup('_'.join((k, cal)))) for cal in calendars)
+
         return cls(data)
 
-    def toXml(self, indent='        ', tab='    '):
+    def toXml(self, calendars=('gregorian',), indent='        ', tab='    '):
         print indent + '<locale>'
         inner = indent + tab
         get = lambda k: getattr(self, k)
@@ -199,13 +200,14 @@ class Locale:
                     'weekendStart', 'weekendEnd',
                     'longDateFormat', 'shortDateFormat',
                     'longTimeFormat', 'shortTimeFormat',
-                    'standaloneLongMonths', 'standaloneShortMonths',
-                    'standaloneNarrowMonths',
-                    'longMonths', 'shortMonths', 'narrowMonths',
                     'longDays', 'shortDays', 'narrowDays',
                     'standaloneLongDays', 'standaloneShortDays', 'standaloneNarrowDays',
                     'currencyIsoCode', 'currencySymbol', 'currencyDisplayName',
-                    'currencyFormat', 'currencyNegativeFormat'):
+                    'currencyFormat', 'currencyNegativeFormat'
+                    ) + tuple(self.propsMonthDay('days')) + tuple(
+                '_'.join((k, cal))
+                for k in self.propsMonthDay('months')
+                for cal in calendars):
             ent = camelCase(key.split('_')) if key.endswith('_endonym') else key
             print inner + "<%s>%s</%s>" % (ent, escape(get(key)).encode('utf-8'), ent)
 
@@ -218,16 +220,70 @@ class Locale:
         if data: self.__dict__.update(data)
         if kw: self.__dict__.update(kw)
 
+    # Tools used by __monthNames:
+    def fullName(i, name): return name
+    def firstThree(i, name): return name[:3]
+    def initial(i, name): return name[:1]
+    def number(i, name): return str(i + 1)
+    def islamicShort(i, name):
+        if not name: return name
+        if name == 'Shawwal': return 'Shaw.'
+        words = name.split()
+        if words[0].startswith('Dhu'):
+            words[0] = words[0][:7] + '.'
+        elif len(words[0]) > 3:
+            words[0] = words[0][:3] + '.'
+        return ' '.join(words)
+    @staticmethod
+    def __monthNames(calendars,
+                     known={ # Map calendar to (names, extractors...):
+            'gregorian': (('January', 'February', 'March', 'April', 'May', 'June', 'July',
+                           'August', 'September', 'October', 'November', 'December'),
+                          # Extractor pairs, (plain, standalone)
+                          (fullName, fullName), # long
+                          (firstThree, firstThree), # short
+                          (number, initial)), # narrow
+            'persian': (('Farvardin', 'Ordibehesht', 'Khordad', 'Tir', 'Mordad',
+                         'Shahrivar', 'Mehr', 'Aban', 'Azar', 'Dey', 'Bahman', 'Esfand'),
+                        (fullName, fullName),
+                        (firstThree, firstThree),
+                        (number, initial)),
+            'islamic': ((u'Muharram', u'Safar', u'Rabiʻ I', u'Rabiʻ II', u'Jumada I',
+                         u'Jumada II', u'Rajab', u'Shaʻban', u'Ramadan', u'Shawwal',
+                         u'Dhuʻl-Qiʻdah', u'Dhuʻl-Hijjah'),
+                        (fullName, fullName),
+                        (islamicShort, islamicShort),
+                        (number, number)),
+            'hebrew': (('Tishri', 'Heshvan', 'Kislev', 'Tevet', 'Shevat', 'Adar I',
+                        'Adar', 'Nisan', 'Iyar', 'Sivan', 'Tamuz', 'Av'),
+                       (fullName, fullName),
+                       (fullName, fullName),
+                       (number, number)),
+            },
+                     sizes=('long', 'short', 'narrow')):
+        for cal in calendars:
+            try:
+                data = known[cal]
+            except KeyError: # Need to add an entry to known, above.
+                print 'Unsupported calendar:', cal
+                raise
+            names, get = data[0] + ('',), data[1:]
+            for n, size in enumerate(sizes):
+                yield ('_'.join((camelCase((size, 'months')), cal)),
+                       ';'.join(get[n][0](i, x) for i, x in enumerate(names)))
+                yield ('_'.join((camelCase(('standalone', size, 'months')), cal)),
+                       ';'.join(get[n][1](i, x) for i, x in enumerate(names)))
+    del fullName, firstThree, initial, number, islamicShort
+
     @classmethod
-    def C(cls,
-          # Empty entries at end to ensure final separator when join()ed:
-          months = ('January', 'February', 'March', 'April', 'May', 'June', 'July',
-                    'August', 'September', 'October', 'November', 'December', ''),
+    def C(cls, calendars=('gregorian',),
+          # Empty entry at end to ensure final separator when join()ed:
           days = ('Sunday', 'Monday', 'Tuesday', 'Wednesday',
                   'Thursday', 'Friday', 'Saturday', ''),
           quantifiers=('k', 'M', 'G', 'T', 'P', 'E')):
         """Returns an object representing the C locale."""
-        return cls(language='C', language_code='0', language_endonym='',
+        return cls(dict(cls.__monthNames(calendars)),
+                   language='C', language_code='0', language_endonym='',
                    script='AnyScript', script_code='0',
                    country='AnyCountry', country_code='0', country_endonym='',
                    decimal='.', group=',', list=';', percent='%',
@@ -245,12 +301,6 @@ class Locale:
                    weekendStart='sat', weekendEnd='sun',
                    longDateFormat='EEEE, d MMMM yyyy', shortDateFormat='d MMM yyyy',
                    longTimeFormat='HH:mm:ss z', shortTimeFormat='HH:mm:ss',
-                   longMonths=';'.join(months),
-                   shortMonths=';'.join(m[:3] for m in months),
-                   narrowMonths='1;2;3;4;5;6;7;8;9;10;11;12;',
-                   standaloneLongMonths=';'.join(months),
-                   standaloneShortMonths=';'.join(m[:3] for m in months),
-                   standaloneNarrowMonths=';'.join(m[:1] for m in months),
                    longDays=';'.join(days),
                    shortDays=';'.join(d[:3] for d in days),
                    narrowDays='7;1;2;3;4;5;6;',
diff --git a/util/locale_database/qlocalexml2cpp.py b/util/locale_database/qlocalexml2cpp.py
index ea89288b4f..a5ff7ebbf4 100755
--- a/util/locale_database/qlocalexml2cpp.py
+++ b/util/locale_database/qlocalexml2cpp.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of the test suite of the Qt Toolkit.
@@ -42,6 +42,24 @@ from enumdata import language_aliases, country_aliases, script_aliases
 
 from localexml import Locale
 
+# TODO: Make calendars a command-line parameter
+# map { CLDR name: Qt file name }
+calendars = {'gregorian': 'roman', 'persian': 'jalali', 'islamic': 'hijri',} # 'hebrew': 'hebrew',
+
+generated_template = """
+/*
+    This part of the file was generated on %s from the
+    Common Locale Data Repository v%s
+
+    http://www.unicode.org/cldr/
+
+    Do not edit this section: instead regenerate it using
+    cldr2qlocalexml.py and qlocalexml2cpp.py on updated (or
+    edited) CLDR data; see qtbase/util/locale_database/.
+*/
+
+"""
+
 class Error:
     def __init__(self, msg):
         self.msg = msg
@@ -151,7 +169,7 @@ def loadLocaleMap(doc, language_map, script_map, country_map, likely_subtags_map
     result = {}
 
     for locale_elt in eachEltInGroup(doc.documentElement, "localeList", "locale"):
-        locale = Locale.fromXmlData(lambda k: firstChildText(locale_elt, k))
+        locale = Locale.fromXmlData(lambda k: firstChildText(locale_elt, k), calendars.keys())
         language_id = languageNameToId(locale.language, language_map)
         if language_id == -1:
             sys.stderr.write("Cannot find a language id for '%s'\n" % locale.language)
@@ -254,6 +272,7 @@ class StringData:
         self.data = []
         self.hash = {}
         self.name = name
+
     def append(self, s):
         if s in self.hash:
             return self.hash[s]
@@ -279,6 +298,11 @@ class StringData:
         self.data += lst
         return token
 
+    def write(self, fd):
+        fd.write("\nstatic const ushort %s[] = {\n" % self.name)
+        fd.write(wrap_list(self.data))
+        fd.write("\n};\n")
+
 def escapedString(s):
     result = ""
     i = 0
@@ -360,20 +384,7 @@ def main():
     dupes = findDupes(language_map, country_map)
 
     cldr_version = firstChildText(doc.documentElement, "version")
-
-    data_temp_file.write("""
-/*
-    This part of the file was generated on %s from the
-    Common Locale Data Repository v%s
-
-    http://www.unicode.org/cldr/
-
-    Do not edit this section: instead regenerate it using
-    cldr2qlocalexml.py and qlocalexml2cpp.py on updated (or
-    edited) CLDR data; see qtbase/util/locale_database/.
-*/
-
-""" % (str(datetime.date.today()), cldr_version) )
+    data_temp_file.write(generated_template % (datetime.date.today(), cldr_version))
 
     # Likely subtags map
     data_temp_file.write("static const QLocaleId likely_subtags[] = {\n")
@@ -442,7 +453,6 @@ def main():
     list_pattern_part_data = StringData('list_pattern_part_data')
     date_format_data = StringData('date_format_data')
     time_format_data = StringData('time_format_data')
-    months_data = StringData('months_data')
     days_data = StringData('days_data')
     am_data = StringData('am_data')
     pm_data = StringData('pm_data')
@@ -482,12 +492,6 @@ def main():
                          + '   lDtFmt   '
                          + '   sTmFmt   ' # Time format
                          + '   lTmFmt   '
-                         + '  ssMonth   ' # Months
-                         + '  slMonth   '
-                         + '  snMonth   '
-                         + '   sMonth   '
-                         + '   lMonth   '
-                         + '   nMonth   '
                          + '   ssDays   ' # Days
                          + '   slDays   '
                          + '   snDays   '
@@ -532,7 +536,7 @@ def main():
                    # Quotation marks:
                    + '%8d,' * 4
                    # List patterns, date/time formats, month/day names, am/pm:
-                   + '%11s,' * 22
+                   + '%11s,' * 16
                    # SI/IEC byte-unit abbreviations:
                    + '%8s,' * 3
                    # Currency ISO code:
@@ -568,12 +572,6 @@ def main():
                         date_format_data.append(l.longDateFormat),
                         time_format_data.append(l.shortTimeFormat),
                         time_format_data.append(l.longTimeFormat),
-                        months_data.append(l.standaloneShortMonths),
-                        months_data.append(l.standaloneLongMonths),
-                        months_data.append(l.standaloneNarrowMonths),
-                        months_data.append(l.shortMonths),
-                        months_data.append(l.longMonths),
-                        months_data.append(l.narrowMonths),
                         days_data.append(l.standaloneShortDays),
                         days_data.append(l.standaloneLongDays),
                         days_data.append(l.standaloneNarrowDays),
@@ -599,7 +597,7 @@ def main():
                         l.weekendEnd)
                              + ", // %s/%s/%s\n" % (l.language, l.script, l.country))
     data_temp_file.write(line_format # All zeros, matching the format:
-                         % ( (0,) * (3 + 8 + 4) + ("0,0",) * (22 + 3)
+                         % ( (0,) * (3 + 8 + 4) + ("0,0",) * (16 + 3)
                              + (currencyIsoCodeData(0),)
                              + ("0,0",) * 6 + (0,) * (2 + 3))
                          + " // trailing 0s\n")
@@ -607,13 +605,11 @@ def main():
 
     # StringData tables:
     for data in (list_pattern_part_data, date_format_data,
-                 time_format_data, months_data, days_data,
+                 time_format_data, days_data,
                  byte_unit_data, am_data, pm_data, currency_symbol_data,
                  currency_display_name_data, currency_format_data,
                  endonyms_data):
-        data_temp_file.write("\nstatic const ushort %s[] = {\n" % data.name)
-        data_temp_file.write(wrap_list(data.data))
-        data_temp_file.write("\n};\n")
+        data.write(data_temp_file)
 
     data_temp_file.write("\n")
 
@@ -738,6 +734,62 @@ def main():
     os.remove(qtsrcdir + "/src/corelib/text/qlocale_data_p.h")
     os.rename(data_temp_file_path, qtsrcdir + "/src/corelib/text/qlocale_data_p.h")
 
+    # Generate calendar data
+    calendar_format = '      {%6d,%6d,%6d,{%5s},{%5s},{%5s},{%5s},{%5s},{%5s}}, '
+    for calendar, stem in calendars.items():
+        months_data = StringData('months_data')
+        calendar_data_file = "q%scalendar_data_p.h" % stem
+        calendar_template_file = open(os.path.join(qtsrcdir, 'src', 'corelib', 'time',
+                                                   calendar_data_file), "r")
+        (calendar_temp_file, calendar_temp_file_path) = tempfile.mkstemp(calendar_data_file, dir=qtsrcdir)
+        calendar_temp_file = os.fdopen(calendar_temp_file, "w")
+        s = calendar_template_file.readline()
+        while s and s != GENERATED_BLOCK_START:
+            calendar_temp_file.write(s)
+            s = calendar_template_file.readline()
+        calendar_temp_file.write(GENERATED_BLOCK_START)
+        calendar_temp_file.write(generated_template % (datetime.date.today(), cldr_version))
+        calendar_temp_file.write("static const QCalendarLocale locale_data[] = {\n")
+        calendar_temp_file.write('   // '
+                                 # IDs, width 7 (6 + comma)
+                                 + ' lang  '
+                                 + ' script'
+                                 + ' terr  '
+                                 # Month-name start-end pairs, width 8 (5 plus '{},'):
+                                     + ' sShort '
+                                 + ' sLong  '
+                                 + ' sNarrow'
+                                 + ' short  '
+                                 + ' long   '
+                                 + ' narrow'
+                                 # No trailing space on last; be sure
+                                 # to pad before adding later entries.
+                                 + '\n')
+        for key in locale_keys:
+            l = locale_map[key]
+            calendar_temp_file.write(
+                calendar_format
+                % (key[0], key[1], key[2],
+                   months_data.append(l.standaloneShortMonths[calendar]),
+                   months_data.append(l.standaloneLongMonths[calendar]),
+                   months_data.append(l.standaloneNarrowMonths[calendar]),
+                   months_data.append(l.shortMonths[calendar]),
+                   months_data.append(l.longMonths[calendar]),
+                   months_data.append(l.narrowMonths[calendar]))
+                + "// %s/%s/%s\n " % (l.language, l.script, l.country))
+        calendar_temp_file.write(calendar_format % ( (0,) * 3 + ('0,0',) * 6 )
+                                      + '// trailing zeros\n')
+        calendar_temp_file.write("};\n")
+        months_data.write(calendar_temp_file)
+        s = calendar_template_file.readline()
+        while s and s != GENERATED_BLOCK_END:
+            s = calendar_template_file.readline()
+        while s:
+            calendar_temp_file.write(s)
+            s = calendar_template_file.readline()
+        os.rename(calendar_temp_file_path,
+                  os.path.join(qtsrcdir, 'src', 'corelib', 'time', calendar_data_file))
+
     # qlocale.h
 
     (qlocaleh_temp_file, qlocaleh_temp_file_path) = tempfile.mkstemp("qlocale.h", dir=qtsrcdir)