Add support for calendars beside Gregorian

Add QCalendarBackend as a base class for calendar implementations and
QCalendar as a facade via which to access it.

QDate's implicit implementation of the Gregorian calendar becomes
QGregorianCalendar and QDate methods now support choice of calendar.

Convert QLocale's CLDR data for month names to a locale-data component
of each supported calendar and relevant QLocale methods now support
choice of calendar. Adapt Python scripts for locale data generation to
extract month name data from CLDR (keeping on version v35.1) into the
new calendar-locale files. The locale data for the Gregorian calendar
is held in a Roman calendar base, for sharing with other calendars.

Add tests for basic uses of the new API.

[ChangeLog][QtCore][QCalendar] Added QCalendar to support diverse
calendars, supported by implementing QCalendarBackend.

[ChangeLog][QtCore][QDate] Allow choice of calendar in various
operations, with Gregorian remaining the default.

Done-with: Lars Knoll <lars.knoll@qt.io>
Done-with: Edward Welbourne <edward.welbourne@qt.io>
Fixes: QTBUG-17110
Fixes: QTBUG-950
Change-Id: I9d6278f394269a183aee8156e990cec4d5198ab8
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

3 files changed, 138 insertions, 54 deletions

diff --git a/util/locale_database/cldr2qlocalexml.py b/util/locale_database/cldr2qlocalexml.py
index 1982e3ba27..4bc735976b 100755
--- a/util/locale_database/cldr2qlocalexml.py
+++ b/util/locale_database/cldr2qlocalexml.py
@@ -1,7 +1,8 @@
 #!/usr/bin/env python2
+# coding=utf8
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of the test suite of the Qt Toolkit.
@@ -62,6 +63,8 @@ from xpathlite import DraftResolution, findAlias, findEntry, findTagsInFile
 from dateconverter import convert_date
 from localexml import Locale
 
+# TODO: make calendars a command-line option
+calendars = ['gregorian'] # 'persian', 'islamic', 'hebrew'
 findEntryInFile = xpathlite._findEntryInFile
 def wrappedwarn(prefix, tokens):
     return sys.stderr.write(
@@ -395,12 +398,12 @@ def _generateLocaleInfo(path, language_code, script_code, country_code, variant_
         ('narrow', 'format', 'narrow'),
         )
 
-    # Month data:
-    for cal in ('gregorian',): # We shall want to add to this
+    # Month names for 12-month calendars:
+    for cal in calendars:
         stem = 'dates/calendars/calendar[' + cal + ']/months/'
         for (key, mode, size) in namings:
             prop = 'monthContext[' + mode + ']/monthWidth[' + size + ']/'
-            result[key + 'Months'] = ';'.join(
+            result[key + 'Months_' + cal] = ';'.join(
                 findEntry(path, stem + prop + "month[%d]" % i)
                 for i in range(1, 13)) + ';'
 
@@ -686,9 +689,9 @@ if skips:
     wrappedwarn('skipping likelySubtags (for unknown language codes): ', skips)
 print "    <localeList>"
 
-Locale.C().toXml()
+Locale.C(calendars).toXml(calendars)
 for key in locale_keys:
-    locale_database[key].toXml()
+    locale_database[key].toXml(calendars)
 
 print "    </localeList>"
 print "</localeDatabase>"
diff --git a/util/locale_database/localexml.py b/util/locale_database/localexml.py
index e95b3aebcc..c83f9cea21 100644
--- a/util/locale_database/localexml.py
+++ b/util/locale_database/localexml.py
@@ -1,6 +1,7 @@
+# coding=utf8
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of the test suite of the Qt Toolkit.
@@ -113,12 +114,11 @@ def convertFormat(format):
     return result
 
 class Locale:
-    # Tool used during class body (see del below), not method:
-    def propsMonthDay(lengths=('long', 'short', 'narrow'), scale=('months', 'days')):
+    @staticmethod
+    def propsMonthDay(scale, lengths=('long', 'short', 'narrow')):
         for L in lengths:
-            for S in scale:
-                yield camelCase((L, S))
-                yield camelCase(('standalone', L, S))
+            yield camelCase((L, scale))
+            yield camelCase(('standalone', L, scale))
 
     # Expected to be numbers, read with int():
     __asint = ("decimal", "group", "zero",
@@ -137,15 +137,13 @@ class Locale:
                "listPatternPartEnd", "listPatternPartTwo", "am", "pm",
                'byte_unit', 'byte_si_quantified', 'byte_iec_quantified',
                "currencyIsoCode", "currencySymbol", "currencyDisplayName",
-               "currencyFormat", "currencyNegativeFormat"
-               ) + tuple(propsMonthDay())
-    del propsMonthDay
+               "currencyFormat", "currencyNegativeFormat")
 
     # Day-of-Week numbering used by Qt:
     __qDoW = {"mon": 1, "tue": 2, "wed": 3, "thu": 4, "fri": 5, "sat": 6, "sun": 7}
 
     @classmethod
-    def fromXmlData(cls, lookup):
+    def fromXmlData(cls, lookup, calendars=('gregorian',)):
         """Constructor from the contents of XML elements.
 
         Single parameter, lookup, is called with the names of XML
@@ -170,12 +168,15 @@ class Locale:
         for k in cls.__asfmt:
             data[k] = convertFormat(lookup(k))
 
-        for k in cls.__astxt:
+        for k in cls.__astxt + tuple(cls.propsMonthDay('days')):
             data[k] = lookup(k)
 
+        for k in cls.propsMonthDay('months'):
+            data[k] = dict((cal, lookup('_'.join((k, cal)))) for cal in calendars)
+
         return cls(data)
 
-    def toXml(self, indent='        ', tab='    '):
+    def toXml(self, calendars=('gregorian',), indent='        ', tab='    '):
         print indent + '<locale>'
         inner = indent + tab
         get = lambda k: getattr(self, k)
@@ -199,13 +200,14 @@ class Locale:
                     'weekendStart', 'weekendEnd',
                     'longDateFormat', 'shortDateFormat',
                     'longTimeFormat', 'shortTimeFormat',
-                    'standaloneLongMonths', 'standaloneShortMonths',
-                    'standaloneNarrowMonths',
-                    'longMonths', 'shortMonths', 'narrowMonths',
                     'longDays', 'shortDays', 'narrowDays',
                     'standaloneLongDays', 'standaloneShortDays', 'standaloneNarrowDays',
                     'currencyIsoCode', 'currencySymbol', 'currencyDisplayName',
-                    'currencyFormat', 'currencyNegativeFormat'):
+                    'currencyFormat', 'currencyNegativeFormat'
+                    ) + tuple(self.propsMonthDay('days')) + tuple(
+                '_'.join((k, cal))
+                for k in self.propsMonthDay('months')
+                for cal in calendars):
             ent = camelCase(key.split('_')) if key.endswith('_endonym') else key
             print inner + "<%s>%s</%s>" % (ent, escape(get(key)).encode('utf-8'), ent)
 
@@ -218,16 +220,50 @@ class Locale:
         if data: self.__dict__.update(data)
         if kw: self.__dict__.update(kw)
 
+    # Tools used by __monthNames:
+    def fullName(i, name): return name
+    def firstThree(i, name): return name[:3]
+    def initial(i, name): return name[:1]
+    def number(i, name): return str(i + 1)
+    @staticmethod
+    def __monthNames(calendars,
+                     known={ # Map calendar to (names, extractors...):
+            'gregorian': (('January', 'February', 'March', 'April', 'May', 'June', 'July',
+                           'August', 'September', 'October', 'November', 'December'),
+                          # Extractor pairs, (plain, standalone)
+                          (fullName, fullName), # long
+                          (firstThree, firstThree), # short
+                          (number, initial)), # narrow
+            'hebrew': (('Tishri', 'Heshvan', 'Kislev', 'Tevet', 'Shevat', 'Adar I',
+                        'Adar', 'Nisan', 'Iyar', 'Sivan', 'Tamuz', 'Av'),
+                       (fullName, fullName),
+                       (fullName, fullName),
+                       (number, number)),
+            },
+                     sizes=('long', 'short', 'narrow')):
+        for cal in calendars:
+            try:
+                data = known[cal]
+            except KeyError: # Need to add an entry to known, above.
+                print 'Unsupported calendar:', cal
+                raise
+            names, get = data[0] + ('',), data[1:]
+            for n, size in enumerate(sizes):
+                yield ('_'.join((camelCase((size, 'months')), cal)),
+                       ';'.join(get[n][0](i, x) for i, x in enumerate(names)))
+                yield ('_'.join((camelCase(('standalone', size, 'months')), cal)),
+                       ';'.join(get[n][1](i, x) for i, x in enumerate(names)))
+    del fullName, firstThree, initial, number
+
     @classmethod
-    def C(cls,
-          # Empty entries at end to ensure final separator when join()ed:
-          months = ('January', 'February', 'March', 'April', 'May', 'June', 'July',
-                    'August', 'September', 'October', 'November', 'December', ''),
+    def C(cls, calendars=('gregorian',),
+          # Empty entry at end to ensure final separator when join()ed:
           days = ('Sunday', 'Monday', 'Tuesday', 'Wednesday',
                   'Thursday', 'Friday', 'Saturday', ''),
           quantifiers=('k', 'M', 'G', 'T', 'P', 'E')):
         """Returns an object representing the C locale."""
-        return cls(language='C', language_code='0', language_endonym='',
+        return cls(dict(cls.__monthNames(calendars)),
+                   language='C', language_code='0', language_endonym='',
                    script='AnyScript', script_code='0',
                    country='AnyCountry', country_code='0', country_endonym='',
                    decimal='.', group=',', list=';', percent='%',
@@ -245,12 +281,6 @@ class Locale:
                    weekendStart='sat', weekendEnd='sun',
                    longDateFormat='EEEE, d MMMM yyyy', shortDateFormat='d MMM yyyy',
                    longTimeFormat='HH:mm:ss z', shortTimeFormat='HH:mm:ss',
-                   longMonths=';'.join(months),
-                   shortMonths=';'.join(m[:3] for m in months),
-                   narrowMonths='1;2;3;4;5;6;7;8;9;10;11;12;',
-                   standaloneLongMonths=';'.join(months),
-                   standaloneShortMonths=';'.join(m[:3] for m in months),
-                   standaloneNarrowMonths=';'.join(m[:1] for m in months),
                    longDays=';'.join(days),
                    shortDays=';'.join(d[:3] for d in days),
                    narrowDays='7;1;2;3;4;5;6;',
diff --git a/util/locale_database/qlocalexml2cpp.py b/util/locale_database/qlocalexml2cpp.py
index c0797f111d..641a80baf5 100755
--- a/util/locale_database/qlocalexml2cpp.py
+++ b/util/locale_database/qlocalexml2cpp.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python2
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of the test suite of the Qt Toolkit.
@@ -42,6 +42,10 @@ from enumdata import language_aliases, country_aliases, script_aliases
 
 from localexml import Locale
 
+# TODO: Make calendars a command-line parameter
+# map { CLDR name: Qt file name }
+calendars = {'gregorian': 'roman',} # 'persian': 'jalali', 'islamic': 'hijri', 'hebrew': 'hebrew',
+
 generated_template = """
 /*
     This part of the file was generated on %s from the
@@ -165,7 +169,7 @@ def loadLocaleMap(doc, language_map, script_map, country_map, likely_subtags_map
     result = {}
 
     for locale_elt in eachEltInGroup(doc.documentElement, "localeList", "locale"):
-        locale = Locale.fromXmlData(lambda k: firstChildText(locale_elt, k))
+        locale = Locale.fromXmlData(lambda k: firstChildText(locale_elt, k), calendars.keys())
         language_id = languageNameToId(locale.language, language_map)
         if language_id == -1:
             sys.stderr.write("Cannot find a language id for '%s'\n" % locale.language)
@@ -268,6 +272,7 @@ class StringData:
         self.data = []
         self.hash = {}
         self.name = name
+
     def append(self, s):
         if s in self.hash:
             return self.hash[s]
@@ -293,6 +298,11 @@ class StringData:
         self.data += lst
         return token
 
+    def write(self, fd):
+        fd.write("\nstatic const ushort %s[] = {\n" % self.name)
+        fd.write(wrap_list(self.data))
+        fd.write("\n};\n")
+
 def escapedString(s):
     result = ""
     i = 0
@@ -443,7 +453,6 @@ def main():
     list_pattern_part_data = StringData('list_pattern_part_data')
     date_format_data = StringData('date_format_data')
     time_format_data = StringData('time_format_data')
-    months_data = StringData('months_data')
     days_data = StringData('days_data')
     am_data = StringData('am_data')
     pm_data = StringData('pm_data')
@@ -483,12 +492,6 @@ def main():
                          + '   lDtFmt   '
                          + '   sTmFmt   ' # Time format
                          + '   lTmFmt   '
-                         + '  ssMonth   ' # Months
-                         + '  slMonth   '
-                         + '  snMonth   '
-                         + '   sMonth   '
-                         + '   lMonth   '
-                         + '   nMonth   '
                          + '   ssDays   ' # Days
                          + '   slDays   '
                          + '   snDays   '
@@ -533,7 +536,7 @@ def main():
                    # Quotation marks:
                    + '%8d,' * 4
                    # List patterns, date/time formats, month/day names, am/pm:
-                   + '%11s,' * 22
+                   + '%11s,' * 16
                    # SI/IEC byte-unit abbreviations:
                    + '%8s,' * 3
                    # Currency ISO code:
@@ -569,12 +572,6 @@ def main():
                         date_format_data.append(l.longDateFormat),
                         time_format_data.append(l.shortTimeFormat),
                         time_format_data.append(l.longTimeFormat),
-                        months_data.append(l.standaloneShortMonths),
-                        months_data.append(l.standaloneLongMonths),
-                        months_data.append(l.standaloneNarrowMonths),
-                        months_data.append(l.shortMonths),
-                        months_data.append(l.longMonths),
-                        months_data.append(l.narrowMonths),
                         days_data.append(l.standaloneShortDays),
                         days_data.append(l.standaloneLongDays),
                         days_data.append(l.standaloneNarrowDays),
@@ -600,7 +597,7 @@ def main():
                         l.weekendEnd)
                              + ", // %s/%s/%s\n" % (l.language, l.script, l.country))
     data_temp_file.write(line_format # All zeros, matching the format:
-                         % ( (0,) * (3 + 8 + 4) + ("0,0",) * (22 + 3)
+                         % ( (0,) * (3 + 8 + 4) + ("0,0",) * (16 + 3)
                              + (currencyIsoCodeData(0),)
                              + ("0,0",) * 6 + (0,) * (2 + 3))
                          + " // trailing 0s\n")
@@ -608,13 +605,11 @@ def main():
 
     # StringData tables:
     for data in (list_pattern_part_data, date_format_data,
-                 time_format_data, months_data, days_data,
+                 time_format_data, days_data,
                  byte_unit_data, am_data, pm_data, currency_symbol_data,
                  currency_display_name_data, currency_format_data,
                  endonyms_data):
-        data_temp_file.write("\nstatic const ushort %s[] = {\n" % data.name)
-        data_temp_file.write(wrap_list(data.data))
-        data_temp_file.write("\n};\n")
+        data.write(data_temp_file)
 
     data_temp_file.write("\n")
 
@@ -739,6 +734,62 @@ def main():
     os.remove(qtsrcdir + "/src/corelib/text/qlocale_data_p.h")
     os.rename(data_temp_file_path, qtsrcdir + "/src/corelib/text/qlocale_data_p.h")
 
+    # Generate calendar data
+    calendar_format = '      {%6d,%6d,%6d,{%5s},{%5s},{%5s},{%5s},{%5s},{%5s}}, '
+    for calendar, stem in calendars.items():
+        months_data = StringData('months_data')
+        calendar_data_file = "q%scalendar_data_p.h" % stem
+        calendar_template_file = open(os.path.join(qtsrcdir, 'src', 'corelib', 'time',
+                                                   calendar_data_file), "r")
+        (calendar_temp_file, calendar_temp_file_path) = tempfile.mkstemp(calendar_data_file, dir=qtsrcdir)
+        calendar_temp_file = os.fdopen(calendar_temp_file, "w")
+        s = calendar_template_file.readline()
+        while s and s != GENERATED_BLOCK_START:
+            calendar_temp_file.write(s)
+            s = calendar_template_file.readline()
+        calendar_temp_file.write(GENERATED_BLOCK_START)
+        calendar_temp_file.write(generated_template % (datetime.date.today(), cldr_version))
+        calendar_temp_file.write("static const QCalendarLocale locale_data[] = {\n")
+        calendar_temp_file.write('   // '
+                                 # IDs, width 7 (6 + comma)
+                                 + ' lang  '
+                                 + ' script'
+                                 + ' terr  '
+                                 # Month-name start-end pairs, width 8 (5 plus '{},'):
+                                     + ' sShort '
+                                 + ' sLong  '
+                                 + ' sNarrow'
+                                 + ' short  '
+                                 + ' long   '
+                                 + ' narrow'
+                                 # No trailing space on last; be sure
+                                 # to pad before adding later entries.
+                                 + '\n')
+        for key in locale_keys:
+            l = locale_map[key]
+            calendar_temp_file.write(
+                calendar_format
+                % (key[0], key[1], key[2],
+                   months_data.append(l.standaloneShortMonths[calendar]),
+                   months_data.append(l.standaloneLongMonths[calendar]),
+                   months_data.append(l.standaloneNarrowMonths[calendar]),
+                   months_data.append(l.shortMonths[calendar]),
+                   months_data.append(l.longMonths[calendar]),
+                   months_data.append(l.narrowMonths[calendar]))
+                + "// %s/%s/%s\n " % (l.language, l.script, l.country))
+        calendar_temp_file.write(calendar_format % ( (0,) * 3 + ('0,0',) * 6 )
+                                      + '// trailing zeros\n')
+        calendar_temp_file.write("};\n")
+        months_data.write(calendar_temp_file)
+        s = calendar_template_file.readline()
+        while s and s != GENERATED_BLOCK_END:
+            s = calendar_template_file.readline()
+        while s:
+            calendar_temp_file.write(s)
+            s = calendar_template_file.readline()
+        os.rename(calendar_temp_file_path,
+                  os.path.join(qtsrcdir, 'src', 'corelib', 'time', calendar_data_file))
+
     # qlocale.h
 
     (qlocaleh_temp_file, qlocaleh_temp_file_path) = tempfile.mkstemp("qlocale.h", dir=qtsrcdir)