From aa8393c94fea01a4806b204fd3aa343a4e90666b Mon Sep 17 00:00:00 2001 From: Soroush Rabiei Date: Sat, 14 Jan 2017 20:23:31 +0330 Subject: Add support for calendars beside Gregorian Add QCalendarBackend as a base class for calendar implementations and QCalendar as a facade via which to access it. QDate's implicit implementation of the Gregorian calendar becomes QGregorianCalendar and QDate methods now support choice of calendar. Convert QLocale's CLDR data for month names to a locale-data component of each supported calendar and relevant QLocale methods now support choice of calendar. Adapt Python scripts for locale data generation to extract month name data from CLDR (keeping on version v35.1) into the new calendar-locale files. The locale data for the Gregorian calendar is held in a Roman calendar base, for sharing with other calendars. Add tests for basic uses of the new API. [ChangeLog][QtCore][QCalendar] Added QCalendar to support diverse calendars, supported by implementing QCalendarBackend. [ChangeLog][QtCore][QDate] Allow choice of calendar in various operations, with Gregorian remaining the default. Done-with: Lars Knoll Done-with: Edward Welbourne Fixes: QTBUG-17110 Fixes: QTBUG-950 Change-Id: I9d6278f394269a183aee8156e990cec4d5198ab8 Reviewed-by: Volker Hilsheimer --- util/locale_database/cldr2qlocalexml.py | 15 +++--- util/locale_database/localexml.py | 84 +++++++++++++++++++---------- util/locale_database/qlocalexml2cpp.py | 93 +++++++++++++++++++++++++-------- 3 files changed, 138 insertions(+), 54 deletions(-) (limited to 'util/locale_database') diff --git a/util/locale_database/cldr2qlocalexml.py b/util/locale_database/cldr2qlocalexml.py index 1982e3ba27..4bc735976b 100755 --- a/util/locale_database/cldr2qlocalexml.py +++ b/util/locale_database/cldr2qlocalexml.py @@ -1,7 +1,8 @@ #!/usr/bin/env python2 +# coding=utf8 ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2018 The Qt Company Ltd. ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the test suite of the Qt Toolkit. @@ -62,6 +63,8 @@ from xpathlite import DraftResolution, findAlias, findEntry, findTagsInFile from dateconverter import convert_date from localexml import Locale +# TODO: make calendars a command-line option +calendars = ['gregorian'] # 'persian', 'islamic', 'hebrew' findEntryInFile = xpathlite._findEntryInFile def wrappedwarn(prefix, tokens): return sys.stderr.write( @@ -395,12 +398,12 @@ def _generateLocaleInfo(path, language_code, script_code, country_code, variant_ ('narrow', 'format', 'narrow'), ) - # Month data: - for cal in ('gregorian',): # We shall want to add to this + # Month names for 12-month calendars: + for cal in calendars: stem = 'dates/calendars/calendar[' + cal + ']/months/' for (key, mode, size) in namings: prop = 'monthContext[' + mode + ']/monthWidth[' + size + ']/' - result[key + 'Months'] = ';'.join( + result[key + 'Months_' + cal] = ';'.join( findEntry(path, stem + prop + "month[%d]" % i) for i in range(1, 13)) + ';' @@ -686,9 +689,9 @@ if skips: wrappedwarn('skipping likelySubtags (for unknown language codes): ', skips) print " " -Locale.C().toXml() +Locale.C(calendars).toXml(calendars) for key in locale_keys: - locale_database[key].toXml() + locale_database[key].toXml(calendars) print " " print "" diff --git a/util/locale_database/localexml.py b/util/locale_database/localexml.py index e95b3aebcc..c83f9cea21 100644 --- a/util/locale_database/localexml.py +++ b/util/locale_database/localexml.py @@ -1,6 +1,7 @@ +# coding=utf8 ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2018 The Qt Company Ltd. ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the test suite of the Qt Toolkit. @@ -113,12 +114,11 @@ def convertFormat(format): return result class Locale: - # Tool used during class body (see del below), not method: - def propsMonthDay(lengths=('long', 'short', 'narrow'), scale=('months', 'days')): + @staticmethod + def propsMonthDay(scale, lengths=('long', 'short', 'narrow')): for L in lengths: - for S in scale: - yield camelCase((L, S)) - yield camelCase(('standalone', L, S)) + yield camelCase((L, scale)) + yield camelCase(('standalone', L, scale)) # Expected to be numbers, read with int(): __asint = ("decimal", "group", "zero", @@ -137,15 +137,13 @@ class Locale: "listPatternPartEnd", "listPatternPartTwo", "am", "pm", 'byte_unit', 'byte_si_quantified', 'byte_iec_quantified', "currencyIsoCode", "currencySymbol", "currencyDisplayName", - "currencyFormat", "currencyNegativeFormat" - ) + tuple(propsMonthDay()) - del propsMonthDay + "currencyFormat", "currencyNegativeFormat") # Day-of-Week numbering used by Qt: __qDoW = {"mon": 1, "tue": 2, "wed": 3, "thu": 4, "fri": 5, "sat": 6, "sun": 7} @classmethod - def fromXmlData(cls, lookup): + def fromXmlData(cls, lookup, calendars=('gregorian',)): """Constructor from the contents of XML elements. Single parameter, lookup, is called with the names of XML @@ -170,12 +168,15 @@ class Locale: for k in cls.__asfmt: data[k] = convertFormat(lookup(k)) - for k in cls.__astxt: + for k in cls.__astxt + tuple(cls.propsMonthDay('days')): data[k] = lookup(k) + for k in cls.propsMonthDay('months'): + data[k] = dict((cal, lookup('_'.join((k, cal)))) for cal in calendars) + return cls(data) - def toXml(self, indent=' ', tab=' '): + def toXml(self, calendars=('gregorian',), indent=' ', tab=' '): print indent + '' inner = indent + tab get = lambda k: getattr(self, k) @@ -199,13 +200,14 @@ class Locale: 'weekendStart', 'weekendEnd', 'longDateFormat', 'shortDateFormat', 'longTimeFormat', 'shortTimeFormat', - 'standaloneLongMonths', 'standaloneShortMonths', - 'standaloneNarrowMonths', - 'longMonths', 'shortMonths', 'narrowMonths', 'longDays', 'shortDays', 'narrowDays', 'standaloneLongDays', 'standaloneShortDays', 'standaloneNarrowDays', 'currencyIsoCode', 'currencySymbol', 'currencyDisplayName', - 'currencyFormat', 'currencyNegativeFormat'): + 'currencyFormat', 'currencyNegativeFormat' + ) + tuple(self.propsMonthDay('days')) + tuple( + '_'.join((k, cal)) + for k in self.propsMonthDay('months') + for cal in calendars): ent = camelCase(key.split('_')) if key.endswith('_endonym') else key print inner + "<%s>%s" % (ent, escape(get(key)).encode('utf-8'), ent) @@ -218,16 +220,50 @@ class Locale: if data: self.__dict__.update(data) if kw: self.__dict__.update(kw) + # Tools used by __monthNames: + def fullName(i, name): return name + def firstThree(i, name): return name[:3] + def initial(i, name): return name[:1] + def number(i, name): return str(i + 1) + @staticmethod + def __monthNames(calendars, + known={ # Map calendar to (names, extractors...): + 'gregorian': (('January', 'February', 'March', 'April', 'May', 'June', 'July', + 'August', 'September', 'October', 'November', 'December'), + # Extractor pairs, (plain, standalone) + (fullName, fullName), # long + (firstThree, firstThree), # short + (number, initial)), # narrow + 'hebrew': (('Tishri', 'Heshvan', 'Kislev', 'Tevet', 'Shevat', 'Adar I', + 'Adar', 'Nisan', 'Iyar', 'Sivan', 'Tamuz', 'Av'), + (fullName, fullName), + (fullName, fullName), + (number, number)), + }, + sizes=('long', 'short', 'narrow')): + for cal in calendars: + try: + data = known[cal] + except KeyError: # Need to add an entry to known, above. + print 'Unsupported calendar:', cal + raise + names, get = data[0] + ('',), data[1:] + for n, size in enumerate(sizes): + yield ('_'.join((camelCase((size, 'months')), cal)), + ';'.join(get[n][0](i, x) for i, x in enumerate(names))) + yield ('_'.join((camelCase(('standalone', size, 'months')), cal)), + ';'.join(get[n][1](i, x) for i, x in enumerate(names))) + del fullName, firstThree, initial, number + @classmethod - def C(cls, - # Empty entries at end to ensure final separator when join()ed: - months = ('January', 'February', 'March', 'April', 'May', 'June', 'July', - 'August', 'September', 'October', 'November', 'December', ''), + def C(cls, calendars=('gregorian',), + # Empty entry at end to ensure final separator when join()ed: days = ('Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', ''), quantifiers=('k', 'M', 'G', 'T', 'P', 'E')): """Returns an object representing the C locale.""" - return cls(language='C', language_code='0', language_endonym='', + return cls(dict(cls.__monthNames(calendars)), + language='C', language_code='0', language_endonym='', script='AnyScript', script_code='0', country='AnyCountry', country_code='0', country_endonym='', decimal='.', group=',', list=';', percent='%', @@ -245,12 +281,6 @@ class Locale: weekendStart='sat', weekendEnd='sun', longDateFormat='EEEE, d MMMM yyyy', shortDateFormat='d MMM yyyy', longTimeFormat='HH:mm:ss z', shortTimeFormat='HH:mm:ss', - longMonths=';'.join(months), - shortMonths=';'.join(m[:3] for m in months), - narrowMonths='1;2;3;4;5;6;7;8;9;10;11;12;', - standaloneLongMonths=';'.join(months), - standaloneShortMonths=';'.join(m[:3] for m in months), - standaloneNarrowMonths=';'.join(m[:1] for m in months), longDays=';'.join(days), shortDays=';'.join(d[:3] for d in days), narrowDays='7;1;2;3;4;5;6;', diff --git a/util/locale_database/qlocalexml2cpp.py b/util/locale_database/qlocalexml2cpp.py index c0797f111d..641a80baf5 100755 --- a/util/locale_database/qlocalexml2cpp.py +++ b/util/locale_database/qlocalexml2cpp.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 ############################################################################# ## -## Copyright (C) 2017 The Qt Company Ltd. +## Copyright (C) 2018 The Qt Company Ltd. ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the test suite of the Qt Toolkit. @@ -42,6 +42,10 @@ from enumdata import language_aliases, country_aliases, script_aliases from localexml import Locale +# TODO: Make calendars a command-line parameter +# map { CLDR name: Qt file name } +calendars = {'gregorian': 'roman',} # 'persian': 'jalali', 'islamic': 'hijri', 'hebrew': 'hebrew', + generated_template = """ /* This part of the file was generated on %s from the @@ -165,7 +169,7 @@ def loadLocaleMap(doc, language_map, script_map, country_map, likely_subtags_map result = {} for locale_elt in eachEltInGroup(doc.documentElement, "localeList", "locale"): - locale = Locale.fromXmlData(lambda k: firstChildText(locale_elt, k)) + locale = Locale.fromXmlData(lambda k: firstChildText(locale_elt, k), calendars.keys()) language_id = languageNameToId(locale.language, language_map) if language_id == -1: sys.stderr.write("Cannot find a language id for '%s'\n" % locale.language) @@ -268,6 +272,7 @@ class StringData: self.data = [] self.hash = {} self.name = name + def append(self, s): if s in self.hash: return self.hash[s] @@ -293,6 +298,11 @@ class StringData: self.data += lst return token + def write(self, fd): + fd.write("\nstatic const ushort %s[] = {\n" % self.name) + fd.write(wrap_list(self.data)) + fd.write("\n};\n") + def escapedString(s): result = "" i = 0 @@ -443,7 +453,6 @@ def main(): list_pattern_part_data = StringData('list_pattern_part_data') date_format_data = StringData('date_format_data') time_format_data = StringData('time_format_data') - months_data = StringData('months_data') days_data = StringData('days_data') am_data = StringData('am_data') pm_data = StringData('pm_data') @@ -483,12 +492,6 @@ def main(): + ' lDtFmt ' + ' sTmFmt ' # Time format + ' lTmFmt ' - + ' ssMonth ' # Months - + ' slMonth ' - + ' snMonth ' - + ' sMonth ' - + ' lMonth ' - + ' nMonth ' + ' ssDays ' # Days + ' slDays ' + ' snDays ' @@ -533,7 +536,7 @@ def main(): # Quotation marks: + '%8d,' * 4 # List patterns, date/time formats, month/day names, am/pm: - + '%11s,' * 22 + + '%11s,' * 16 # SI/IEC byte-unit abbreviations: + '%8s,' * 3 # Currency ISO code: @@ -569,12 +572,6 @@ def main(): date_format_data.append(l.longDateFormat), time_format_data.append(l.shortTimeFormat), time_format_data.append(l.longTimeFormat), - months_data.append(l.standaloneShortMonths), - months_data.append(l.standaloneLongMonths), - months_data.append(l.standaloneNarrowMonths), - months_data.append(l.shortMonths), - months_data.append(l.longMonths), - months_data.append(l.narrowMonths), days_data.append(l.standaloneShortDays), days_data.append(l.standaloneLongDays), days_data.append(l.standaloneNarrowDays), @@ -600,7 +597,7 @@ def main(): l.weekendEnd) + ", // %s/%s/%s\n" % (l.language, l.script, l.country)) data_temp_file.write(line_format # All zeros, matching the format: - % ( (0,) * (3 + 8 + 4) + ("0,0",) * (22 + 3) + % ( (0,) * (3 + 8 + 4) + ("0,0",) * (16 + 3) + (currencyIsoCodeData(0),) + ("0,0",) * 6 + (0,) * (2 + 3)) + " // trailing 0s\n") @@ -608,13 +605,11 @@ def main(): # StringData tables: for data in (list_pattern_part_data, date_format_data, - time_format_data, months_data, days_data, + time_format_data, days_data, byte_unit_data, am_data, pm_data, currency_symbol_data, currency_display_name_data, currency_format_data, endonyms_data): - data_temp_file.write("\nstatic const ushort %s[] = {\n" % data.name) - data_temp_file.write(wrap_list(data.data)) - data_temp_file.write("\n};\n") + data.write(data_temp_file) data_temp_file.write("\n") @@ -739,6 +734,62 @@ def main(): os.remove(qtsrcdir + "/src/corelib/text/qlocale_data_p.h") os.rename(data_temp_file_path, qtsrcdir + "/src/corelib/text/qlocale_data_p.h") + # Generate calendar data + calendar_format = ' {%6d,%6d,%6d,{%5s},{%5s},{%5s},{%5s},{%5s},{%5s}}, ' + for calendar, stem in calendars.items(): + months_data = StringData('months_data') + calendar_data_file = "q%scalendar_data_p.h" % stem + calendar_template_file = open(os.path.join(qtsrcdir, 'src', 'corelib', 'time', + calendar_data_file), "r") + (calendar_temp_file, calendar_temp_file_path) = tempfile.mkstemp(calendar_data_file, dir=qtsrcdir) + calendar_temp_file = os.fdopen(calendar_temp_file, "w") + s = calendar_template_file.readline() + while s and s != GENERATED_BLOCK_START: + calendar_temp_file.write(s) + s = calendar_template_file.readline() + calendar_temp_file.write(GENERATED_BLOCK_START) + calendar_temp_file.write(generated_template % (datetime.date.today(), cldr_version)) + calendar_temp_file.write("static const QCalendarLocale locale_data[] = {\n") + calendar_temp_file.write(' // ' + # IDs, width 7 (6 + comma) + + ' lang ' + + ' script' + + ' terr ' + # Month-name start-end pairs, width 8 (5 plus '{},'): + + ' sShort ' + + ' sLong ' + + ' sNarrow' + + ' short ' + + ' long ' + + ' narrow' + # No trailing space on last; be sure + # to pad before adding later entries. + + '\n') + for key in locale_keys: + l = locale_map[key] + calendar_temp_file.write( + calendar_format + % (key[0], key[1], key[2], + months_data.append(l.standaloneShortMonths[calendar]), + months_data.append(l.standaloneLongMonths[calendar]), + months_data.append(l.standaloneNarrowMonths[calendar]), + months_data.append(l.shortMonths[calendar]), + months_data.append(l.longMonths[calendar]), + months_data.append(l.narrowMonths[calendar])) + + "// %s/%s/%s\n " % (l.language, l.script, l.country)) + calendar_temp_file.write(calendar_format % ( (0,) * 3 + ('0,0',) * 6 ) + + '// trailing zeros\n') + calendar_temp_file.write("};\n") + months_data.write(calendar_temp_file) + s = calendar_template_file.readline() + while s and s != GENERATED_BLOCK_END: + s = calendar_template_file.readline() + while s: + calendar_temp_file.write(s) + s = calendar_template_file.readline() + os.rename(calendar_temp_file_path, + os.path.join(qtsrcdir, 'src', 'corelib', 'time', calendar_data_file)) + # qlocale.h (qlocaleh_temp_file, qlocaleh_temp_file_path) = tempfile.mkstemp("qlocale.h", dir=qtsrcdir) -- cgit v1.2.3