From be3dfd7a71a276b10bac50075b26c6af58b9d02b Mon Sep 17 00:00:00 2001
From: Edward Welbourne <edward.welbourne@qt.io>
Date: Thu, 27 Feb 2020 13:58:58 +0100
Subject: Rework cldr2qlocalexml.py's reading of CLDR data

Move the code out to a CldrReader class in cldr.py, expand CldrAccess
with facilities that needs, expand ldml.py to include support for more
features, finally making xpathlite.py redundant. This initial commit
aims, though, to be bug-for-bug compatible with xpathlite in its
reading of the CLDR data.

It turns out we've been using draftier data than we were aware of
(which might not be a bad thing). The xpathlite code appeared to check
for draft attributes, but these only appear on leaf nodes and most
data were fetched by finding a parent and then scanning its children
without the draft check; only am/pm data was actually being excluded
based on draft values.  (We allowed contributed, for am/pm, in
addition to approved, which is all the xpathlite code allows
otherwise.) There are also some less equivocal bugs; I'll deal with
these in later commits.

Simplified number-system data look-ups; the old get_number_in_system()
was taking care of old LDML versions' placement of the number system
attribute; this is no longer needed. (It was also being used for a
currency value to which it was not appropriate, which is now handled
separately; this is one of the bugs mentioned above.) Ditched a
fall-back to nativeZeroDigit, which no longer exists in CLDR.

Change the command-line to take the root of the CLDR data tree, rather
than its common/main/ sub-directory. Support naming the file to which
to write output, as a second command-line argument, instead of always
writing to stdout (which remains the default) and leaving whoever runs
the script to redirect stdout.

Support (internally for now, while adding TODOs to give main() more
command-line options) separating the stderr output into its more and
less interesting parts; for now, continue producing both, but suppress
the least interesting entirely.

Task-number: QTBUG-81344
Change-Id: Ie611b47403a9452b51feaeeaaa0fbc8f7e84dc71
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
---
 util/locale_database/cldr.py            | 498 ++++++++++++++++++++++++-
 util/locale_database/cldr2qlocalexml.py | 636 +++-----------------------------
 util/locale_database/ldml.py            | 450 +++++++++++++++++++++-
 util/locale_database/qlocalexml2cpp.py  |   2 +-
 util/locale_database/xpathlite.py       | 284 --------------
 5 files changed, 972 insertions(+), 898 deletions(-)
 delete mode 100644 util/locale_database/xpathlite.py

diff --git a/util/locale_database/cldr.py b/util/locale_database/cldr.py
index 7890adf307..94459b9e3f 100644
--- a/util/locale_database/cldr.py
+++ b/util/locale_database/cldr.py
@@ -27,7 +27,8 @@
 #############################################################################
 """Digesting the CLDR's data.
 
-Provides two class:
+Provides two classes:
+  CldrReader -- driver for reading CLDR data
   CldrAccess -- used by the reader to access the tree of data files
 
 The former should normally be all you need to access.
@@ -38,9 +39,206 @@ from xml.dom import minidom
 from weakref import WeakValueDictionary as CacheDict
 import os
 
-from localetools import Error
-from ldml import Node, Supplement
+from ldml import Error, Node, XmlScanner, Supplement, LocaleScanner
+from qlocalexml import Locale
 
+class CldrReader (object):
+    def __init__(self, root, grumble = lambda msg: None, whitter = lambda msg: None):
+        """Set up a reader object for reading CLDR data.
+
+        Single parameter, root, is the file-system path to the root of
+        the unpacked CLDR archive; its common/ sub-directory should
+        contain dtd/, main/ and supplemental/ sub-directories.
+
+        Optional second argument, grumble, is a callable that logs
+        warnings and complaints, e.g. sys.stderr.write would be a
+        suitable callable.  The default is a no-op that ignores its
+        single argument.  Optional third argument is similar, used for
+        less interesting output; pass sys.stderr.write for it for
+        verbose output."""
+        self.root = CldrAccess(root)
+        self.whitter, self.grumble = whitter, grumble
+
+    def likelySubTags(self):
+        """Generator for likely subtag information.
+
+        Yields pairs (have, give) of 4-tuples; if what you have
+        matches the left member, giving the right member is probably
+        sensible. Each 4-tuple's entries are the full names of a
+        language, a script, a country (strictly territory) and a
+        variant (currently ignored)."""
+        skips = []
+        for got, use in self.root.likelySubTags():
+            try:
+                have = self.__parseTags(got)
+                give = self.__parseTags(use)
+            except Error as e:
+                if ((use.startswith(got) or got.startswith('und_'))
+                    and e.message.startswith('Unknown ') and ' code ' in e.message):
+                    skips.append(use)
+                else:
+                    self.grumble('Skipping likelySubtag "{}" -> "{}" ({})\n'.format(got, use, e.message))
+                continue
+            if all(code.startswith('Any') and code[3].isupper() for code in have[:-1]):
+                continue
+
+            give = (give[0],
+                    # Substitute according to http://www.unicode.org/reports/tr35/#Likely_Subtags
+                    have[1] if give[1] == 'AnyScript' else give[1],
+                    have[2] if give[2] == 'AnyCountry' else give[2],
+                    give[3]) # AnyVariant similarly ?
+
+            yield have, give
+
+        if skips:
+            # TODO: look at LDML's reserved locale tag names; they
+            # show up a lot in this, and may be grounds for filtering
+            # more out.
+            pass # self.__wrapped(self.whitter, 'Skipping likelySubtags (for unknown codes): ', skips)
+
+    def readLocales(self, calendars = ('gregorian',)):
+        locales = tuple(self.__allLocales(calendars))
+        return dict(((k.language_id, k.script_id, k.country_id, k.variant_code),
+                     k) for k in locales)
+
+    def __allLocales(self, calendars):
+        def skip(locale, reason):
+            return 'Skipping defaultContent locale "{}" ({})\n'.format(locale, reason)
+
+        for locale in self.root.defaultContentLocales:
+            try:
+                language, script, country, variant = self.__splitLocale(locale)
+            except ValueError:
+                self.whitter(skip(locale, 'only language tag'))
+                continue
+
+            if not (script or country):
+                self.grumble(skip(locale, 'second tag is neither script nor territory'))
+                continue
+
+            if not (language and country):
+                continue
+
+            try:
+                yield self.__getLocaleData(self.root.locale(locale), calendars,
+                                           language, script, country, variant)
+            except Error as e:
+                self.grumble(skip(locale, e.message))
+
+        for locale in self.root.fileLocales:
+            try:
+                chain = self.root.locale(locale)
+                language, script, country, variant = chain.tagCodes()
+                assert language
+                # TODO: this skip should probably be based on likely
+                # sub-tags, instead of empty country: if locale has a
+                # likely-subtag expansion, that's what QLocale uses,
+                # and we'll be saving its data for the expanded locale
+                # anyway, so don't need to record it for itself.
+                # See also QLocaleXmlReader.loadLocaleMap's grumble.
+                if not country:
+                    continue
+                yield self.__getLocaleData(chain, calendars, language, script, country, variant)
+            except Error as e:
+                self.grumble('Skipping file locale "{}" ({})\n'.format(locale, e.message))
+
+    import textwrap
+    @staticmethod
+    def __wrapped(writer, prefix, tokens, wrap = textwrap.wrap):
+        writer('\n'.join(wrap(prefix + ', '.join(tokens),
+                              subsequent_indent=' ', width=80)) + '\n')
+    del textwrap
+
+    def __parseTags(self, locale):
+        tags = self.__splitLocale(locale)
+        language = tags.next()
+        script = country = variant = ''
+        try:
+            script, country, variant = tags
+        except ValueError:
+            pass
+        return tuple(p[1] for p in self.root.codesToIdName(language, script, country, variant))
+
+    def __splitLocale(self, name):
+        """Generate (language, script, territory, variant) from a locale name
+
+        Ignores any trailing fields (with a warning), leaves script (a
+        capitalised four-letter token), territory (either a number or
+        an all-uppercase token) or variant (upper case and digits)
+        empty if unspecified.  Only generates one entry if name is a
+        single tag (i.e. contains no underscores).  Always yields 1 or
+        4 values, never 2 or 3."""
+        tags = iter(name.split('_'))
+        yield tags.next() # Language
+        tag = tags.next() # may raise StopIteration
+
+        # Script is always four letters, always capitalised:
+        if len(tag) == 4 and tag[0].isupper() and tag[1:].islower():
+            yield tag
+            try:
+                tag = tags.next()
+            except StopIteration:
+                tag = ''
+        else:
+            yield ''
+
+        # Territory is upper-case or numeric:
+        if tag and tag.isupper() or tag.isdigit():
+            yield tag
+            try:
+                tag = tags.next()
+            except StopIteration:
+                tag = ''
+        else:
+            yield ''
+
+        # Variant can be any mixture of upper-case and digits.
+        if tag and all(c.isupper() or c.isdigit() for c in tag):
+            yield tag
+            tag = ''
+        else:
+            yield ''
+
+        # If nothing is left, StopIteration will avoid the warning:
+        if not tag:
+            tag = tags.next()
+        self.grumble('Ignoring unparsed cruft {} in {}\n'.format('_'.join(tag + tuple(tags)), name))
+
+    def __getLocaleData(self, scan, calendars, language, script, country, variant):
+        ids, names = zip(*self.root.codesToIdName(language, script, country, variant))
+        assert ids[0] > 0 and ids[2] > 0, (language, script, country, variant)
+        locale = Locale(
+            language = names[0], language_code = language, language_id = ids[0],
+            script = names[1], script_code = script, script_id = ids[1],
+            country = names[2], country_code = country, country_id = ids[2],
+            variant_code = variant)
+
+        firstDay, weStart, weEnd = self.root.weekData(country)
+        assert all(day in ('mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun')
+                   for day in (firstDay, weStart, weEnd))
+
+        locale.update(firstDayOfWeek = firstDay,
+                      weekendStart = weStart,
+                      weekendEnd = weEnd)
+
+        iso, digits, rounding = self.root.currencyData(country)
+        locale.update(currencyIsoCode = iso,
+                      currencyDigits = int(digits),
+                      currencyRounding = int(rounding))
+
+        locale.update(scan.currencyData(iso))
+        locale.update(scan.numericData(self.root.numberSystem, self.whitter))
+        locale.update(scan.textPatternData())
+        locale.update(scan.endonyms(language, script, country, variant))
+        locale.update(scan.unitData()) # byte, kB, MB, GB, ..., KiB, MiB, GiB, ...
+        locale.update(scan.calendarNames(calendars)) # Names of days and months
+
+        return locale
+
+# Note: various caches assume this class is a singleton, so the
+# "default" value for a parameter no caller should pass can serve as
+# the cache. If a process were to instantiate this class with distinct
+# roots, each cache would be filled by the first to need it !
 class CldrAccess (object):
     def __init__(self, root):
         """Set up a master object for accessing CLDR data.
@@ -50,6 +248,12 @@ class CldrAccess (object):
         contain dtd/, main/ and supplemental/ sub-directories."""
         self.root = root
 
+    def xml(self, *path):
+        """Load a single XML file and return its root element as an XmlScanner.
+
+        The path is interpreted relative to self.root"""
+        return XmlScanner(Node(self.__xml(path)))
+
     def supplement(self, name):
         """Loads supplemental data as a Supplement object.
 
@@ -57,6 +261,117 @@ class CldrAccess (object):
         """
         return Supplement(Node(self.__xml(('common', 'supplemental', name))))
 
+    def locale(self, name):
+        """Loads all data for a locale as a LocaleScanner object.
+
+        The name should be a locale name; adding suffix '.xml' to it
+        should usually yield a file in common/main/.  The returned
+        LocaleScanner object packages this file along with all those
+        from which it inherits; its methods know how to handle that
+        inheritance, where relevant."""
+        return LocaleScanner(name, self.__localeRoots(name), self.__rootLocale)
+
+    @property
+    def fileLocales(self, joinPath = os.path.join, listDirectory = os.listdir,
+                    splitExtension = os.path.splitext):
+        """Generator for locale IDs seen in file-names.
+
+        All *.xml other than root.xml in common/main/ are assumed to
+        identify locales."""
+        for name in listDirectory(joinPath(self.root, 'common', 'main')):
+            stem, ext = splitExtension(name)
+            if ext == '.xml' and stem != 'root':
+                yield stem
+
+    @property
+    def defaultContentLocales(self):
+        """Generator for the default content locales."""
+        for name, attrs in self.supplement('supplementalMetadata.xml').find('metadata/defaultContent'):
+            try:
+                locales = attrs['locales']
+            except KeyError:
+                pass
+            else:
+                for locale in locales.split():
+                    yield locale
+
+    def likelySubTags(self):
+        for ignore, attrs in self.supplement('likelySubtags.xml').find('likelySubtags'):
+            yield attrs['from'], attrs['to']
+
+    def numberSystem(self, system):
+        """Get a description of a numbering system.
+
+        Returns a mapping, with keys u'digits', u'type' and u'id'; the
+        value for this last is system. Raises KeyError for unknown
+        number system, ldml.Error on failure to load data."""
+        try:
+            return self.__numberSystems[system]
+        except KeyError:
+            raise Error('Unsupported number system: {}'.format(system))
+
+    def weekData(self, country):
+        """Data on the weekly cycle.
+
+        Returns a triple (W, S, E) of en's short names for week-days;
+        W is the first day of the week, S the start of the week-end
+        and E the end of the week-end.  Where data for a country is
+        unavailable, the data for CLDR's territory 001 (The World) is
+        used."""
+        try:
+            return self.__weekData[country]
+        except KeyError:
+            return self.__weekData['001']
+
+    def currencyData(self, country):
+        """Returns currency data for the given country code.
+
+        Return value is a tuple (ISO4217 code, digit count, rounding
+        mode).  If CLDR provides no data for this country, ('', 2, 1)
+        is the default result.
+        """
+        try:
+            return self.__currencyData[country]
+        except KeyError:
+            return '', 2, 1
+
+    def codesToIdName(self, language, script, country, variant = ''):
+        """Maps each code to the appropriate ID and name.
+
+        Returns a 4-tuple of (ID, name) pairs corresponding to the
+        language, script, country and variant given.  Raises a
+        suitable error if any of them is unknown, indicating all that
+        are unknown plus suitable names for any that could sensibly be
+        added to enumdata.py to make them known.
+
+        Until we implement variant support (QTBUG-81051), the fourth
+        member of the returned tuple is always 0 paired with a string
+        that should not be used."""
+        enum = self.__enumMap
+        try:
+            return (enum('language')[language],
+                    enum('script')[script],
+                    enum('country')[country],
+                    enum('variant')[variant])
+        except KeyError:
+            pass
+
+        parts, values = [], [language, script, country, variant]
+        for index, key in enumerate(('language', 'script', 'country', 'variant')):
+            naming, enums = self.__codeMap(key), enum(key)
+            value = values[index]
+            if value not in enums:
+                text = '{} code {}'.format(key, value)
+                name = naming.get(value)
+                if name and value != 'POSIX':
+                    text += u' (could add {})'.format(name)
+                parts.append(text)
+        if len(parts) > 1:
+            parts[-1] = 'and ' + parts[-1]
+        assert parts
+        raise Error('Unknown ' + ', '.join(parts),
+                    language, script, country, variant)
+
     def readWindowsTimeZones(self, lookup): # For use by cldr2qtimezone.py
         """Digest CLDR's MS-Win time-zone name mapping.
 
@@ -138,12 +453,98 @@ class CldrAccess (object):
     def __open(self, path, joinPath=os.path.join):
         return open(joinPath(self.root, *path))
 
+    @property
+    def __rootLocale(self, cache = []):
+        if not cache:
+            cache.append(self.xml('common', 'main', 'root.xml'))
+        return cache[0]
+
     @property
     def __supplementalData(self, cache = []):
         if not cache:
             cache.append(self.supplement('supplementalData.xml'))
         return cache[0]
 
+    @property
+    def __numberSystems(self, cache = {}, joinPath=os.path.join):
+        if not cache:
+            for ignore, attrs in self.supplement('numberingSystems.xml').find('numberingSystems'):
+                if ord(attrs.get('digits', u'\x10000')[0]) > 0xffff:
+                    # FIXME, QTBUG-69324: make this redundant:
+                    # omit number system if zero doesn't fit in single-char16 UTF-16 :-(
+                    continue
+
+                cache[attrs['id']] = attrs
+            assert cache
+        return cache
+
+    @property
+    def __weekData(self, cache = {}):
+        if not cache:
+            firstDay, weStart, weEnd = self.__getWeekData()
+            # Massage those into an easily-consulted form:
+            # World defaults given for code '001':
+            mon, sat, sun = firstDay['001'], weStart['001'], weEnd['001']
+            lands = set(firstDay) | set(weStart) | set(weEnd)
+            cache.update((land,
+                          (firstDay.get(land, mon), weStart.get(land, sat), weEnd.get(land, sun)))
+                         for land in lands)
+            assert cache
+        return cache
+
+    def __getWeekData(self):
+        """Scan for data on the weekly cycle.
+
+        Yields three mappings from locales to en's short names for
+        week-days; if a locale isn't a key of a given mapping, it
+        should use the '001' (world) locale's value. The first mapping
+        gives the day on which the week starts, the second gives the
+        day on which the week-end starts, the third gives the last day
+        of the week-end."""
+        source = self.__supplementalData
+        for key in ('firstDay', 'weekendStart', 'weekendEnd'):
+            result = {}
+            for ignore, attrs in source.find('weekData/' + key):
+                assert ignore == key
+                day = attrs['day']
+                assert day in ('mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'), day
+                if 'alt' in attrs:
+                    continue
+                for loc in attrs.get('territories', '').split():
+                    result[loc] = day
+            yield result
+
+    @property
+    def __currencyData(self, cache = {}):
+        if not cache:
+            source = self.__supplementalData
+            for elt in source.findNodes('currencyData/region'):
+                iso, digits, rounding = '', 2, 1
+                try:
+                    country = elt.dom.attributes['iso3166'].nodeValue
+                except KeyError:
+                    continue
+                for child in elt.findAllChildren('currency'):
+                    try:
+                        if child.dom.attributes['tender'].nodeValue == 'false':
+                            continue
+                    except KeyError:
+                        pass
+                    try:
+                        child.dom.attributes['to'] # Is set if this element has gone out of date.
+                    except KeyError:
+                        iso = child.dom.attributes['iso4217'].nodeValue
+                        break
+                if iso:
+                    for tag, data in source.find(
+                        'currencyData/fractions/info[iso4217={}]'.format(iso)):
+                        digits = data['digits']
+                        rounding = data['rounding']
+                cache[country] = iso, digits, rounding
+            assert cache
+
+        return cache
+
     def __scanLdmlDtd(self, joinPath = os.path.join):
         """Scan the LDML DTD, record CLDR version."""
         with self.__open(('common', 'dtd', 'ldml.dtd')) as dtd:
@@ -151,7 +552,8 @@ class CldrAccess (object):
                 if line.startswith('<!ATTLIST '):
                     parts = line.split()
                     if parts[1:5] == ['version', 'cldrVersion', 'CDATA', '#FIXED']:
-                        # parts[5] is the version, in quotes, although the final > might be stuck on its end:
+                        # parts[5] is the version, in quotes, maybe
+                        # with a final > attached to its end:
                         self.__cldrVersion = parts[5].split('"')[1]
                         break
 
@@ -178,5 +580,93 @@ class CldrAccess (object):
 
         return cache[key]
 
+    def __codeMap(self, key, cache = {},
+                  # Maps our name for it to CLDR's name:
+                  naming = {'language': 'languages', 'script': 'scripts',
+                            'country': 'territories', 'variant': 'variants'}):
+        if not cache:
+            root = self.xml('common', 'main', 'en.xml').root.findUniqueChild('localeDisplayNames')
+            for dst, src in naming.items():
+                cache[dst] = dict(self.__codeMapScan(root.findUniqueChild(src)))
+            assert cache
+
+        return cache[key]
+
+    def __codeMapScan(self, node):
+        """Get mapping from codes to element values.
+
+        Passed in node is a <languages>, <scripts>, <territories> or
+        <variants> node, each child of which is a <language>,
+        <script>, <territory> or <variant> node as appropriate, whose
+        type is a code (of the appropriate flavour) and content is its
+        full name.  In some cases, two child nodes have the same type;
+        in these cases, one always has an alt attribute and we should
+        prefer the other.  Yields all such type, content pairs found
+        in node's children (skipping any with an alt attribute, if
+        their type has been seen previously)."""
+        seen = set()
+        for elt in node.dom.childNodes:
+            try:
+                key, value = elt.attributes['type'].nodeValue, elt.childNodes[0].wholeText
+            except (KeyError, ValueError, TypeError):
+                pass
+            else:
+                if key not in seen or not elt.attributes.has_key('alt'):
+                    yield key, value
+                    seen.add(key)
+
+    # CLDR uses inheritance between locales to save repetition:
+    def __parentLocale(self, name, cache = {}):
+        # see http://www.unicode.org/reports/tr35/#Parent_Locales
+        if not cache:
+            for tag, attrs in self.__supplementalData.find('parentLocales'):
+                parent = attrs.get('parent', '')
+                for child in attrs['locales'].split():
+                    cache[child] = parent
+            assert cache
+
+        return cache[name]
+
+    def __localeAsDoc(self, name, aliasFor = None,
+                      joinPath = os.path.join, exists = os.path.isfile):
+        path = ('common', 'main', name + '.xml')
+        if exists(joinPath(self.root, *path)):
+            elt = self.__xml(path)
+            for child in Node(elt).findAllChildren('alias'):
+                try:
+                    alias = child.dom.attributes['source'].nodeValue
+                except (KeyError, AttributeError):
+                    pass
+                else:
+                    return self.__localeAsDoc(alias, aliasFor or name)
+            # No alias child with a source:
+            return elt
+
+        if aliasFor:
+            raise Error('Fatal error: found an alias "{}" -> "{}", but found no file for the alias'
+                        .format(aliasFor, name))
+
+    def __scanLocaleRoots(self, name):
+        while name and name != 'root':
+            doc = self.__localeAsDoc(name)
+            if doc is not None:
+                yield Node(doc)
+
+            try:
+                name = self.__parentLocale(name)
+            except KeyError:
+                try:
+                    name, tail = name.rsplit('_', 1)
+                except ValueError: # No tail to discard: we're done
+                    break
+
+    class __Seq (list): pass # No weakref for tuple and list, but list sub-class is ok.
+    def __localeRoots(self, name, cache = CacheDict()):
+        try:
+            chain = cache[name]
+        except KeyError:
+            cache[name] = chain = self.__Seq(self.__scanLocaleRoots(name))
+        return chain
+
 # Unpolute the namespace: we don't need to export these.
 del minidom, CacheDict, os
diff --git a/util/locale_database/cldr2qlocalexml.py b/util/locale_database/cldr2qlocalexml.py
index 41795ff634..b28dcecc45 100755
--- a/util/locale_database/cldr2qlocalexml.py
+++ b/util/locale_database/cldr2qlocalexml.py
@@ -2,7 +2,7 @@
 # coding=utf8
 #############################################################################
 ##
-## Copyright (C) 2018 The Qt Company Ltd.
+## Copyright (C) 2020 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of the test suite of the Qt Toolkit.
@@ -31,15 +31,17 @@
 
 The CLDR data can be downloaded from CLDR_, which has a sub-directory
 for each version; you need the ``core.zip`` file for your version of
-choice (typically the latest).  This script has had updates to cope up
-to v35; for later versions, we may need adaptations.  Unpack the
+choice (typically the latest). This script has had updates to cope up
+to v35; for later versions, we may need adaptations. Unpack the
 downloaded ``core.zip`` and check it has a common/main/ sub-directory:
-pass the path of that sub-directory to this script as its single
-command-line argument.  Save its standard output (but not error) to a
-file for later processing by ``./qlocalexml2cpp.py``
+pass the path of that root of the download to this script as its first
+command-line argument. Pass the name of the file in which to write
+output as the second argument; either omit it or use '-' to select the
+standard output. This file is the input needed by
+``./qlocalexml2cpp.py``
 
 When you update the CLDR data, be sure to also update
-src/corelib/text/qt_attribution.json's entry for unicode-cldr.  Check
+src/corelib/text/qt_attribution.json's entry for unicode-cldr. Check
 this script's output for unknown language, country or script messages;
 if any can be resolved, use their entry in common/main/en.xml to
 append new entries to enumdata.py's lists and update documentation in
@@ -53,610 +55,62 @@ time zone names; see cldr2qtimezone.py for details.
 """
 
 import os
-import sys
-import re
-import textwrap
 
-import enumdata
 from localetools import Error
-from xpathlite import DraftResolution, findAlias, findEntry, findTagsInFile, codeMapsFromFile, \
-    _findEntryInFile as findEntryInFile
-from dateconverter import convert_date
-from qlocalexml import Locale, QLocaleXmlWriter
-
-# TODO: make calendars a command-line option
-calendars = ['gregorian', 'persian', 'islamic'] # 'hebrew'
-def wrappedwarn(err, prefix, tokens):
-    return err.write(
-        '\n'.join(textwrap.wrap(prefix + ', '.join(tokens),
-                                subsequent_indent=' ', width=80)) + '\n')
-
-def parse_number_format(patterns, data):
-    # this is a very limited parsing of the number format for currency only.
-    def skip_repeating_pattern(x):
-        p = x.replace('0', '#').replace(',', '').replace('.', '')
-        seen = False
-        result = ''
-        for c in p:
-            if c == '#':
-                if seen:
-                    continue
-                seen = True
-            else:
-                seen = False
-            result = result + c
-        return result
-    patterns = patterns.split(';')
-    result = []
-    for pattern in patterns:
-        pattern = skip_repeating_pattern(pattern)
-        pattern = pattern.replace('#', "%1")
-        # according to http://www.unicode.org/reports/tr35/#Number_Format_Patterns
-        # there can be doubled or trippled currency sign, however none of the
-        # locales use that.
-        pattern = pattern.replace(u'\xa4', "%2")
-        pattern = pattern.replace("''", "###").replace("'", '').replace("###", "'")
-        pattern = pattern.replace('-', data['minus'])
-        pattern = pattern.replace('+', data['plus'])
-        result.append(pattern)
-    return result
-
-cldr_dir = None
-def raiseUnknownCode(code, form, cache={}):
-    """Check whether an unknown code could be supported.
-
-    We declare a language, script or country code unknown if it's not
-    known to enumdata.py; however, if it's present in main/en.xml's
-    mapping of codes to names, we have the option of adding support.
-    This caches the necessary look-up (so we only read main/en.xml
-    once) and returns the name we should use if we do add support.
-
-    First parameter, code, is the unknown code.  Second parameter,
-    form, is one of 'language', 'script' or 'country' to select the
-    type of code to look up.  Do not pass further parameters (the next
-    will deprive you of the cache).
-
-    Raises localetools.Error with a suitable message, that includes
-    the unknown code's full name if found.
-
-    Relies on global cldr_dir being set before it's called; see tail
-    of this file.
-    """
-    if not cache:
-        cache.update(codeMapsFromFile(os.path.join(cldr_dir, 'en.xml')))
-    name = cache[form].get(code)
-    msg = 'unknown %s code "%s"' % (form, code)
-    if name:
-        msg += ' - could use "%s"' % name
-    raise Error(msg)
-
-def parse_list_pattern_part_format(pattern):
-    # This is a very limited parsing of the format for list pattern part only.
-    return pattern.replace("{0}", "%1").replace("{1}", "%2").replace("{2}", "%3")
-
-def unit_quantifiers(find, path, stem, suffix, known,
-                     # Stop at exa/exbi: 16 exbi = 2^{64} < zetta =
-                     # 1000^7 < zebi = 2^{70}, the next quantifiers up:
-                     si_quantifiers = ('kilo', 'mega', 'giga', 'tera', 'peta', 'exa')):
-    """Work out the unit quantifiers.
-
-    Unfortunately, the CLDR data only go up to terabytes and we want
-    all the way to exabytes; but we can recognize the SI quantifiers
-    as prefixes, strip and identify the tail as the localized
-    translation for 'B' (e.g. French has 'octet' for 'byte' and uses
-    ko, Mo, Go, To from which we can extrapolate Po, Eo).
-
-    Should be called first for the SI quantifiers, with suffix = 'B',
-    then for the IEC ones, with suffix = 'iB'; the list known
-    (initially empty before first call) is used to let the second call
-    know what the first learned about the localized unit.
-    """
-    if suffix == 'B': # first call, known = []
-        tail = suffix
-        for q in si_quantifiers:
-            it = find(path, stem % q)
-            # kB for kilobyte, in contrast with KiB for IEC:
-            q = q[0] if q == 'kilo' else q[0].upper()
-            if not it:
-                it = q + tail
-            elif it.startswith(q):
-                rest = it[1:]
-                tail = rest if all(rest == k for k in known) else suffix
-                known.append(rest)
-            yield it
-    else: # second call, re-using first's known
-        assert suffix == 'iB'
-        if known:
-            byte = known.pop()
-            if all(byte == k for k in known):
-                suffix = 'i' + byte
-        for q in si_quantifiers:
-            yield find(path, stem % q[:2],
-                       # Those don't (yet, v31) exist in CLDR, so we always fall back to:
-                       q[0].upper() + suffix)
-
-def generateLocaleInfo(path):
-    if not path.endswith(".xml"):
-        return {}
-
-    # skip legacy/compatibility ones
-    alias = findAlias(path)
-    if alias:
-        raise Error('Alias to "{}"'.format(alias))
-
-    def code(tag):
-        return findEntryInFile(path, 'identity/' + tag, attribute="type")[0]
-
-    return _generateLocaleInfo(path, code('language'), code('script'),
-                               code('territory'), code('variant'))
-
-def getNumberSystems(cache={}):
-    """Cached look-up of number system information.
-
-    Pass no arguments.  Returns a mapping from number system names to,
-    for each system, a mapping with keys 'digits', 'type' and 'id'.
-    Relies on global cldr_dir being set before it's first called.\n"""
-    if not cache:
-        for ns in findTagsInFile(os.path.join(cldr_dir, '..', 'supplemental',
-                                              'numberingSystems.xml'),
-                                 'numberingSystems'):
-            # ns has form: [u'numberingSystem', [(u'digits', u'0123456789'), (u'type', u'numeric'), (u'id', u'latn')]]
-            entry = dict(ns[1])
-            name = entry[u'id']
-            if u'digits' in entry and ord(entry[u'digits'][0]) > 0xffff:
-                # FIXME, QTBUG-69324: make this redundant:
-                # omit number system if zero doesn't fit in single-char16 UTF-16 :-(
-                sys.stderr.write('skipping number system "%s" [can\'t represent its zero, U+%X]\n'
-                                 % (name, ord(entry[u'digits'][0])))
-            else:
-                cache[name] = entry
-    return cache
-
-def _generateLocaleInfo(path, language_code, script_code, country_code, variant_code=""):
-    if not path.endswith(".xml"):
-        return {}
-
-    if language_code == 'root':
-        # just skip it
-        return {}
-
-    # we do not support variants
-    # ### actually there is only one locale with variant: en_US_POSIX
-    #     does anybody care about it at all?
-    if variant_code:
-        raise Error('We do not support variants ("{}")'.format(variant_code))
-
-    language_id = enumdata.languageCodeToId(language_code)
-    if language_id <= 0:
-        raiseUnknownCode(language_code, 'language')
-
-    script_id = enumdata.scriptCodeToId(script_code)
-    if script_id == -1:
-        raiseUnknownCode(script_code, 'script')
-
-    # we should handle fully qualified names with the territory
-    if not country_code:
-        return {}
-    country_id = enumdata.countryCodeToId(country_code)
-    if country_id <= 0:
-        raiseUnknownCode(country_code, 'country')
-
-    # So we say we accept only those values that have "contributed" or
-    # "approved" resolution. see http://www.unicode.org/cldr/process.html
-    # But we only respect the resolution for new datas for backward
-    # compatibility.
-    draft = DraftResolution.contributed
-
-    result = dict(
-        language=enumdata.language_list[language_id][0],
-        language_code=language_code, language_id=language_id,
-        script=enumdata.script_list[script_id][0],
-        script_code=script_code, script_id=script_id,
-        country=enumdata.country_list[country_id][0],
-        country_code=country_code, country_id=country_id,
-        variant_code=variant_code)
-
-    (dir_name, file_name) = os.path.split(path)
-    def from_supplement(tag,
-                        path=os.path.join(dir_name, '..', 'supplemental',
-                                          'supplementalData.xml')):
-        return findTagsInFile(path, tag)
-    currencies = from_supplement('currencyData/region[iso3166=%s]' % country_code)
-    result['currencyIsoCode'] = ''
-    result['currencyDigits'] = 2
-    result['currencyRounding'] = 1
-    if currencies:
-        for e in currencies:
-            if e[0] == 'currency':
-                t = [x[1] == 'false' for x in e[1] if x[0] == 'tender']
-                if t and t[0]:
-                    pass
-                elif not any(x[0] == 'to' for x in e[1]):
-                    result['currencyIsoCode'] = (x[1] for x in e[1] if x[0] == 'iso4217').next()
-                    break
-        if result['currencyIsoCode']:
-            t = from_supplement("currencyData/fractions/info[iso4217=%s]"
-                                % result['currencyIsoCode'])
-            if t and t[0][0] == 'info':
-                result['currencyDigits'] = (int(x[1]) for x in t[0][1] if x[0] == 'digits').next()
-                result['currencyRounding'] = (int(x[1]) for x in t[0][1] if x[0] == 'rounding').next()
-    numbering_system = None
-    try:
-        numbering_system = findEntry(path, "numbers/defaultNumberingSystem")
-    except Error:
-        pass
-    def findEntryDef(path, xpath, value=''):
-        try:
-            return findEntry(path, xpath)
-        except Error:
-            return value
-    def get_number_in_system(path, xpath, numbering_system):
-        if numbering_system:
-            try:
-                return findEntry(path, xpath + "[numberSystem=" + numbering_system + "]")
-            except Error:
-                # in CLDR 1.9 number system was refactored for numbers (but not for currency)
-                # so if previous findEntry doesn't work we should try this:
-                try:
-                    return findEntry(path, xpath.replace("/symbols/", "/symbols[numberSystem=" + numbering_system + "]/"))
-                except Error:
-                    # fallback to default
-                    pass
-        return findEntry(path, xpath)
-
-    result['decimal'] = get_number_in_system(path, "numbers/symbols/decimal", numbering_system)
-    result['group'] = get_number_in_system(path, "numbers/symbols/group", numbering_system)
-    assert result['decimal'] != result['group']
-    result['list'] = get_number_in_system(path, "numbers/symbols/list", numbering_system)
-    result['percent'] = get_number_in_system(path, "numbers/symbols/percentSign", numbering_system)
-    try:
-        result['zero'] = getNumberSystems()[numbering_system][u"digits"][0]
-    except Exception as e:
-        sys.stderr.write("Native zero detection problem: %s\n" % repr(e))
-        result['zero'] = get_number_in_system(path, "numbers/symbols/nativeZeroDigit", numbering_system)
-    result['minus'] = get_number_in_system(path, "numbers/symbols/minusSign", numbering_system)
-    result['plus'] = get_number_in_system(path, "numbers/symbols/plusSign", numbering_system)
-    result['exp'] = get_number_in_system(path, "numbers/symbols/exponential", numbering_system).lower()
-    result['quotationStart'] = findEntry(path, "delimiters/quotationStart")
-    result['quotationEnd'] = findEntry(path, "delimiters/quotationEnd")
-    result['alternateQuotationStart'] = findEntry(path, "delimiters/alternateQuotationStart")
-    result['alternateQuotationEnd'] = findEntry(path, "delimiters/alternateQuotationEnd")
-    result['listPatternPartStart'] = parse_list_pattern_part_format(findEntry(path, "listPatterns/listPattern/listPatternPart[start]"))
-    result['listPatternPartMiddle'] = parse_list_pattern_part_format(findEntry(path, "listPatterns/listPattern/listPatternPart[middle]"))
-    result['listPatternPartEnd'] = parse_list_pattern_part_format(findEntry(path, "listPatterns/listPattern/listPatternPart[end]"))
-    result['listPatternPartTwo'] = parse_list_pattern_part_format(findEntry(path, "listPatterns/listPattern/listPatternPart[2]"))
-    result['am'] = findEntry(path, "dates/calendars/calendar[gregorian]/dayPeriods/dayPeriodContext[format]/dayPeriodWidth[wide]/dayPeriod[am]", draft)
-    result['pm'] = findEntry(path, "dates/calendars/calendar[gregorian]/dayPeriods/dayPeriodContext[format]/dayPeriodWidth[wide]/dayPeriod[pm]", draft)
-    result['longDateFormat'] = convert_date(findEntry(path, "dates/calendars/calendar[gregorian]/dateFormats/dateFormatLength[full]/dateFormat/pattern"))
-    result['shortDateFormat'] = convert_date(findEntry(path, "dates/calendars/calendar[gregorian]/dateFormats/dateFormatLength[short]/dateFormat/pattern"))
-    result['longTimeFormat'] = convert_date(findEntry(path, "dates/calendars/calendar[gregorian]/timeFormats/timeFormatLength[full]/timeFormat/pattern"))
-    result['shortTimeFormat'] = convert_date(findEntry(path, "dates/calendars/calendar[gregorian]/timeFormats/timeFormatLength[short]/timeFormat/pattern"))
-
-    endonym = None
-    if country_code and script_code:
-        endonym = findEntryDef(path, "localeDisplayNames/languages/language[type=%s_%s_%s]" % (language_code, script_code, country_code))
-    if not endonym and script_code:
-        endonym = findEntryDef(path, "localeDisplayNames/languages/language[type=%s_%s]" % (language_code, script_code))
-    if not endonym and country_code:
-        endonym = findEntryDef(path, "localeDisplayNames/languages/language[type=%s_%s]" % (language_code, country_code))
-    if not endonym:
-        endonym = findEntryDef(path, "localeDisplayNames/languages/language[type=%s]" % (language_code))
-    result['languageEndonym'] = endonym
-    result['countryEndonym'] = findEntryDef(path, "localeDisplayNames/territories/territory[type=%s]" % (country_code))
-
-    currency_format = get_number_in_system(path, "numbers/currencyFormats/currencyFormatLength/currencyFormat/pattern", numbering_system)
-    currency_format = parse_number_format(currency_format, result)
-    result['currencyFormat'] = currency_format[0]
-    result['currencyNegativeFormat'] = ''
-    if len(currency_format) > 1:
-        result['currencyNegativeFormat'] = currency_format[1]
-
-    result['currencySymbol'] = ''
-    result['currencyDisplayName'] = ''
-    if result['currencyIsoCode']:
-        result['currencySymbol'] = findEntryDef(path, "numbers/currencies/currency[%s]/symbol" % result['currencyIsoCode'])
-        result['currencyDisplayName'] = ';'.join(
-            findEntryDef(path, 'numbers/currencies/currency[' + result['currencyIsoCode']
-                         + ']/displayName' + tail)
-            for tail in ['',] + [
-                '[count=%s]' % x for x in ('zero', 'one', 'two', 'few', 'many', 'other')
-                ]) + ';'
-
-    def findUnitDef(path, stem, fallback=''):
-        # The displayName for a quantified unit in en.xml is kByte
-        # instead of kB (etc.), so prefer any unitPattern provided:
-        for count in ('many', 'few', 'two', 'other', 'zero', 'one'):
-            try:
-                ans = findEntry(path, stem + 'unitPattern[count=%s]' % count)
-            except Error:
-                continue
-
-            # TODO: epxloit count-handling, instead of discarding placeholders
-            if ans.startswith('{0}'):
-                ans = ans[3:].lstrip()
-            if ans:
-                return ans
-
-        return findEntryDef(path, stem + 'displayName', fallback)
-
-    # First without quantifier, then quantified each way:
-    result['byte_unit'] = findEntryDef(
-        path, 'units/unitLength[type=long]/unit[type=digital-byte]/displayName',
-        'bytes')
-    stem = 'units/unitLength[type=short]/unit[type=digital-%sbyte]/'
-    known = [] # cases where we *do* have a given version:
-    result['byte_si_quantified'] = ';'.join(unit_quantifiers(findUnitDef, path, stem, 'B', known))
-    # IEC 60027-2
-    # http://physics.nist.gov/cuu/Units/binary.html
-    result['byte_iec_quantified'] = ';'.join(unit_quantifiers(findUnitDef, path, stem % '%sbi', 'iB', known))
-
-    # Used for month and day data:
-    namings = (
-        ('standaloneLong', 'stand-alone', 'wide'),
-        ('standaloneShort', 'stand-alone', 'abbreviated'),
-        ('standaloneNarrow', 'stand-alone', 'narrow'),
-        ('long', 'format', 'wide'),
-        ('short', 'format', 'abbreviated'),
-        ('narrow', 'format', 'narrow'),
-        )
-
-    # Month names for 12-month calendars:
-    for cal in calendars:
-        stem = 'dates/calendars/calendar[' + cal + ']/months/'
-        for (key, mode, size) in namings:
-            prop = 'monthContext[' + mode + ']/monthWidth[' + size + ']/'
-            result[key + 'Months_' + cal] = ';'.join(
-                findEntry(path, stem + prop + "month[%d]" % i)
-                for i in range(1, 13)) + ';'
-
-    # Day data (for Gregorian, at least):
-    stem = 'dates/calendars/calendar[gregorian]/days/'
-    days = ('sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat')
-    for (key, mode, size) in namings:
-        prop = 'dayContext[' + mode + ']/dayWidth[' + size + ']/day'
-        result[key + 'Days'] = ';'.join(
-            findEntry(path, stem + prop + '[' + day + ']')
-            for day in days) + ';'
-
-    return Locale(result)
-
-def integrateWeekData(filePath, locale_database):
-    if not filePath.endswith(".xml"):
-        return {}
-
-    def lookup(key):
-        return findEntryInFile(filePath, key, attribute='territories')[0].split()
-    days = ('mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun')
-
-    firstDayByCountryCode = {}
-    for day in days:
-        for countryCode in lookup('weekData/firstDay[day=%s]' % day):
-            firstDayByCountryCode[countryCode] = day
-
-    weekendStartByCountryCode = {}
-    for day in days:
-        for countryCode in lookup('weekData/weekendStart[day=%s]' % day):
-            weekendStartByCountryCode[countryCode] = day
-
-    weekendEndByCountryCode = {}
-    for day in days:
-        for countryCode in lookup('weekData/weekendEnd[day=%s]' % day):
-            weekendEndByCountryCode[countryCode] = day
-
-    for (key, locale) in locale_database.iteritems():
-        countryCode = locale.country_code
-        if countryCode in firstDayByCountryCode:
-            locale.firstDayOfWeek = firstDayByCountryCode[countryCode]
-        else:
-            locale.firstDayOfWeek = firstDayByCountryCode["001"]
-
-        if countryCode in weekendStartByCountryCode:
-            locale.weekendStart = weekendStartByCountryCode[countryCode]
-        else:
-            locale.weekendStart = weekendStartByCountryCode["001"]
-
-        if countryCode in weekendEndByCountryCode:
-            locale.weekendEnd = weekendEndByCountryCode[countryCode]
-        else:
-            locale.weekendEnd = weekendEndByCountryCode["001"]
-
-def splitLocale(name):
-    """Split name into (language, script, territory) triple as generator.
-
-    Ignores any trailing fields (with a warning), leaves script (a capitalised
-    four-letter token) or territory (either a number or an all-uppercase token)
-    empty if unspecified, returns a single-entry generator if name is a single
-    tag (i.e. contains no underscores).  Always yields 1 or 3 values, never 2."""
-    tags = iter(name.split('_'))
-    yield tags.next() # Language
-    tag = tags.next()
-
-    # Script is always four letters, always capitalised:
-    if len(tag) == 4 and tag[0].isupper() and tag[1:].islower():
-        yield tag
-        try:
-            tag = tags.next()
-        except StopIteration:
-            tag = ''
-    else:
-        yield ''
-
-    # Territory is upper-case or numeric:
-    if tag and tag.isupper() or tag.isdigit():
-        yield tag
-        tag = ''
-    else:
-        yield ''
-
-    # If nothing is left, StopIteration will avoid the warning:
-    tag = (tag if tag else tags.next(),)
-    sys.stderr.write('Ignoring unparsed cruft %s in %s\n' % ('_'.join(tag + tuple(tags)), name))
-
-def _parseLocale(l):
-    language = "AnyLanguage"
-    script = "AnyScript"
-    country = "AnyCountry"
-
-    if l == "und":
-        raise Error('We treat unknown locale like C')
-
-    parsed = splitLocale(l)
-    language_code = parsed.next()
-    script_code = country_code = ''
-    try:
-        script_code, country_code = parsed
-    except ValueError:
-        pass
-
-    if language_code != "und":
-        language_id = enumdata.languageCodeToId(language_code)
-        if language_id == -1:
-            raise Error('Unknown language code "{}"'.format(language_code))
-        language = enumdata.language_list[language_id][0]
-
-    if script_code:
-        script_id = enumdata.scriptCodeToId(script_code)
-        if script_id == -1:
-            raise Error('Unknown script code "{}"'.format(script_code))
-        script = enumdata.script_list[script_id][0]
-
-    if country_code:
-        country_id = enumdata.countryCodeToId(country_code)
-        if country_id == -1:
-            raise Error('Unknown country code "{}"'.format(country_code))
-        country = enumdata.country_list[country_id][0]
-
-    return (language, script, country)
-
-def likelySubtags(root, err):
-    skips = []
-    for ns in findTagsInFile(os.path.join(root, 'supplemental', 'likelySubtags.xml'), "likelySubtags"):
-        tmp = {}
-        for data in ns[1:][0]: # ns looks like this: [u'likelySubtag', [(u'from', u'aa'), (u'to', u'aa_Latn_ET')]]
-            tmp[data[0]] = data[1]
-
-        try:
-            from_language, from_script, from_country = _parseLocale(tmp[u"from"])
-            to_language, to_script, to_country = _parseLocale(tmp[u"to"])
-        except Error as e:
-            if (tmp['to'].startswith(tmp['from'])
-                and e.message == 'Unknown language code "{}"'.format(tmp['from'])):
-                skips.append(tmp['to'])
-            else:
-                sys.stderr.write('skipping likelySubtag "{}" -> "{}" ({})\n'.format(
-                        tmp[u"from"], tmp[u"to"], e.message))
-            continue
-        # substitute according to http://www.unicode.org/reports/tr35/#Likely_Subtags
-        if to_country == "AnyCountry" and from_country != to_country:
-            to_country = from_country
-        if to_script == "AnyScript" and from_script != to_script:
-            to_script = from_script
-
-        yield ((from_language, from_script, from_country),
-               (to_language, to_script, to_country))
-    if skips:
-        wrappedwarn(err, 'skipping likelySubtags (for unknown language codes): ', skips)
+from cldr import CldrReader
+from qlocalexml import QLocaleXmlWriter
+from enumdata import language_list, script_list, country_list
 
 def usage(err, name, message = ''):
-    err.write("""Usage: {} <path-to-cldr-main> [out-file.xml]
-""".format(name)) # TODO: expand
+    err.write("""Usage: {} path/to/cldr/common/main [out-file.xml]
+""".format(name)) # TODO: expand command-line, improve help message
     if message:
         err.write('\n' + message + '\n')
 
 def main(args, out, err):
-    name = args.pop(0)
+    # TODO: make calendars a command-line option
+    calendars = ['gregorian', 'persian', 'islamic'] # 'hebrew'
 
-    if len(args) < 1:
-        usage(err, name)
+    # TODO: make argument parsing more sophisticated
+    name = args.pop(0)
+    if not args:
+        usage(name, err, 'Where is your CLDR data tree ?')
         return 1
 
-    global cldr_dir
-    cldr_dir = args.pop(0)
-    if not os.path.isdir(cldr_dir):
-        usage(err, name, 'Where did you unpack the CLDR data files ?')
+    root = args.pop(0)
+    if not os.path.exists(os.path.join(root, 'common', 'main', 'root.xml')):
+        usage(name, err,
+              'First argument is the root of the CLDR tree: found no common/main/root.xml under '
+              + root)
         return 1
 
-    if len(args) > 1:
-        usage(err, name, 'Too many arguments passed')
+    xml = args.pop(0) if args else None
+    if not xml or xml == '-':
+        emit = out
+    elif not xml.endswith('.xml'):
+        usage(name, err, 'Please use a .xml extension on your output file name, not ' + xml)
         return 1
-    if args:
-        qxml = open(args.pop(0), 'w')
     else:
-        qxml = out
-
-    getNumberSystems(cldr_dir)
-    cldr_files = os.listdir(cldr_dir)
-    locale_database = {}
-
-    # see http://www.unicode.org/reports/tr35/tr35-info.html#Default_Content
-    defaultContent_locales = []
-    for ns in findTagsInFile(os.path.join(cldr_dir, '..', 'supplemental',
-                                          'supplementalMetadata.xml'),
-                             'metadata/defaultContent'):
-        for data in ns[1:][0]:
-            if data[0] == u"locales":
-                defaultContent_locales += data[1].split()
-
-    skips = []
-    for file in defaultContent_locales:
         try:
-            language_code, script_code, country_code = splitLocale(file)
-        except ValueError:
-            sys.stderr.write('skipping defaultContent locale "' + file + '" [neither two nor three tags]\n')
-            continue
+            emit = open(xml, 'w')
+        except IOError as e:
+            usage(name, err, 'Failed to open "{}" to write output to it\n'.format(xml))
+            return 1
 
-        if not (script_code or country_code):
-            sys.stderr.write('skipping defaultContent locale "' + file + '" [second tag is neither script nor territory]\n')
-            continue
-
-        try:
-            l = _generateLocaleInfo(cldr_dir + "/" + file + ".xml", language_code, script_code, country_code)
-            if not l:
-                skips.append(file)
-                continue
-        except Error as e:
-            sys.stderr.write('skipping defaultContent locale "{}" ({})\n'.format(file, e.message))
-            continue
-
-        locale_database[(l.language_id, l.script_id, l.country_id, l.variant_code)] = l
-
-    if skips:
-        wrappedwarn(err, 'skipping defaultContent locales [no locale info generated]: ', skips)
-        skips = []
-
-    for file in cldr_files:
-        try:
-            l = generateLocaleInfo(cldr_dir + "/" + file)
-            if not l:
-                skips.append(file)
-                continue
-        except Error as e:
-            sys.stderr.write('skipping file "{}" ({})\n'.format(file, e.message))
-            continue
-
-        locale_database[(l.language_id, l.script_id, l.country_id, l.variant_code)] = l
-
-    if skips:
-        wrappedwarn(err, 'skipping files [no locale info generated]: ', skips)
+    if args:
+        usage(name, err, 'Too many arguments - excess: ' + ' '.join(args))
+        return 1
 
-    integrateWeekData(cldr_dir + "/../supplemental/supplementalData.xml", locale_database)
-    cldr_version = 'unknown'
-    with open(cldr_dir+"/../dtd/ldml.dtd", "r") as ldml:
-        for line in ldml:
-            if 'version cldrVersion CDATA #FIXED' in line:
-                cldr_version = line.split('"')[1]
+    # TODO - command line options to tune choice of grumble and whitter:
+    reader = CldrReader(root, err.write, err.write)
+    writer = QLocaleXmlWriter(emit.write)
 
-    xmlOut = QLocaleXmlWriter(qxml.write)
-    xmlOut.version(cldr_version)
-    xmlOut.enumData(enumdata.language_list,
-                    enumdata.script_list,
-                    enumdata.country_list)
-    xmlOut.likelySubTags(likelySubtags(os.path.split(cldr_dir)[0], err))
-    xmlOut.locales(locale_database, calendars)
-    xmlOut.close()
-    if qxml is not out:
-        qxml.close()
+    writer.version(reader.root.cldrVersion)
+    writer.enumData(language_list, script_list, country_list)
+    writer.likelySubTags(reader.likelySubTags())
+    writer.locales(reader.readLocales(calendars), calendars)
 
+    writer.close()
     return 0
 
 if __name__ == '__main__':
diff --git a/util/locale_database/ldml.py b/util/locale_database/ldml.py
index 4aaa728a86..ff94f3da73 100644
--- a/util/locale_database/ldml.py
+++ b/util/locale_database/ldml.py
@@ -39,10 +39,12 @@ returned by minidom.parse() and their child-nodes:
   Node -- wraps any node in the DOM tree
   XmlScanner -- wraps the root element of a stand-alone XML file
   Supplement -- specializes XmlScanner for supplemental data files
+  LocaleScanner -- wraps a locale's inheritance-chain of file roots
 
 See individual classes for further detail.
 """
 from localetools import Error
+from dateconverter import convert_date
 
 class Node (object):
     """Wrapper for an arbitrary DOM node.
@@ -51,11 +53,20 @@ class Node (object):
     nodes are returned wrapped as Node objects.  A Node exposes the
     raw DOM node it wraps via its .dom attribute."""
 
-    def __init__(self, elt):
+    def __init__(self, elt, draft = 0):
         """Wraps a DOM node for ease of access.
 
-        Single argument, elt, is the DOM node to wrap."""
+        First argument, elt, is the DOM node to wrap. (Optional second
+        argument, draft, should only be supplied by this class's
+        creation of child nodes; it is the maximum draft score of any
+        ancestor of the new node.)"""
         self.dom = elt
+        try:
+            attr = elt.attributes['draft'].nodeValue
+        except KeyError:
+            self.draft = draft
+        else:
+            self.draft = max(draft, self.draftScore(attr))
 
     def findAllChildren(self, tag, wanted = None):
         """All children that do have the given tag and attributes.
@@ -65,34 +76,60 @@ class Node (object):
 
         Optional second argument, wanted, should either be None or map
         attribute names to the values they must have. Only child nodes
-        with these attributes set to the given values are yielded."""
+        with thes attributes set to the given values are yielded."""
 
-        cutoff = 4 # Only accept approved, for now
         for child in self.dom.childNodes:
             if child.nodeType != child.ELEMENT_NODE:
                 continue
             if child.nodeName != tag:
                 continue
 
-            try:
-                draft = child.attributes['draft']
-            except KeyError:
-                pass
-            else:
-                if self.__draftScores.get(draft, 0) < cutoff:
-                    continue
-
-            if wanted is not None:
+            if wanted:
                 try:
-                    if wanted and any(child.attributes[k].nodeValue != v for k, v in wanted.items()):
+                    if any(child.attributes[k].nodeValue != v
+                           for k, v in wanted.items()):
                         continue
                 except KeyError: # Some wanted attribute is missing
                     continue
 
-            yield Node(child)
+            yield Node(child, self.draft)
+
+    def findUniqueChild(self, tag):
+        """Returns the single child with the given nodeName.
+
+        Raises Error if there is no such child or there is more than
+        one."""
+        seq = self.findAllChildren(tag)
+        try:
+            node = seq.next()
+        except StopIteration:
+            raise Error('No child found where one was expected', tag)
+        for it in seq:
+            raise Error('Many children found where only one was expected', tag)
+        return node
+
+    @classmethod
+    def draftScore(cls, level):
+        """Maps draft level names to numeric scores.
+
+        Single parameter, level, is the least sure value of the draft
+        attribute on a node that you're willing to accept; returns a
+        numeric value (lower is less drafty).
 
-    __draftScores = dict(true = 0, unconfirmed = 1, provisional = 2,
-                         contributed = 3, approved = 4, false = 4)
+        Tempting as it is to insist on low draft scores, there are
+        many locales in which pretty much every leaf is
+        unconfirmed. It may make sense to actually check each
+        XmlScanner object, or each node in each LocaleScanner's nodes
+        list, to see what its distribution of draft level looks like,
+        so as to set the acceptable draft score for its elements
+        accordingly. However, for the moment, we mostly just accept
+        all elements, regardless of draft values (the one exception is
+        am/pm indicators)."""
+        return cls.__draftScores.get(level, 5) if level else 0
+
+    # Implementation details:
+    __draftScores = dict(true = 4, unconfirmed = 3, provisional = 2,
+                         contributed = 1, approved = 0, false = 0)
 
 def _parseXPath(selector):
     # Split "tag[attr=val][...]" into tag-name and attribute mapping
@@ -129,7 +166,6 @@ class XmlScanner (object):
         return elts
 
 class Supplement (XmlScanner):
-    # Replaces xpathlite.findTagsInFile()
     def find(self, xpath):
         elts = self.findNodes(xpath)
         for elt in _iterateEach(e.dom.childNodes if e.dom.childNodes else (e.dom,)
@@ -138,3 +174,381 @@ class Supplement (XmlScanner):
                 yield (elt.nodeName,
                        dict((k, v if isinstance(v, basestring) else v.nodeValue)
                             for k, v in elt.attributes.items()))
+
+class LocaleScanner (object):
+    def __init__(self, name, nodes, root):
+        self.name, self.nodes, self.base = name, nodes, root
+
+    def find(self, xpath, draft = None):
+        tags = xpath.split('/')
+        while True:
+            replace = None
+            for elt in self.nodes:
+                for selector in tags:
+                    tag, attrs = _parseXPath(selector)
+                    for elt in elt.findAllChildren(tag, attrs):
+                        if draft is None or elt.draft <= draft:
+                            break # and process the next selector
+                    else:
+                        break # no child, try next elt in self.nodes
+                else:
+                    # processed all selectors
+                    try:
+                        return elt.dom.firstChild.nodeValue
+                    except (AttributeError, KeyError):
+                        pass # move on to next elt in self.nodes
+
+            # No match in self.nodes; check root
+            elt = self.base.root
+            for i, selector in enumerate(tags):
+                tag, attrs = _parseXPath(selector)
+                for alias in elt.findAllChildren('alias'):
+                    if alias.dom.attributes['source'].nodeValue == 'locale':
+                        replace = alias.dom.attributes['path'].nodeValue.split('/')
+                        tags = self.__xpathJoin(tags[:i], replace, tags[i:])
+                        break
+                else:
+                    for elt in elt.findAllChildren(tag, attrs):
+                        if draft is None or elt.draft <= draft:
+                            break # and process the next selector
+                    else:
+                        break
+                if replace:
+                    break
+            else:
+                # processed all selectors
+                try:
+                    return elt.dom.firstChild.nodeValue
+                except (AttributeError, KeyError):
+                    # No match
+                    pass
+            if not replace:
+                break
+
+        sought = '/'.join(tags)
+        if sought != xpath:
+            sought += ' (for {})'.format(xpath)
+        raise Error('No {} in {}'.format(sought, self.name))
+
+    def findOr(self, xpath, fallback = ''):
+        """Use a fall-back value if we don't find data.
+
+        Like find, but takes a fall-back value to return instead of
+        raising Error on failure."""
+        try:
+            return self.find(xpath)
+        except Error:
+            return fallback
+
+    def tagCodes(self):
+        """Yields four tag codes
+
+        The tag codes are language, script, country and variant; an
+        empty value for any of them indicates that no value was
+        provided.  The values are obtained from the primary file's
+        top-level <identity> element.  An Error is raised if any
+        top-level <alias> element of this file has a non-empty source
+        attribute; that attribute value is mentioned in the error's
+        message."""
+        root = self.nodes[0]
+        for alias in root.findAllChildren('alias'):
+            try:
+                source = alias.dom.attributes['source'].nodeValue
+            except (KeyError, AttributeError):
+                pass
+            else:
+                raise Error('Alias to {}'.format(source))
+
+        ids = root.findUniqueChild('identity')
+        for code in ('language', 'script', 'territory', 'variant'):
+            for node in ids.findAllChildren(code):
+                try:
+                    yield node.dom.attributes['type'].nodeValue
+                except (KeyError, AttributeError):
+                    pass
+                else:
+                    break # only want one value for each code
+            else: # No value for this code, use empty
+                yield ''
+
+    def currencyData(self, isoCode):
+        """Fetches currency data for this locale.
+
+        Single argument, isoCode, is the ISO currency code for the
+        currency in use in the country. See also numericData, which
+        includes some currency formats.
+        """
+        if isoCode:
+            stem = 'numbers/currencies/currency[{}]/'.format(isoCode)
+            symbol = self.findOr(stem + 'symbol')
+            name = ';'.join(
+                self.findOr(stem + 'displayName' + tail)
+                for tail in ('',) + tuple(
+                    '[count={}]'.format(x) for x in ('zero', 'one', 'two', 'few', 'many', 'other')
+                )) + ';'
+        else:
+            symbol = name = ''
+        yield 'currencySymbol', symbol
+        yield 'currencyDisplayName', name
+
+    def numericData(self, lookup, complain = lambda text: None):
+        """Generate assorted numeric data for the locale.
+
+        First argument, lookup, is a callable that maps a numbering
+        system's name to certain data about the system, as a mapping;
+        we expect this to have u'digits' as a key.
+        """
+        system = self.find('numbers/defaultNumberingSystem')
+        stem = 'numbers/symbols[numberSystem={}]/'.format(system)
+        decimal = self.find(stem + 'decimal')
+        group = self.find(stem + 'group')
+        assert decimal != group, (self.name, system, decimal)
+        yield 'decimal', decimal
+        yield 'group', group
+        yield 'percent', self.find(stem + 'percentSign')
+        yield 'list', self.find(stem + 'list')
+        # FIXME: don't lower-case:
+        yield 'exp', self.find(stem + 'exponential').lower()
+
+        digits = lookup(system)['digits']
+        assert len(digits) == 10
+        zero = digits[0]
+        # Qt's number-formatting code assumes digits are consecutive:
+        assert all(ord(c) == i for i, c in enumerate(digits, ord(zero)))
+        yield 'zero', zero
+
+        plus = self.find(stem + 'plusSign')
+        minus = self.find(stem + 'minusSign')
+        yield 'plus', plus
+        yield 'minus', minus
+
+        # Currency formatting (currencyFormat may have a type field):
+        money = self.find('numbers/currencyFormats/currencyFormatLength/currencyFormat/pattern')
+        money = self.__currencyFormats(money, plus, minus)
+        yield 'currencyFormat', money.next()
+        neg = ''
+        for it in money:
+            assert not neg, 'There should be at most one more pattern'
+            neg = it
+        yield 'currencyNegativeFormat', neg
+
+    def textPatternData(self):
+        for key in ('quotationStart', 'alternateQuotationEnd',
+                    'quotationEnd', 'alternateQuotationStart'):
+            yield key, self.find('delimiters/' + key)
+
+        for key in ('start', 'middle', 'end'):
+            yield ('listPatternPart' + key.capitalize(),
+                   self.__fromLdmlListPattern(self.find(
+                        'listPatterns/listPattern/listPatternPart[{}]'.format(key))))
+        yield ('listPatternPartTwo',
+               self.__fromLdmlListPattern(self.find(
+                    'listPatterns/listPattern/listPatternPart[2]')))
+
+        stem = 'dates/calendars/calendar[gregorian]/'
+        # TODO: is wide really the right width to use here ?
+        # abbreviated might be an option ... or try both ?
+        meridiem = stem + 'dayPeriods/dayPeriodContext[format]/dayPeriodWidth[wide]/'
+        for key in ('am', 'pm'):
+            yield key, self.find(meridiem + 'dayPeriod[{}]'.format(key),
+                                 draft = Node.draftScore('contributed'))
+
+        for pair in (('long', 'full'), ('short', 'short')):
+            for key in ('time', 'date'):
+                yield (pair[0] + key.capitalize() + 'Format',
+                       convert_date(self.find(
+                            stem + '{}Formats/{}FormatLength[{}]/{}Format/pattern'.format(
+                                key, key, pair[1], key))))
+
+    def endonyms(self, language, script, country, variant):
+        # TODO: take variant into account ?
+        for seq in ((language, script, country),
+                    (language, script), (language, country), (language,)):
+            if not all(seq):
+                continue
+            try:
+                yield ('languageEndonym',
+                       self.find('localeDisplayNames/languages/language[{}]'
+                                 .format('_'.join(seq))))
+            except Error:
+                pass
+            else:
+                break
+        else:
+            # grumble(failed to find endonym for language)
+            yield 'languageEndonym', ''
+
+        yield ('countryEndonym',
+               self.findOr('localeDisplayNames/territories/territory[{}]'
+                           .format(country)))
+
+    def unitData(self):
+        yield ('byte_unit',
+               self.findOr('units/unitLength[long]/unit[digital-byte]/displayName',
+                           'bytes'))
+
+        unit = self.__findUnit('', 'B')
+        cache = [] # Populated by the SI call, to give hints to the IEC call
+        yield ('byte_si_quantified',
+               ';'.join(self.__unitCount('', unit, cache)))
+        # IEC 60027-2
+        # http://physics.nist.gov/cuu/Units/binary.html
+        yield ('byte_iec_quantified',
+               ';'.join(self.__unitCount('bi', 'iB', cache)))
+
+    def calendarNames(self, calendars):
+        namings = self.__nameForms
+        for cal in calendars:
+            stem = 'dates/calendars/calendar[' + cal + ']/months/'
+            for key, mode, size in namings:
+                prop = 'monthContext[' + mode + ']/monthWidth[' + size + ']/'
+                yield (key + 'Months_' + cal,
+                       ';'.join(self.find(stem + prop + 'month[{}]'.format(i))
+                                for i in range(1, 13)) + ';')
+
+        # Day data (for Gregorian, at least):
+        stem = 'dates/calendars/calendar[gregorian]/days/'
+        days = ('sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat')
+        for (key, mode, size) in namings:
+            prop = 'dayContext[' + mode + ']/dayWidth[' + size + ']/day'
+            yield (key + 'Days',
+                   ';'.join(self.find(stem + prop + '[' + day + ']')
+                            for day in days) + ';')
+
+    # Implementation details
+    __nameForms = (
+        ('standaloneLong', 'stand-alone', 'wide'),
+        ('standaloneShort', 'stand-alone', 'abbreviated'),
+        ('standaloneNarrow', 'stand-alone', 'narrow'),
+        ('long', 'format', 'wide'),
+        ('short', 'format', 'abbreviated'),
+        ('narrow', 'format', 'narrow'),
+        ) # Used for month and day names
+
+    def __findUnit(self, keySuffix, quantify, fallback=''):
+        # The displayName for a quantified unit in en.xml is kByte
+        # (even for unitLength[narrow]) instead of kB (etc.), so
+        # prefer any unitPattern provided, but prune its placeholder:
+        for size in ('short', 'narrow'): # TODO: reverse order ?
+            stem = 'units/unitLength[{}]/unit[digital-{}byte]/'.format(size + keySuffix, quantify)
+            for count in ('many', 'few', 'two', 'other', 'zero', 'one'):
+                try:
+                    ans = self.find(stem + 'unitPattern[count={}]'.format(count))
+                except Error:
+                    continue
+
+                # TODO: do count-handling, instead of discarding placeholders
+                if False: # TODO: do it this way, instead !
+                    ans = ans.replace('{0}', '').strip()
+                elif ans.startswith('{0}'):
+                    ans = ans[3:].lstrip()
+                if ans:
+                    return ans
+
+            try:
+                return self.find(stem + 'displayName')
+            except Error:
+                pass
+
+        return fallback
+
+    def __unitCount(self, keySuffix, suffix, cache,
+                    # Stop at exa/exbi: 16 exbi = 2^{64} < zetta =
+                    # 1000^7 < zebi = 2^{70}, the next quantifiers up:
+                    siQuantifiers = ('kilo', 'mega', 'giga', 'tera', 'peta', 'exa')):
+        """Work out the unit quantifiers.
+
+        Unfortunately, the CLDR data only go up to terabytes and we
+        want all the way to exabytes; but we can recognize the SI
+        quantifiers as prefixes, strip and identify the tail as the
+        localized translation for 'B' (e.g. French has 'octet' for
+        'byte' and uses ko, Mo, Go, To from which we can extrapolate
+        Po, Eo).
+
+        Should be called first for the SI quantifiers, with suffix =
+        'B', then for the IEC ones, with suffix = 'iB'; the list cache
+        (initially empty before first call) is used to let the second
+        call know what the first learned about the localized unit.
+        """
+        if suffix == 'iB': # second call, re-using first's cache
+            if cache:
+                byte = cache.pop()
+                if all(byte == k for k in cache):
+                    suffix = 'i' + byte
+            for q in siQuantifiers:
+                # Those don't (yet, v36) exist in CLDR, so we always get the fall-back:
+                yield self.__findUnit(keySuffix, q[:2], q[0].upper() + suffix)
+        else: # first call
+            tail = suffix = suffix or 'B'
+            for q in siQuantifiers:
+                it = self.__findUnit(keySuffix, q)
+                # kB for kilobyte, in contrast with KiB for IEC:
+                q = q[0] if q == 'kilo' else q[0].upper()
+                if not it:
+                    it = q + tail
+                elif it.startswith(q):
+                    rest = it[1:]
+                    tail = rest if all(rest == k for k in cache) else suffix
+                    cache.append(rest)
+                yield it
+
+    @staticmethod
+    def __currencyFormats(patterns, plus, minus):
+        for p in patterns.split(';'):
+            p = p.replace('0', '#').replace(',', '').replace('.', '')
+            try:
+                cut = p.find('#') + 1
+            except ValueError:
+                pass
+            else:
+                p = p[:cut] + p[cut:].replace('#', '')
+            p = p.replace('#', "%1")
+            # According to http://www.unicode.org/reports/tr35/#Number_Format_Patterns
+            # there can be doubled or trippled currency sign, however none of the
+            # locales use that.
+            p = p.replace(u'\xa4', "%2")
+            # Single quote goes away, but double goes to single:
+            p = p.replace("''", '###').replace("'", '').replace('###', "'")
+            # Use number system's signs:
+            p = p.replace('+', plus).replace('-', minus)
+            yield p
+
+    @staticmethod
+    def __fromLdmlListPattern(pattern):
+        # This is a very limited parsing of the format for list pattern part only.
+        return pattern.replace('{0}', '%1').replace('{1}', '%2').replace('{2}', '%3')
+
+    @staticmethod
+    def __fromLdmlPath(seq): # tool function for __xpathJoin()
+        """Convert LDML's [@name='value'] to our [name=value] form."""
+        for it in seq:
+            # First dismember it:
+            attrs = it.split('[')
+            tag = attrs.pop(0)
+            if not attrs: # Short-cut the easy case:
+                yield it
+                continue
+
+            assert all(x.endswith(']') for x in attrs)
+            attrs = [x[:-1].split('=') for x in attrs]
+            # Then fix each attribute specification in it:
+            attrs = [(x[0][1:] if x[0].startswith('@') else x[0],
+                      x[1][1:-1] if x[1].startswith("'") and x[1].endswith("'") else x[1])
+                     for x in attrs]
+            # Finally, put it all back together:
+            attrs = ['='.join(x) + ']' for x in attrs]
+            attrs.insert(0, tag)
+            yield '['.join(attrs)
+
+    @classmethod
+    def __xpathJoin(cls, head, insert, tail):
+        """Join three lists of XPath selectors.
+
+        Each of head, insert and tail is a sequence of selectors but
+        insert may start with some uses of '..', that we want to
+        resolve away, and may use LDML's attribute format, that we
+        want to convert to our format."""
+        while insert and insert[0] == '..':
+            insert.pop(0)
+            head.pop()
+        return head + list(cls.__fromLdmlPath(insert)) + tail
diff --git a/util/locale_database/qlocalexml2cpp.py b/util/locale_database/qlocalexml2cpp.py
index 59161ed9d0..1938be19ea 100755
--- a/util/locale_database/qlocalexml2cpp.py
+++ b/util/locale_database/qlocalexml2cpp.py
@@ -480,7 +480,7 @@ def main(args, out, err):
         return 1
 
     reader = QLocaleXmlReader(qlocalexml)
-    locale_map = dict(reader.loadLocaleMap(calendars, sys.stderr.write))
+    locale_map = dict(reader.loadLocaleMap(calendars, err.write))
 
     locale_keys = locale_map.keys()
     compareLocaleKeys.default_map = dict(reader.defaultMap())
diff --git a/util/locale_database/xpathlite.py b/util/locale_database/xpathlite.py
deleted file mode 100644
index 3da8b24656..0000000000
--- a/util/locale_database/xpathlite.py
+++ /dev/null
@@ -1,284 +0,0 @@
-#!/usr/bin/env python
-#############################################################################
-##
-## Copyright (C) 2016 The Qt Company Ltd.
-## Contact: https://www.qt.io/licensing/
-##
-## This file is part of the test suite of the Qt Toolkit.
-##
-## $QT_BEGIN_LICENSE:GPL-EXCEPT$
-## Commercial License Usage
-## Licensees holding valid commercial Qt licenses may use this file in
-## accordance with the commercial license agreement provided with the
-## Software or, alternatively, in accordance with the terms contained in
-## a written agreement between you and The Qt Company. For licensing terms
-## and conditions see https://www.qt.io/terms-conditions. For further
-## information use the contact form at https://www.qt.io/contact-us.
-##
-## GNU General Public License Usage
-## Alternatively, this file may be used under the terms of the GNU
-## General Public License version 3 as published by the Free Software
-## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
-## included in the packaging of this file. Please review the following
-## information to ensure the GNU General Public License requirements will
-## be met: https://www.gnu.org/licenses/gpl-3.0.html.
-##
-## $QT_END_LICENSE$
-##
-#############################################################################
-
-import sys
-import os
-import xml.dom.minidom
-
-from localetools import Error
-
-class DraftResolution:
-    # See http://www.unicode.org/cldr/process.html for description
-    unconfirmed = 'unconfirmed'
-    provisional = 'provisional'
-    contributed = 'contributed'
-    approved = 'approved'
-    _values = { unconfirmed : 1, provisional : 2, contributed : 3, approved : 4 }
-    def __init__(self, resolution):
-        self.resolution = resolution
-    def toInt(self):
-        return DraftResolution._values[self.resolution]
-
-doc_cache = {}
-def parseDoc(file):
-    if not doc_cache.has_key(file):
-        doc_cache[file] = xml.dom.minidom.parse(file)
-    return doc_cache[file]
-
-def findChild(parent, tag_name, arg_name=None, arg_value=None, draft=None):
-    for node in parent.childNodes:
-        if node.nodeType != node.ELEMENT_NODE:
-            continue
-        if node.nodeName != tag_name:
-            continue
-        if arg_value:
-            if not node.attributes.has_key(arg_name):
-                continue
-            if node.attributes[arg_name].nodeValue != arg_value:
-                continue
-        if draft:
-            if not node.attributes.has_key('draft'):
-                # if draft is not specified then it's approved
-                return node
-            value = node.attributes['draft'].nodeValue
-            value = DraftResolution(value).toInt()
-            exemplar = DraftResolution(draft).toInt()
-            if exemplar > value:
-                continue
-        return node
-    return False
-
-def codeMapsFromFile(file):
-    """Extract mappings of language, script and country codes to names.
-
-    The file shall typically be common/main/en.xml, which contains a
-    localeDisplayNames element with children languages, scripts and
-    territories; each element in each of these has a code as its type
-    attribute and its name as element content.  This returns a mapping
-    withe keys 'language', 'script' and 'country', each of which
-    has, as value, a mapping of the relevant codes to names.
-    """
-    parent = findChild(findChild(parseDoc(file), 'ldml'), 'localeDisplayNames')
-    keys, result = {'languages': 'language', 'scripts': 'script', 'territories': 'country'}, {}
-    for src, dst in keys.items():
-        child = findChild(parent, src)
-        data = result[dst] = {}
-        for elt in child.childNodes:
-            if elt.attributes and elt.attributes.has_key('type'):
-                key, value = elt.attributes['type'].value, elt.childNodes[0].wholeText
-                # Don't over-write previously-read data for an alt form:
-                if elt.attributes.has_key('alt') and data.has_key(key):
-                    continue
-                data[key] = value
-
-    return result
-
-def findTagsInFile(file, path):
-    doc = parseDoc(file)
-
-    elt = doc.documentElement
-    tag_spec_list = path.split("/")
-    last_entry = None
-    for tag_spec in tag_spec_list:
-        tag_name = tag_spec
-        arg_name = 'type'
-        arg_value = ''
-        left_bracket = tag_spec.find('[')
-        if left_bracket != -1:
-            tag_name = tag_spec[:left_bracket]
-            arg_value = tag_spec[left_bracket+1:-1].split("=")
-            if len(arg_value) == 2:
-                arg_name = arg_value[0]
-                arg_value = arg_value[1]
-            else:
-                arg_value = arg_value[0]
-        elt = findChild(elt, tag_name, arg_name, arg_value)
-        if not elt:
-            return None
-    ret = []
-    if elt.childNodes:
-        for node in elt.childNodes:
-            if node.attributes:
-                element = [node.nodeName, None]
-                element[1] = node.attributes.items()
-                ret.append(element)
-    else:
-        if elt.attributes:
-            element = [elt.nodeName, None]
-            element[1] = elt.attributes.items()
-            ret.append(element)
-    return ret
-
-def _findEntryInFile(file, path, draft=None, attribute=None):
-    doc = parseDoc(file)
-
-    elt = doc.documentElement
-    tag_spec_list = path.split("/")
-    last_entry = None
-    for i in range(len(tag_spec_list)):
-        tag_spec = tag_spec_list[i]
-        tag_name = tag_spec
-        arg_name = 'type'
-        arg_value = ''
-        left_bracket = tag_spec.find('[')
-        if left_bracket != -1:
-            tag_name = tag_spec[:left_bracket]
-            arg_value = tag_spec[left_bracket+1:-1].split("=")
-            if len(arg_value) == 2:
-                arg_name = arg_value[0].replace("@", "").replace("'", "")
-                arg_value = arg_value[1]
-            else:
-                arg_value = arg_value[0]
-        alias = findChild(elt, 'alias')
-        if alias and alias.attributes['source'].nodeValue == 'locale':
-            path = alias.attributes['path'].nodeValue
-            aliaspath = tag_spec_list[:i] + path.split("/")
-            def resolve(x, y):
-                if y == '..':
-                    return x[:-1]
-                return x + [y]
-            # resolve all dot-dot parts of the path
-            aliaspath = reduce(resolve, aliaspath, [])
-            # remove attribute specification that our xpathlite doesnt support
-            aliaspath = map(lambda x: x.replace("@type=", "").replace("'", ""), aliaspath)
-            # append the remaining path
-            aliaspath = aliaspath + tag_spec_list[i:]
-            aliaspath = "/".join(aliaspath)
-            # "locale" aliases are special - we need to start lookup from scratch
-            return (None, aliaspath)
-        elt = findChild(elt, tag_name, arg_name, arg_value, draft)
-        if not elt:
-            return ("", None)
-    if attribute is not None:
-        if elt.attributes.has_key(attribute):
-            return (elt.attributes[attribute].nodeValue, None)
-        return (None, None)
-    try:
-        return (elt.firstChild.nodeValue, None)
-    except:
-        pass
-    return (None, None)
-
-def findAlias(file):
-    doc = parseDoc(file)
-
-    alias_elt = findChild(doc.documentElement, "alias")
-    if not alias_elt:
-        return False
-    if not alias_elt.attributes.has_key('source'):
-        return False
-    return alias_elt.attributes['source'].nodeValue
-
-lookup_chain_cache = {}
-parent_locales = {}
-def _fixedLookupChain(dirname, name):
-    if lookup_chain_cache.has_key(name):
-        return lookup_chain_cache[name]
-
-    # see http://www.unicode.org/reports/tr35/#Parent_Locales
-    if not parent_locales:
-        for ns in findTagsInFile(dirname + "/../supplemental/supplementalData.xml", "parentLocales"):
-            tmp = {}
-            parent_locale = ""
-            for data in ns[1:][0]: # ns looks like this: [u'parentLocale', [(u'parent', u'root'), (u'locales', u'az_Cyrl bs_Cyrl en_Dsrt ..')]]
-                tmp[data[0]] = data[1]
-                if data[0] == u"parent":
-                    parent_locale = data[1]
-            parent_locales[parent_locale] = tmp[u"locales"].split(" ")
-
-    items = name.split("_")
-    # split locale name into items and iterate through them from back to front
-    # example: az_Latn_AZ => [az_Latn_AZ, az_Latn, az]
-    items = list(reversed(map(lambda x: "_".join(items[:x+1]), range(len(items)))))
-
-    for i in range(len(items)):
-        item = items[i]
-        for parent_locale in parent_locales.keys():
-            for locale in parent_locales[parent_locale]:
-                if item == locale:
-                    if parent_locale == u"root":
-                        items = items[:i+1]
-                    else:
-                        items = items[:i+1] + _fixedLookupChain(dirname, parent_locale)
-                    lookup_chain_cache[name] = items
-                    return items
-
-    lookup_chain_cache[name] = items
-    return items
-
-def _findEntry(base, path, draft=None, attribute=None):
-    if base.endswith(".xml"):
-        base = base[:-4]
-    (dirname, filename) = os.path.split(base)
-
-    items = _fixedLookupChain(dirname, filename)
-    for item in items:
-        file = dirname + "/" + item + ".xml"
-        if os.path.isfile(file):
-            alias = findAlias(file)
-            if alias:
-                # if alias is found we should follow it and stop processing current file
-                # see http://www.unicode.org/reports/tr35/#Common_Elements
-                aliasfile = os.path.dirname(file) + "/" + alias + ".xml"
-                if not os.path.isfile(aliasfile):
-                    raise Error("findEntry: fatal error: found an alias '%s' to '%s', but the alias file couldn't be found" % (filename, alias))
-                # found an alias, recurse into parsing it
-                result = _findEntry(aliasfile, path, draft, attribute)
-                return result
-            (result, aliaspath) = _findEntryInFile(file, path, draft, attribute)
-            if aliaspath:
-                # start lookup again because of the alias source="locale"
-                return _findEntry(base, aliaspath, draft, attribute)
-            if result:
-                return result
-    return None
-
-def findEntry(base, path, draft=None, attribute=None):
-    file = base
-    if base.endswith(".xml"):
-        file = base
-        base = base[:-4]
-    else:
-        file = base + ".xml"
-    (dirname, filename) = os.path.split(base)
-
-    result = None
-    while path:
-        result = _findEntry(base, path, draft, attribute)
-        if result:
-            return result
-        (result, aliaspath) = _findEntryInFile(dirname + "/root.xml", path, draft, attribute)
-        if result:
-            return result
-        if not aliaspath:
-            raise Error("findEntry: fatal error: %s: cannot find key %s" % (filename, path))
-        path = aliaspath
-
-    return result
-
-- 
cgit v1.2.3