Improve init-repository.py to fetch up-to-date dependency list

This patch introduces a new mechanism to retrieve submodules.
The utilities in version_resolver.py provide means to download
the upstream DEPS file for a specified Chromium version.
This makes it possible to check out the needed svn branches
and specific revisions used for the release. We still use the
git repositories found in the upstream .DEPS.git file but since
it is not regularly updated after the branch-off, we try to match
the release svn revisions to git shasums which is possible since
each commit includes git-svn-id's.
Also add a maintenance tool get_version.py which reads the Chrome
release information from http://omahaproxy.appspot.com.

We now pin the latest stable channel Chromium release from branch
1650 with the version number 31.0.1650.63 and a branch-off date of
Sept 23rd.

Change-Id: Ibbf565f876af79dc008bcd161fddcbdd8bc73977
Reviewed-by: Jocelyn Turcotte <jocelyn.turcotte@digia.com>

3 files changed, 303 insertions, 83 deletions

diff --git a/tools/scripts/get_version.py b/tools/scripts/get_version.py
new file mode 100755
index 000000000..c9f4534bc
--- /dev/null
+++ b/tools/scripts/get_version.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+#############################################################################
+#
+# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
+# Contact: http://www.qt-project.org/legal
+#
+# This file is part of the QtWebEngine module of the Qt Toolkit.
+#
+# $QT_BEGIN_LICENSE:LGPL$
+# Commercial License Usage
+# Licensees holding valid commercial Qt licenses may use this file in
+# accordance with the commercial license agreement provided with the
+# Software or, alternatively, in accordance with the terms contained in
+# a written agreement between you and Digia.  For licensing terms and
+# conditions see http://qt.digia.com/licensing.  For further information
+# use the contact form at http://qt.digia.com/contact-us.
+#
+# GNU Lesser General Public License Usage
+# Alternatively, this file may be used under the terms of the GNU Lesser
+# General Public License version 2.1 as published by the Free Software
+# Foundation and appearing in the file LICENSE.LGPL included in the
+# packaging of this file.  Please review the following information to
+# ensure the GNU Lesser General Public License version 2.1 requirements
+# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+#
+# In addition, as a special exception, Digia gives you certain additional
+# rights.  These rights are described in the Digia Qt LGPL Exception
+# version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+#
+# GNU General Public License Usage
+# Alternatively, this file may be used under the terms of the GNU
+# General Public License version 3.0 as published by the Free Software
+# Foundation and appearing in the file LICENSE.GPL included in the
+# packaging of this file.  Please review the following information to
+# ensure the GNU General Public License version 3.0 requirements will be
+# met: http://www.gnu.org/copyleft/gpl.html.
+#
+#
+# $QT_END_LICENSE$
+#
+#############################################################################
+
+import sys
+import json
+import version_resolver as resolver
+
+channels = resolver.readReleaseChannels()
+
+if len(sys.argv) == 1 or sys.argv[1] == 'all':
+    print(json.dumps(channels, sort_keys=True, indent=2))
+
+for platform in sys.argv:
+    if platform in channels:
+        print('"' + platform + '": ' + json.dumps(channels[platform], sort_keys=True, indent=2))
diff --git a/tools/scripts/git_submodule.py b/tools/scripts/git_submodule.py
index 8e088e5bd..c17e2926d 100644
--- a/tools/scripts/git_submodule.py
+++ b/tools/scripts/git_submodule.py
@@ -43,6 +43,7 @@ import glob
 import os
 import subprocess
 import sys
+import version_resolver as resolver
 
 extra_os = []
 
@@ -75,25 +76,65 @@ class DEPSParser:
                 subdir = dep
                 if subdir.startswith('src/'):
                     subdir = subdir[4:]
-                submodule = Submodule(subdir, repo, rev, os)
-                submodule.deps = True
+                if subdir.startswith('src'):
+                    # Ignore the information about chromium since we get that from git.
+                    continue
+                shasum = ''
+                if len(rev) == 40: # Length of a git shasum
+                    shasum = rev
+                submodule = Submodule(subdir, repo)
+                submodule.os = os
+                submodule.shasum = shasum
+                if not submodule.shasum:
+                    # We need to parse the svn branch and revision number.
+                    ref = repo
+                    # Try to find out the branch.
+                    ref_path = repo.split('branches/')
+                    if len(ref_path) > 1:
+                        ref = ref_path[1]
+                    if 'trunk' in ref or 'svn' in ref:
+                        # Trunk is used, so we can use the remote master.
+                        ref = 'master'
+                    name_path = subdir.split('/')
+                    if len(name_path) > 1:
+                        # At this point some svn repository paths still include the repo name
+                        # after the actual branch so we have to strip it together with the leading /
+                        name = name_path[-1]
+                        if ref.endswith(name):
+                            branch = ref[:-(len(name) + 1)]
+                            ref = 'refs/branch-heads/' + branch
+                        if name in ref:
+                            # At this point the last partition in the path
+                            # is the branch name, so compose the git ref.
+                            branch = ref.split('/')[-1]
+                            ref = 'refs/branch-heads/' + branch
+                    if 'master' not in ref and 'refs/branch-heads' not in ref:
+                        # Finally compose the refs that did not mach previous rules.
+                        ref = 'refs/branch-heads/' + ref
+                    submodule.ref = ref
+                    submodule.revision = int(rev)
                 submodules.append(submodule)
         return submodules
 
     def sanityCheckModules(self, submodules):
-        subdirectories = []
+        submodule_dict = {}
         for submodule in submodules:
-            if submodule.path in subdirectories:
-                print 'SUBMODULE WARNING: duplicate for submodule' + submodule.path
-            subdirectories.append(submodule.path)
+            if not submodule.matchesOS():
+                print '-- skipping ' + submodule.path + ' for this operating system. --'
+                continue
+            if submodule.path in submodule_dict:
+                prev_module = submodule_dict[submodule.path]
+                # We might have to create our own DEPS file if different platforms use different branches,
+                # but for now it should be safe to select the latest revision from the requirements.
+                if submodule.shasum or prev_module.revision >= submodule.revision:
+                    continue
+                if prev_module.ref != submodule.ref:
+                    sys.exit('ERROR: branch mismatch for ' + submodule.path + '(' + prev_module.ref + ' vs ' + submodule.ref + ')')
+                print('Duplicate submodule ' + submodule.path + '. Using latest revison ' + str(submodule.revision) + '.')
+            submodule_dict[submodule.path] = submodule
+        return list(submodule_dict.values())
 
-    def parseFile(self, deps_file_name):
-        currentDir = os.getcwd()
-        if not os.path.isfile(deps_file_name):
-            return []
-        deps_file = open(deps_file_name)
-        deps_content = deps_file.read().decode('utf-8')
-        deps_file.close()
+    def parse(self, deps_content):
         exec(deps_content, self.global_scope, self.local_scope)
 
         submodules = []
@@ -101,9 +142,16 @@ class DEPSParser:
         for os_dep in self.local_scope['deps_os']:
             submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps_os'][os_dep], os_dep))
 
-        self.sanityCheckModules(submodules)
+        return self.sanityCheckModules(submodules)
 
-        return submodules
+    def parseFile(self, deps_file_name):
+        currentDir = os.getcwd()
+        if not os.path.isfile(deps_file_name):
+            return []
+        deps_file = open(deps_file_name)
+        deps_content = deps_file.read().decode('utf-8')
+        deps_file.close()
+        return self.parse(deps_content)
 
 
 
@@ -114,7 +162,7 @@ class Submodule:
         self.shasum = shasum
         self.os = os
         self.ref = ''
-        self.deps = False
+        self.revision = None
 
     def matchesOS(self):
         if not self.os:
@@ -132,18 +180,45 @@ class Submodule:
                 return True
         return False
 
-    def findSha(self):
-        if self.shasum != '':
-            return
-        line = subprocessCheckOutput(['git', 'submodule', 'status', self.path])
-        line = line.lstrip(' -')
-        self.shasum = line.split(' ')[0]
+    def findShaAndCheckout(self):
+        oldCwd = os.getcwd()
+        os.chdir(self.path)
+        error = 0
+        if self.ref:
+            # Fetch the ref we parsed from the DEPS file.
+            val = subprocessCall(['git', 'fetch', 'origin', self.ref])
+            if val != 0:
+                sys.exit("Could not fetch branch from upstream " + self.ref)
+            error = subprocessCall(['git', 'checkout', 'FETCH_HEAD']);
+
+            search_string = ''
+            if self.path.endswith('/chromium'):
+                search_string = resolver.currentVersion()
+            elif self.revision:
+                search_string = '@' + str(self.revision) + ' '
+            if search_string:
+                line = subprocessCheckOutput(['git', 'log', '-n1', '--pretty=oneline', '--grep=' + search_string])
+                if line:
+                    self.shasum = line.split(' ')[0]
+            else: # No revision set, use the submodule shasum
+                os.chdir(oldCwd)
+                line = subprocessCheckOutput(['git', 'submodule', 'status', self.path])
+                os.chdir(self.path)
+                line = line.lstrip(' -')
+                self.shasum = line.split(' ')[0]
+
+        current_shasum = subprocessCheckOutput(['git', 'show', '-s', '--oneline']).split(' ')[0]
+        if not self.shasum.startswith(current_shasum):
+            # In case HEAD differs check out the actual shasum we require.
+            error = subprocessCall(['git', 'checkout', self.shasum])
+        os.chdir(oldCwd)
+        return error
 
     def findGitDir(self):
         try:
             return subprocessCheckOutput(['git', 'rev-parse', '--git-dir']).strip()
         except subprocess.CalledProcessError, e:
-            sys.exit("git dir could not be determined! - Initialization failed!" + e.output)
+            sys.exit("git dir could not be determined! - Initialization failed! " + e.output)
 
     def reset(self):
         currentDir = os.getcwd()
@@ -167,26 +242,14 @@ class Submodule:
             print '-- initializing ' + self.path + ' --'
             if os.path.isdir(self.path):
                 self.reset()
-            if self.deps:
+
+            if self.url:
                 subprocessCall(['git', 'submodule', 'add', '-f', self.url, self.path])
             subprocessCall(['git', 'submodule', 'init', self.path])
             subprocessCall(['git', 'submodule', 'update', self.path])
-            self.findSha()
-            currentDir = os.getcwd()
-            os.chdir(self.path)
-            # Make sure we have checked out the right shasum.
-            # In case there were other patches applied before.
-            if self.ref:
-                val = subprocessCall(['git', 'fetch', 'origin', self.ref]);
-                if val != 0:
-                    sys.exit("Could not fetch branch from upstream.")
-                subprocessCall(['git', 'checkout', 'FETCH_HEAD']);
-            else:
-                val = subprocessCall(['git', 'checkout', self.shasum])
-                if val != 0:
-                    sys.exit("!!! initialization failed !!!")
-            self.initSubmodules()
-            os.chdir(currentDir)
+
+            if self.findShaAndCheckout() != 0:
+                sys.exit("!!! initialization failed !!!")
         else:
             print '-- skipping ' + self.path + ' for this operating system. --'
 
@@ -203,53 +266,49 @@ class Submodule:
 
 
     def readSubmodules(self):
-        if not os.path.isfile('.gitmodules'):
-            return []
-        gitmodules_file = open('.gitmodules')
-        gitmodules_lines = gitmodules_file.readlines()
-        gitmodules_file.close()
-
         submodules = []
-        currentSubmodule = None
-        for line in gitmodules_lines:
-            if line.find('[submodule') == 0:
-                if currentSubmodule:
-                    submodules.append(currentSubmodule)
-                currentSubmodule = Submodule()
-            tokens = line.split('=')
-            if len(tokens) >= 2:
-                key = tokens[0].strip()
-                value = tokens[1].strip()
-                if key == 'path':
-                    currentSubmodule.path = value
-                elif key == 'url':
-                    currentSubmodule.url = value
-                elif key == 'os':
-                    currentSubmodule.os = value.split(',')
-        if currentSubmodule:
-            submodules.append(currentSubmodule)
-        return submodules
+        if self.ref:
+            submodules = resolver.readSubmodules()
+            print 'DEPS file provides the following submodules:'
+            for submodule in submodules:
+                submodule_ref = submodule.shasum
+                if submodule.revision:
+                    submodule_ref = submodule.ref + '@' + str(submodule.revision)
+                print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule_ref
+        else: # Try .gitmodules since no ref has been specified
+            if not os.path.isfile('.gitmodules'):
+                return []
+            gitmodules_file = open('.gitmodules')
+            gitmodules_lines = gitmodules_file.readlines()
+            gitmodules_file.close()
 
-    def readDeps(self):
-        parser = DEPSParser()
-        return parser.parseFile('.DEPS.git')
+            submodules = []
+            currentSubmodule = None
+            for line in gitmodules_lines:
+                if line.find('[submodule') == 0:
+                    if currentSubmodule:
+                        submodules.append(currentSubmodule)
+                    currentSubmodule = Submodule()
+                tokens = line.split('=')
+                if len(tokens) >= 2:
+                    key = tokens[0].strip()
+                    value = tokens[1].strip()
+                    if key == 'path':
+                        currentSubmodule.path = value
+                    elif key == 'url':
+                        currentSubmodule.url = value
+                    elif key == 'os':
+                        currentSubmodule.os = value.split(',')
+            if currentSubmodule:
+                submodules.append(currentSubmodule)
+        return submodules
 
     def initSubmodules(self):
-        # try reading submodules from .gitmodules.
+        oldCwd = os.getcwd()
+        os.chdir(self.path)
         submodules = self.readSubmodules()
         for submodule in submodules:
             submodule.initialize()
-        if submodules:
-            return
-        # if we could not find any submodules in .gitmodules, try .DEPS.git
-        submodules = self.readDeps()
-
-        if not submodules:
-            return
-
-        print 'DEPS file provides the following submodules:'
-        for submodule in submodules:
-            print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule.shasum
-        for submodule in submodules:
-            submodule.initialize()
-        subprocessCall(['git', 'commit', '-a', '-m', '"initialize submodules"'])
+        if self.ref:
+            subprocessCall(['git', 'commit', '-a', '-m', 'initialize submodules'])
+        os.chdir(oldCwd)
diff --git a/tools/scripts/version_resolver.py b/tools/scripts/version_resolver.py
new file mode 100644
index 000000000..94ba9e1ab
--- /dev/null
+++ b/tools/scripts/version_resolver.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+
+#############################################################################
+#
+# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
+# Contact: http://www.qt-project.org/legal
+#
+# This file is part of the QtWebEngine module of the Qt Toolkit.
+#
+# $QT_BEGIN_LICENSE:LGPL$
+# Commercial License Usage
+# Licensees holding valid commercial Qt licenses may use this file in
+# accordance with the commercial license agreement provided with the
+# Software or, alternatively, in accordance with the terms contained in
+# a written agreement between you and Digia.  For licensing terms and
+# conditions see http://qt.digia.com/licensing.  For further information
+# use the contact form at http://qt.digia.com/contact-us.
+#
+# GNU Lesser General Public License Usage
+# Alternatively, this file may be used under the terms of the GNU Lesser
+# General Public License version 2.1 as published by the Free Software
+# Foundation and appearing in the file LICENSE.LGPL included in the
+# packaging of this file.  Please review the following information to
+# ensure the GNU Lesser General Public License version 2.1 requirements
+# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+#
+# In addition, as a special exception, Digia gives you certain additional
+# rights.  These rights are described in the Digia Qt LGPL Exception
+# version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+#
+# GNU General Public License Usage
+# Alternatively, this file may be used under the terms of the GNU
+# General Public License version 3.0 as published by the Free Software
+# Foundation and appearing in the file LICENSE.GPL included in the
+# packaging of this file.  Please review the following information to
+# ensure the GNU General Public License version 3.0 requirements will be
+# met: http://www.gnu.org/copyleft/gpl.html.
+#
+#
+# $QT_END_LICENSE$
+#
+#############################################################################
+
+import os
+import sys
+import json
+import urllib2
+import git_submodule as GitSubmodule
+
+chromium_version = '31.0.1650.63'
+chromium_branch = '1650'
+
+json_url = 'http://omahaproxy.appspot.com/all.json'
+git_deps_url = 'http://src.chromium.org/chrome/branches/' + chromium_branch + '/src/.DEPS.git'
+base_deps_url = 'http://src.chromium.org/chrome/releases/'
+
+def currentVersion():
+    return chromium_version
+
+def currentBranch():
+    return chromium_branch
+
+def readReleaseChannels():
+    response = urllib2.urlopen(json_url)
+    raw_json = response.read().strip()
+    data = json.loads(raw_json)
+    channels = {}
+
+    for obj in data:
+        os = obj['os']
+        channels[os] = []
+        for ver in obj['versions']:
+            channels[os].append({ 'channel': ver['channel'], 'version': ver['version'], 'branch': ver['true_branch'] })
+    return channels
+
+def readSubmodules():
+    response = urllib2.urlopen(base_deps_url + chromium_version + '/DEPS')
+    svn_deps = response.read().strip()
+
+    response = urllib2.urlopen(git_deps_url)
+    git_deps = response.read().strip()
+
+    parser = GitSubmodule.DEPSParser()
+    svn_submodules = parser.parse(svn_deps)
+    git_submodules = parser.parse(git_deps)
+
+    submodule_dict = {}
+    git_dict = {}
+
+    for sub in git_submodules:
+        git_dict[sub.path] = sub
+
+    for sub in svn_submodules:
+        if 'reference_build' not in sub.path and (sub.revision or sub.shasum) and sub.path in git_dict:
+            submodule_dict[sub.path] = sub
+
+    for git in git_submodules:
+        if git.path in submodule_dict:
+            # We'll use the git repository instead of svn.
+            module = submodule_dict[git.path]
+            module.url = git.url
+            if not module.shasum:
+                # We use the git shasum as fallback.
+                module.shasum = git.shasum
+
+    return list(submodule_dict.values())