diff options
-rwxr-xr-x | init-repository.py | 9 | ||||
-rwxr-xr-x | tools/scripts/get_version.py | 55 | ||||
-rw-r--r-- | tools/scripts/git_submodule.py | 225 | ||||
-rw-r--r-- | tools/scripts/version_resolver.py | 106 |
4 files changed, 306 insertions, 89 deletions
diff --git a/init-repository.py b/init-repository.py index 2affa196a..1716a289a 100755 --- a/init-repository.py +++ b/init-repository.py @@ -52,6 +52,7 @@ qtwebengine_root = os.path.abspath(os.path.join(os.path.dirname(__file__))) sys.path.append(os.path.join(qtwebengine_root, 'tools', 'scripts')) import git_submodule as GitSubmodule +import version_resolver as resolver chromium_src = os.environ.get('CHROMIUM_SRC_DIR') ninja_src = os.path.join(qtwebengine_root, 'src/3rdparty_upstream/ninja') @@ -119,11 +120,7 @@ def initUpstreamSubmodules(): ninja_url = 'https://github.com/martine/ninja.git' chromium_url = 'https://chromium.googlesource.com/chromium/src.git' ninja_shasum = '40b51a0b986b8675e15b0cd1b10c272bf51fdb84' - chromium_shasum = '29d2d710e0e7961dff032ad4ab73887cc33122bb' - # Do not define a branch for now. - # We will turn this on, once we actually switch to using the release branch. - #chromium_ref = 'refs/branch-heads/1599' - chromium_ref = '' + chromium_ref = 'refs/branch-heads/' + resolver.currentBranch() os.chdir(qtwebengine_root) current_submodules = subprocess.check_output(['git', 'submodule']) @@ -142,13 +139,13 @@ def initUpstreamSubmodules(): if not use_external_chromium: chromiumSubmodule = GitSubmodule.Submodule() chromiumSubmodule.path = 'src/3rdparty_upstream/chromium' - chromiumSubmodule.shasum = chromium_shasum chromiumSubmodule.ref = chromium_ref chromiumSubmodule.url = chromium_url chromiumSubmodule.os = 'all' if args.android: GitSubmodule.extra_os = ['android'] chromiumSubmodule.initialize() + chromiumSubmodule.initSubmodules() def initSnapshot(): snapshot = GitSubmodule.Submodule() diff --git a/tools/scripts/get_version.py b/tools/scripts/get_version.py new file mode 100755 index 000000000..c9f4534bc --- /dev/null +++ b/tools/scripts/get_version.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +############################################################################# +# +# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). +# Contact: http://www.qt-project.org/legal +# +# This file is part of the QtWebEngine module of the Qt Toolkit. +# +# $QT_BEGIN_LICENSE:LGPL$ +# Commercial License Usage +# Licensees holding valid commercial Qt licenses may use this file in +# accordance with the commercial license agreement provided with the +# Software or, alternatively, in accordance with the terms contained in +# a written agreement between you and Digia. For licensing terms and +# conditions see http://qt.digia.com/licensing. For further information +# use the contact form at http://qt.digia.com/contact-us. +# +# GNU Lesser General Public License Usage +# Alternatively, this file may be used under the terms of the GNU Lesser +# General Public License version 2.1 as published by the Free Software +# Foundation and appearing in the file LICENSE.LGPL included in the +# packaging of this file. Please review the following information to +# ensure the GNU Lesser General Public License version 2.1 requirements +# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +# +# In addition, as a special exception, Digia gives you certain additional +# rights. These rights are described in the Digia Qt LGPL Exception +# version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +# +# GNU General Public License Usage +# Alternatively, this file may be used under the terms of the GNU +# General Public License version 3.0 as published by the Free Software +# Foundation and appearing in the file LICENSE.GPL included in the +# packaging of this file. Please review the following information to +# ensure the GNU General Public License version 3.0 requirements will be +# met: http://www.gnu.org/copyleft/gpl.html. +# +# +# $QT_END_LICENSE$ +# +############################################################################# + +import sys +import json +import version_resolver as resolver + +channels = resolver.readReleaseChannels() + +if len(sys.argv) == 1 or sys.argv[1] == 'all': + print(json.dumps(channels, sort_keys=True, indent=2)) + +for platform in sys.argv: + if platform in channels: + print('"' + platform + '": ' + json.dumps(channels[platform], sort_keys=True, indent=2)) diff --git a/tools/scripts/git_submodule.py b/tools/scripts/git_submodule.py index 8e088e5bd..c17e2926d 100644 --- a/tools/scripts/git_submodule.py +++ b/tools/scripts/git_submodule.py @@ -43,6 +43,7 @@ import glob import os import subprocess import sys +import version_resolver as resolver extra_os = [] @@ -75,25 +76,65 @@ class DEPSParser: subdir = dep if subdir.startswith('src/'): subdir = subdir[4:] - submodule = Submodule(subdir, repo, rev, os) - submodule.deps = True + if subdir.startswith('src'): + # Ignore the information about chromium since we get that from git. + continue + shasum = '' + if len(rev) == 40: # Length of a git shasum + shasum = rev + submodule = Submodule(subdir, repo) + submodule.os = os + submodule.shasum = shasum + if not submodule.shasum: + # We need to parse the svn branch and revision number. + ref = repo + # Try to find out the branch. + ref_path = repo.split('branches/') + if len(ref_path) > 1: + ref = ref_path[1] + if 'trunk' in ref or 'svn' in ref: + # Trunk is used, so we can use the remote master. + ref = 'master' + name_path = subdir.split('/') + if len(name_path) > 1: + # At this point some svn repository paths still include the repo name + # after the actual branch so we have to strip it together with the leading / + name = name_path[-1] + if ref.endswith(name): + branch = ref[:-(len(name) + 1)] + ref = 'refs/branch-heads/' + branch + if name in ref: + # At this point the last partition in the path + # is the branch name, so compose the git ref. + branch = ref.split('/')[-1] + ref = 'refs/branch-heads/' + branch + if 'master' not in ref and 'refs/branch-heads' not in ref: + # Finally compose the refs that did not mach previous rules. + ref = 'refs/branch-heads/' + ref + submodule.ref = ref + submodule.revision = int(rev) submodules.append(submodule) return submodules def sanityCheckModules(self, submodules): - subdirectories = [] + submodule_dict = {} for submodule in submodules: - if submodule.path in subdirectories: - print 'SUBMODULE WARNING: duplicate for submodule' + submodule.path - subdirectories.append(submodule.path) + if not submodule.matchesOS(): + print '-- skipping ' + submodule.path + ' for this operating system. --' + continue + if submodule.path in submodule_dict: + prev_module = submodule_dict[submodule.path] + # We might have to create our own DEPS file if different platforms use different branches, + # but for now it should be safe to select the latest revision from the requirements. + if submodule.shasum or prev_module.revision >= submodule.revision: + continue + if prev_module.ref != submodule.ref: + sys.exit('ERROR: branch mismatch for ' + submodule.path + '(' + prev_module.ref + ' vs ' + submodule.ref + ')') + print('Duplicate submodule ' + submodule.path + '. Using latest revison ' + str(submodule.revision) + '.') + submodule_dict[submodule.path] = submodule + return list(submodule_dict.values()) - def parseFile(self, deps_file_name): - currentDir = os.getcwd() - if not os.path.isfile(deps_file_name): - return [] - deps_file = open(deps_file_name) - deps_content = deps_file.read().decode('utf-8') - deps_file.close() + def parse(self, deps_content): exec(deps_content, self.global_scope, self.local_scope) submodules = [] @@ -101,9 +142,16 @@ class DEPSParser: for os_dep in self.local_scope['deps_os']: submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps_os'][os_dep], os_dep)) - self.sanityCheckModules(submodules) + return self.sanityCheckModules(submodules) - return submodules + def parseFile(self, deps_file_name): + currentDir = os.getcwd() + if not os.path.isfile(deps_file_name): + return [] + deps_file = open(deps_file_name) + deps_content = deps_file.read().decode('utf-8') + deps_file.close() + return self.parse(deps_content) @@ -114,7 +162,7 @@ class Submodule: self.shasum = shasum self.os = os self.ref = '' - self.deps = False + self.revision = None def matchesOS(self): if not self.os: @@ -132,18 +180,45 @@ class Submodule: return True return False - def findSha(self): - if self.shasum != '': - return - line = subprocessCheckOutput(['git', 'submodule', 'status', self.path]) - line = line.lstrip(' -') - self.shasum = line.split(' ')[0] + def findShaAndCheckout(self): + oldCwd = os.getcwd() + os.chdir(self.path) + error = 0 + if self.ref: + # Fetch the ref we parsed from the DEPS file. + val = subprocessCall(['git', 'fetch', 'origin', self.ref]) + if val != 0: + sys.exit("Could not fetch branch from upstream " + self.ref) + error = subprocessCall(['git', 'checkout', 'FETCH_HEAD']); + + search_string = '' + if self.path.endswith('/chromium'): + search_string = resolver.currentVersion() + elif self.revision: + search_string = '@' + str(self.revision) + ' ' + if search_string: + line = subprocessCheckOutput(['git', 'log', '-n1', '--pretty=oneline', '--grep=' + search_string]) + if line: + self.shasum = line.split(' ')[0] + else: # No revision set, use the submodule shasum + os.chdir(oldCwd) + line = subprocessCheckOutput(['git', 'submodule', 'status', self.path]) + os.chdir(self.path) + line = line.lstrip(' -') + self.shasum = line.split(' ')[0] + + current_shasum = subprocessCheckOutput(['git', 'show', '-s', '--oneline']).split(' ')[0] + if not self.shasum.startswith(current_shasum): + # In case HEAD differs check out the actual shasum we require. + error = subprocessCall(['git', 'checkout', self.shasum]) + os.chdir(oldCwd) + return error def findGitDir(self): try: return subprocessCheckOutput(['git', 'rev-parse', '--git-dir']).strip() except subprocess.CalledProcessError, e: - sys.exit("git dir could not be determined! - Initialization failed!" + e.output) + sys.exit("git dir could not be determined! - Initialization failed! " + e.output) def reset(self): currentDir = os.getcwd() @@ -167,26 +242,14 @@ class Submodule: print '-- initializing ' + self.path + ' --' if os.path.isdir(self.path): self.reset() - if self.deps: + + if self.url: subprocessCall(['git', 'submodule', 'add', '-f', self.url, self.path]) subprocessCall(['git', 'submodule', 'init', self.path]) subprocessCall(['git', 'submodule', 'update', self.path]) - self.findSha() - currentDir = os.getcwd() - os.chdir(self.path) - # Make sure we have checked out the right shasum. - # In case there were other patches applied before. - if self.ref: - val = subprocessCall(['git', 'fetch', 'origin', self.ref]); - if val != 0: - sys.exit("Could not fetch branch from upstream.") - subprocessCall(['git', 'checkout', 'FETCH_HEAD']); - else: - val = subprocessCall(['git', 'checkout', self.shasum]) - if val != 0: - sys.exit("!!! initialization failed !!!") - self.initSubmodules() - os.chdir(currentDir) + + if self.findShaAndCheckout() != 0: + sys.exit("!!! initialization failed !!!") else: print '-- skipping ' + self.path + ' for this operating system. --' @@ -203,53 +266,49 @@ class Submodule: def readSubmodules(self): - if not os.path.isfile('.gitmodules'): - return [] - gitmodules_file = open('.gitmodules') - gitmodules_lines = gitmodules_file.readlines() - gitmodules_file.close() - submodules = [] - currentSubmodule = None - for line in gitmodules_lines: - if line.find('[submodule') == 0: - if currentSubmodule: - submodules.append(currentSubmodule) - currentSubmodule = Submodule() - tokens = line.split('=') - if len(tokens) >= 2: - key = tokens[0].strip() - value = tokens[1].strip() - if key == 'path': - currentSubmodule.path = value - elif key == 'url': - currentSubmodule.url = value - elif key == 'os': - currentSubmodule.os = value.split(',') - if currentSubmodule: - submodules.append(currentSubmodule) - return submodules + if self.ref: + submodules = resolver.readSubmodules() + print 'DEPS file provides the following submodules:' + for submodule in submodules: + submodule_ref = submodule.shasum + if submodule.revision: + submodule_ref = submodule.ref + '@' + str(submodule.revision) + print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule_ref + else: # Try .gitmodules since no ref has been specified + if not os.path.isfile('.gitmodules'): + return [] + gitmodules_file = open('.gitmodules') + gitmodules_lines = gitmodules_file.readlines() + gitmodules_file.close() - def readDeps(self): - parser = DEPSParser() - return parser.parseFile('.DEPS.git') + submodules = [] + currentSubmodule = None + for line in gitmodules_lines: + if line.find('[submodule') == 0: + if currentSubmodule: + submodules.append(currentSubmodule) + currentSubmodule = Submodule() + tokens = line.split('=') + if len(tokens) >= 2: + key = tokens[0].strip() + value = tokens[1].strip() + if key == 'path': + currentSubmodule.path = value + elif key == 'url': + currentSubmodule.url = value + elif key == 'os': + currentSubmodule.os = value.split(',') + if currentSubmodule: + submodules.append(currentSubmodule) + return submodules def initSubmodules(self): - # try reading submodules from .gitmodules. + oldCwd = os.getcwd() + os.chdir(self.path) submodules = self.readSubmodules() for submodule in submodules: submodule.initialize() - if submodules: - return - # if we could not find any submodules in .gitmodules, try .DEPS.git - submodules = self.readDeps() - - if not submodules: - return - - print 'DEPS file provides the following submodules:' - for submodule in submodules: - print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule.shasum - for submodule in submodules: - submodule.initialize() - subprocessCall(['git', 'commit', '-a', '-m', '"initialize submodules"']) + if self.ref: + subprocessCall(['git', 'commit', '-a', '-m', 'initialize submodules']) + os.chdir(oldCwd) diff --git a/tools/scripts/version_resolver.py b/tools/scripts/version_resolver.py new file mode 100644 index 000000000..94ba9e1ab --- /dev/null +++ b/tools/scripts/version_resolver.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python + +############################################################################# +# +# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). +# Contact: http://www.qt-project.org/legal +# +# This file is part of the QtWebEngine module of the Qt Toolkit. +# +# $QT_BEGIN_LICENSE:LGPL$ +# Commercial License Usage +# Licensees holding valid commercial Qt licenses may use this file in +# accordance with the commercial license agreement provided with the +# Software or, alternatively, in accordance with the terms contained in +# a written agreement between you and Digia. For licensing terms and +# conditions see http://qt.digia.com/licensing. For further information +# use the contact form at http://qt.digia.com/contact-us. +# +# GNU Lesser General Public License Usage +# Alternatively, this file may be used under the terms of the GNU Lesser +# General Public License version 2.1 as published by the Free Software +# Foundation and appearing in the file LICENSE.LGPL included in the +# packaging of this file. Please review the following information to +# ensure the GNU Lesser General Public License version 2.1 requirements +# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +# +# In addition, as a special exception, Digia gives you certain additional +# rights. These rights are described in the Digia Qt LGPL Exception +# version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +# +# GNU General Public License Usage +# Alternatively, this file may be used under the terms of the GNU +# General Public License version 3.0 as published by the Free Software +# Foundation and appearing in the file LICENSE.GPL included in the +# packaging of this file. Please review the following information to +# ensure the GNU General Public License version 3.0 requirements will be +# met: http://www.gnu.org/copyleft/gpl.html. +# +# +# $QT_END_LICENSE$ +# +############################################################################# + +import os +import sys +import json +import urllib2 +import git_submodule as GitSubmodule + +chromium_version = '31.0.1650.63' +chromium_branch = '1650' + +json_url = 'http://omahaproxy.appspot.com/all.json' +git_deps_url = 'http://src.chromium.org/chrome/branches/' + chromium_branch + '/src/.DEPS.git' +base_deps_url = 'http://src.chromium.org/chrome/releases/' + +def currentVersion(): + return chromium_version + +def currentBranch(): + return chromium_branch + +def readReleaseChannels(): + response = urllib2.urlopen(json_url) + raw_json = response.read().strip() + data = json.loads(raw_json) + channels = {} + + for obj in data: + os = obj['os'] + channels[os] = [] + for ver in obj['versions']: + channels[os].append({ 'channel': ver['channel'], 'version': ver['version'], 'branch': ver['true_branch'] }) + return channels + +def readSubmodules(): + response = urllib2.urlopen(base_deps_url + chromium_version + '/DEPS') + svn_deps = response.read().strip() + + response = urllib2.urlopen(git_deps_url) + git_deps = response.read().strip() + + parser = GitSubmodule.DEPSParser() + svn_submodules = parser.parse(svn_deps) + git_submodules = parser.parse(git_deps) + + submodule_dict = {} + git_dict = {} + + for sub in git_submodules: + git_dict[sub.path] = sub + + for sub in svn_submodules: + if 'reference_build' not in sub.path and (sub.revision or sub.shasum) and sub.path in git_dict: + submodule_dict[sub.path] = sub + + for git in git_submodules: + if git.path in submodule_dict: + # We'll use the git repository instead of svn. + module = submodule_dict[git.path] + module.url = git.url + if not module.shasum: + # We use the git shasum as fallback. + module.shasum = git.shasum + + return list(submodule_dict.values()) |