From 584a8c3c637997fbb6772ab75aea1bd587b82a37 Mon Sep 17 00:00:00 2001 From: Andras Becsi Date: Fri, 6 Dec 2013 14:46:31 +0100 Subject: Improve init-repository.py to fetch up-to-date dependency list This patch introduces a new mechanism to retrieve submodules. The utilities in version_resolver.py provide means to download the upstream DEPS file for a specified Chromium version. This makes it possible to check out the needed svn branches and specific revisions used for the release. We still use the git repositories found in the upstream .DEPS.git file but since it is not regularly updated after the branch-off, we try to match the release svn revisions to git shasums which is possible since each commit includes git-svn-id's. Also add a maintenance tool get_version.py which reads the Chrome release information from http://omahaproxy.appspot.com. We now pin the latest stable channel Chromium release from branch 1650 with the version number 31.0.1650.63 and a branch-off date of Sept 23rd. Change-Id: Ibbf565f876af79dc008bcd161fddcbdd8bc73977 Reviewed-by: Jocelyn Turcotte --- tools/scripts/git_submodule.py | 225 ++++++++++++++++++++++++++--------------- 1 file changed, 142 insertions(+), 83 deletions(-) (limited to 'tools/scripts/git_submodule.py') diff --git a/tools/scripts/git_submodule.py b/tools/scripts/git_submodule.py index 8e088e5bd..c17e2926d 100644 --- a/tools/scripts/git_submodule.py +++ b/tools/scripts/git_submodule.py @@ -43,6 +43,7 @@ import glob import os import subprocess import sys +import version_resolver as resolver extra_os = [] @@ -75,25 +76,65 @@ class DEPSParser: subdir = dep if subdir.startswith('src/'): subdir = subdir[4:] - submodule = Submodule(subdir, repo, rev, os) - submodule.deps = True + if subdir.startswith('src'): + # Ignore the information about chromium since we get that from git. + continue + shasum = '' + if len(rev) == 40: # Length of a git shasum + shasum = rev + submodule = Submodule(subdir, repo) + submodule.os = os + submodule.shasum = shasum + if not submodule.shasum: + # We need to parse the svn branch and revision number. + ref = repo + # Try to find out the branch. + ref_path = repo.split('branches/') + if len(ref_path) > 1: + ref = ref_path[1] + if 'trunk' in ref or 'svn' in ref: + # Trunk is used, so we can use the remote master. + ref = 'master' + name_path = subdir.split('/') + if len(name_path) > 1: + # At this point some svn repository paths still include the repo name + # after the actual branch so we have to strip it together with the leading / + name = name_path[-1] + if ref.endswith(name): + branch = ref[:-(len(name) + 1)] + ref = 'refs/branch-heads/' + branch + if name in ref: + # At this point the last partition in the path + # is the branch name, so compose the git ref. + branch = ref.split('/')[-1] + ref = 'refs/branch-heads/' + branch + if 'master' not in ref and 'refs/branch-heads' not in ref: + # Finally compose the refs that did not mach previous rules. + ref = 'refs/branch-heads/' + ref + submodule.ref = ref + submodule.revision = int(rev) submodules.append(submodule) return submodules def sanityCheckModules(self, submodules): - subdirectories = [] + submodule_dict = {} for submodule in submodules: - if submodule.path in subdirectories: - print 'SUBMODULE WARNING: duplicate for submodule' + submodule.path - subdirectories.append(submodule.path) + if not submodule.matchesOS(): + print '-- skipping ' + submodule.path + ' for this operating system. --' + continue + if submodule.path in submodule_dict: + prev_module = submodule_dict[submodule.path] + # We might have to create our own DEPS file if different platforms use different branches, + # but for now it should be safe to select the latest revision from the requirements. + if submodule.shasum or prev_module.revision >= submodule.revision: + continue + if prev_module.ref != submodule.ref: + sys.exit('ERROR: branch mismatch for ' + submodule.path + '(' + prev_module.ref + ' vs ' + submodule.ref + ')') + print('Duplicate submodule ' + submodule.path + '. Using latest revison ' + str(submodule.revision) + '.') + submodule_dict[submodule.path] = submodule + return list(submodule_dict.values()) - def parseFile(self, deps_file_name): - currentDir = os.getcwd() - if not os.path.isfile(deps_file_name): - return [] - deps_file = open(deps_file_name) - deps_content = deps_file.read().decode('utf-8') - deps_file.close() + def parse(self, deps_content): exec(deps_content, self.global_scope, self.local_scope) submodules = [] @@ -101,9 +142,16 @@ class DEPSParser: for os_dep in self.local_scope['deps_os']: submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps_os'][os_dep], os_dep)) - self.sanityCheckModules(submodules) + return self.sanityCheckModules(submodules) - return submodules + def parseFile(self, deps_file_name): + currentDir = os.getcwd() + if not os.path.isfile(deps_file_name): + return [] + deps_file = open(deps_file_name) + deps_content = deps_file.read().decode('utf-8') + deps_file.close() + return self.parse(deps_content) @@ -114,7 +162,7 @@ class Submodule: self.shasum = shasum self.os = os self.ref = '' - self.deps = False + self.revision = None def matchesOS(self): if not self.os: @@ -132,18 +180,45 @@ class Submodule: return True return False - def findSha(self): - if self.shasum != '': - return - line = subprocessCheckOutput(['git', 'submodule', 'status', self.path]) - line = line.lstrip(' -') - self.shasum = line.split(' ')[0] + def findShaAndCheckout(self): + oldCwd = os.getcwd() + os.chdir(self.path) + error = 0 + if self.ref: + # Fetch the ref we parsed from the DEPS file. + val = subprocessCall(['git', 'fetch', 'origin', self.ref]) + if val != 0: + sys.exit("Could not fetch branch from upstream " + self.ref) + error = subprocessCall(['git', 'checkout', 'FETCH_HEAD']); + + search_string = '' + if self.path.endswith('/chromium'): + search_string = resolver.currentVersion() + elif self.revision: + search_string = '@' + str(self.revision) + ' ' + if search_string: + line = subprocessCheckOutput(['git', 'log', '-n1', '--pretty=oneline', '--grep=' + search_string]) + if line: + self.shasum = line.split(' ')[0] + else: # No revision set, use the submodule shasum + os.chdir(oldCwd) + line = subprocessCheckOutput(['git', 'submodule', 'status', self.path]) + os.chdir(self.path) + line = line.lstrip(' -') + self.shasum = line.split(' ')[0] + + current_shasum = subprocessCheckOutput(['git', 'show', '-s', '--oneline']).split(' ')[0] + if not self.shasum.startswith(current_shasum): + # In case HEAD differs check out the actual shasum we require. + error = subprocessCall(['git', 'checkout', self.shasum]) + os.chdir(oldCwd) + return error def findGitDir(self): try: return subprocessCheckOutput(['git', 'rev-parse', '--git-dir']).strip() except subprocess.CalledProcessError, e: - sys.exit("git dir could not be determined! - Initialization failed!" + e.output) + sys.exit("git dir could not be determined! - Initialization failed! " + e.output) def reset(self): currentDir = os.getcwd() @@ -167,26 +242,14 @@ class Submodule: print '-- initializing ' + self.path + ' --' if os.path.isdir(self.path): self.reset() - if self.deps: + + if self.url: subprocessCall(['git', 'submodule', 'add', '-f', self.url, self.path]) subprocessCall(['git', 'submodule', 'init', self.path]) subprocessCall(['git', 'submodule', 'update', self.path]) - self.findSha() - currentDir = os.getcwd() - os.chdir(self.path) - # Make sure we have checked out the right shasum. - # In case there were other patches applied before. - if self.ref: - val = subprocessCall(['git', 'fetch', 'origin', self.ref]); - if val != 0: - sys.exit("Could not fetch branch from upstream.") - subprocessCall(['git', 'checkout', 'FETCH_HEAD']); - else: - val = subprocessCall(['git', 'checkout', self.shasum]) - if val != 0: - sys.exit("!!! initialization failed !!!") - self.initSubmodules() - os.chdir(currentDir) + + if self.findShaAndCheckout() != 0: + sys.exit("!!! initialization failed !!!") else: print '-- skipping ' + self.path + ' for this operating system. --' @@ -203,53 +266,49 @@ class Submodule: def readSubmodules(self): - if not os.path.isfile('.gitmodules'): - return [] - gitmodules_file = open('.gitmodules') - gitmodules_lines = gitmodules_file.readlines() - gitmodules_file.close() - submodules = [] - currentSubmodule = None - for line in gitmodules_lines: - if line.find('[submodule') == 0: - if currentSubmodule: - submodules.append(currentSubmodule) - currentSubmodule = Submodule() - tokens = line.split('=') - if len(tokens) >= 2: - key = tokens[0].strip() - value = tokens[1].strip() - if key == 'path': - currentSubmodule.path = value - elif key == 'url': - currentSubmodule.url = value - elif key == 'os': - currentSubmodule.os = value.split(',') - if currentSubmodule: - submodules.append(currentSubmodule) - return submodules + if self.ref: + submodules = resolver.readSubmodules() + print 'DEPS file provides the following submodules:' + for submodule in submodules: + submodule_ref = submodule.shasum + if submodule.revision: + submodule_ref = submodule.ref + '@' + str(submodule.revision) + print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule_ref + else: # Try .gitmodules since no ref has been specified + if not os.path.isfile('.gitmodules'): + return [] + gitmodules_file = open('.gitmodules') + gitmodules_lines = gitmodules_file.readlines() + gitmodules_file.close() - def readDeps(self): - parser = DEPSParser() - return parser.parseFile('.DEPS.git') + submodules = [] + currentSubmodule = None + for line in gitmodules_lines: + if line.find('[submodule') == 0: + if currentSubmodule: + submodules.append(currentSubmodule) + currentSubmodule = Submodule() + tokens = line.split('=') + if len(tokens) >= 2: + key = tokens[0].strip() + value = tokens[1].strip() + if key == 'path': + currentSubmodule.path = value + elif key == 'url': + currentSubmodule.url = value + elif key == 'os': + currentSubmodule.os = value.split(',') + if currentSubmodule: + submodules.append(currentSubmodule) + return submodules def initSubmodules(self): - # try reading submodules from .gitmodules. + oldCwd = os.getcwd() + os.chdir(self.path) submodules = self.readSubmodules() for submodule in submodules: submodule.initialize() - if submodules: - return - # if we could not find any submodules in .gitmodules, try .DEPS.git - submodules = self.readDeps() - - if not submodules: - return - - print 'DEPS file provides the following submodules:' - for submodule in submodules: - print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule.shasum - for submodule in submodules: - submodule.initialize() - subprocessCall(['git', 'commit', '-a', '-m', '"initialize submodules"']) + if self.ref: + subprocessCall(['git', 'commit', '-a', '-m', 'initialize submodules']) + os.chdir(oldCwd) -- cgit v1.2.3