From 25355b4210284591661e22643321588749277f6d Mon Sep 17 00:00:00 2001 From: Jocelyn Turcotte Date: Mon, 16 Dec 2013 10:41:28 +0100 Subject: Revert "Improve init-repository.py to fetch up-to-date dependency list" Let's make sure that the submodule integration succeeds first. This reverts commit 584a8c3c637997fbb6772ab75aea1bd587b82a37. Change-Id: I972e5540031636489f0995ea864c4dcfc8e5cd7a Reviewed-by: Andras Becsi --- tools/scripts/get_version.py | 55 ---------- tools/scripts/git_submodule.py | 225 ++++++++++++++------------------------ tools/scripts/version_resolver.py | 106 ------------------ 3 files changed, 83 insertions(+), 303 deletions(-) delete mode 100755 tools/scripts/get_version.py delete mode 100644 tools/scripts/version_resolver.py (limited to 'tools/scripts') diff --git a/tools/scripts/get_version.py b/tools/scripts/get_version.py deleted file mode 100755 index c9f4534bc..000000000 --- a/tools/scripts/get_version.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python - -############################################################################# -# -# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). -# Contact: http://www.qt-project.org/legal -# -# This file is part of the QtWebEngine module of the Qt Toolkit. -# -# $QT_BEGIN_LICENSE:LGPL$ -# Commercial License Usage -# Licensees holding valid commercial Qt licenses may use this file in -# accordance with the commercial license agreement provided with the -# Software or, alternatively, in accordance with the terms contained in -# a written agreement between you and Digia. For licensing terms and -# conditions see http://qt.digia.com/licensing. For further information -# use the contact form at http://qt.digia.com/contact-us. -# -# GNU Lesser General Public License Usage -# Alternatively, this file may be used under the terms of the GNU Lesser -# General Public License version 2.1 as published by the Free Software -# Foundation and appearing in the file LICENSE.LGPL included in the -# packaging of this file. Please review the following information to -# ensure the GNU Lesser General Public License version 2.1 requirements -# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -# -# In addition, as a special exception, Digia gives you certain additional -# rights. These rights are described in the Digia Qt LGPL Exception -# version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -# -# GNU General Public License Usage -# Alternatively, this file may be used under the terms of the GNU -# General Public License version 3.0 as published by the Free Software -# Foundation and appearing in the file LICENSE.GPL included in the -# packaging of this file. Please review the following information to -# ensure the GNU General Public License version 3.0 requirements will be -# met: http://www.gnu.org/copyleft/gpl.html. -# -# -# $QT_END_LICENSE$ -# -############################################################################# - -import sys -import json -import version_resolver as resolver - -channels = resolver.readReleaseChannels() - -if len(sys.argv) == 1 or sys.argv[1] == 'all': - print(json.dumps(channels, sort_keys=True, indent=2)) - -for platform in sys.argv: - if platform in channels: - print('"' + platform + '": ' + json.dumps(channels[platform], sort_keys=True, indent=2)) diff --git a/tools/scripts/git_submodule.py b/tools/scripts/git_submodule.py index 4599379aa..f45f08cae 100644 --- a/tools/scripts/git_submodule.py +++ b/tools/scripts/git_submodule.py @@ -43,7 +43,6 @@ import glob import os import subprocess import sys -import version_resolver as resolver extra_os = ['android', 'mac'] @@ -76,65 +75,25 @@ class DEPSParser: subdir = dep if subdir.startswith('src/'): subdir = subdir[4:] - if subdir.startswith('src'): - # Ignore the information about chromium since we get that from git. - continue - shasum = '' - if len(rev) == 40: # Length of a git shasum - shasum = rev - submodule = Submodule(subdir, repo) - submodule.os = os - submodule.shasum = shasum - if not submodule.shasum: - # We need to parse the svn branch and revision number. - ref = repo - # Try to find out the branch. - ref_path = repo.split('branches/') - if len(ref_path) > 1: - ref = ref_path[1] - if 'trunk' in ref or 'svn' in ref: - # Trunk is used, so we can use the remote master. - ref = 'master' - name_path = subdir.split('/') - if len(name_path) > 1: - # At this point some svn repository paths still include the repo name - # after the actual branch so we have to strip it together with the leading / - name = name_path[-1] - if ref.endswith(name): - branch = ref[:-(len(name) + 1)] - ref = 'refs/branch-heads/' + branch - if name in ref: - # At this point the last partition in the path - # is the branch name, so compose the git ref. - branch = ref.split('/')[-1] - ref = 'refs/branch-heads/' + branch - if 'master' not in ref and 'refs/branch-heads' not in ref: - # Finally compose the refs that did not mach previous rules. - ref = 'refs/branch-heads/' + ref - submodule.ref = ref - submodule.revision = int(rev) + submodule = Submodule(subdir, repo, rev, os) + submodule.deps = True submodules.append(submodule) return submodules def sanityCheckModules(self, submodules): - submodule_dict = {} + subdirectories = [] for submodule in submodules: - if not submodule.matchesOS(): - print '-- skipping ' + submodule.path + ' for this operating system. --' - continue - if submodule.path in submodule_dict: - prev_module = submodule_dict[submodule.path] - # We might have to create our own DEPS file if different platforms use different branches, - # but for now it should be safe to select the latest revision from the requirements. - if submodule.shasum or prev_module.revision >= submodule.revision: - continue - if prev_module.ref != submodule.ref: - sys.exit('ERROR: branch mismatch for ' + submodule.path + '(' + prev_module.ref + ' vs ' + submodule.ref + ')') - print('Duplicate submodule ' + submodule.path + '. Using latest revison ' + str(submodule.revision) + '.') - submodule_dict[submodule.path] = submodule - return list(submodule_dict.values()) + if submodule.path in subdirectories: + print 'SUBMODULE WARNING: duplicate for submodule' + submodule.path + subdirectories.append(submodule.path) - def parse(self, deps_content): + def parseFile(self, deps_file_name): + currentDir = os.getcwd() + if not os.path.isfile(deps_file_name): + return [] + deps_file = open(deps_file_name) + deps_content = deps_file.read().decode('utf-8') + deps_file.close() exec(deps_content, self.global_scope, self.local_scope) submodules = [] @@ -142,16 +101,9 @@ class DEPSParser: for os_dep in self.local_scope['deps_os']: submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps_os'][os_dep], os_dep)) - return self.sanityCheckModules(submodules) + self.sanityCheckModules(submodules) - def parseFile(self, deps_file_name): - currentDir = os.getcwd() - if not os.path.isfile(deps_file_name): - return [] - deps_file = open(deps_file_name) - deps_content = deps_file.read().decode('utf-8') - deps_file.close() - return self.parse(deps_content) + return submodules @@ -162,7 +114,7 @@ class Submodule: self.shasum = shasum self.os = os self.ref = '' - self.revision = None + self.deps = False def matchesOS(self): if not self.os: @@ -180,45 +132,18 @@ class Submodule: return True return False - def findShaAndCheckout(self): - oldCwd = os.getcwd() - os.chdir(self.path) - error = 0 - if self.ref: - # Fetch the ref we parsed from the DEPS file. - val = subprocessCall(['git', 'fetch', 'origin', self.ref]) - if val != 0: - sys.exit("Could not fetch branch from upstream " + self.ref) - error = subprocessCall(['git', 'checkout', 'FETCH_HEAD']); - - search_string = '' - if self.path.endswith('/chromium'): - search_string = resolver.currentVersion() - elif self.revision: - search_string = '@' + str(self.revision) + ' ' - if search_string: - line = subprocessCheckOutput(['git', 'log', '-n1', '--pretty=oneline', '--grep=' + search_string]) - if line: - self.shasum = line.split(' ')[0] - else: # No revision set, use the submodule shasum - os.chdir(oldCwd) - line = subprocessCheckOutput(['git', 'submodule', 'status', self.path]) - os.chdir(self.path) - line = line.lstrip(' -') - self.shasum = line.split(' ')[0] - - current_shasum = subprocessCheckOutput(['git', 'show', '-s', '--oneline']).split(' ')[0] - if not self.shasum.startswith(current_shasum): - # In case HEAD differs check out the actual shasum we require. - error = subprocessCall(['git', 'checkout', self.shasum]) - os.chdir(oldCwd) - return error + def findSha(self): + if self.shasum != '': + return + line = subprocessCheckOutput(['git', 'submodule', 'status', self.path]) + line = line.lstrip(' -') + self.shasum = line.split(' ')[0] def findGitDir(self): try: return subprocessCheckOutput(['git', 'rev-parse', '--git-dir']).strip() except subprocess.CalledProcessError, e: - sys.exit("git dir could not be determined! - Initialization failed! " + e.output) + sys.exit("git dir could not be determined! - Initialization failed!" + e.output) def reset(self): currentDir = os.getcwd() @@ -242,14 +167,26 @@ class Submodule: print '-- initializing ' + self.path + ' --' if os.path.isdir(self.path): self.reset() - - if self.url: + if self.deps: subprocessCall(['git', 'submodule', 'add', '-f', self.url, self.path]) subprocessCall(['git', 'submodule', 'init', self.path]) subprocessCall(['git', 'submodule', 'update', self.path]) - - if self.findShaAndCheckout() != 0: - sys.exit("!!! initialization failed !!!") + self.findSha() + currentDir = os.getcwd() + os.chdir(self.path) + # Make sure we have checked out the right shasum. + # In case there were other patches applied before. + if self.ref: + val = subprocessCall(['git', 'fetch', 'origin', self.ref]); + if val != 0: + sys.exit("Could not fetch branch from upstream.") + subprocessCall(['git', 'checkout', 'FETCH_HEAD']); + else: + val = subprocessCall(['git', 'checkout', self.shasum]) + if val != 0: + sys.exit("!!! initialization failed !!!") + self.initSubmodules() + os.chdir(currentDir) else: print '-- skipping ' + self.path + ' for this operating system. --' @@ -266,49 +203,53 @@ class Submodule: def readSubmodules(self): - submodules = [] - if self.ref: - submodules = resolver.readSubmodules() - print 'DEPS file provides the following submodules:' - for submodule in submodules: - submodule_ref = submodule.shasum - if submodule.revision: - submodule_ref = submodule.ref + '@' + str(submodule.revision) - print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule_ref - else: # Try .gitmodules since no ref has been specified - if not os.path.isfile('.gitmodules'): - return [] - gitmodules_file = open('.gitmodules') - gitmodules_lines = gitmodules_file.readlines() - gitmodules_file.close() + if not os.path.isfile('.gitmodules'): + return [] + gitmodules_file = open('.gitmodules') + gitmodules_lines = gitmodules_file.readlines() + gitmodules_file.close() - submodules = [] - currentSubmodule = None - for line in gitmodules_lines: - if line.find('[submodule') == 0: - if currentSubmodule: - submodules.append(currentSubmodule) - currentSubmodule = Submodule() - tokens = line.split('=') - if len(tokens) >= 2: - key = tokens[0].strip() - value = tokens[1].strip() - if key == 'path': - currentSubmodule.path = value - elif key == 'url': - currentSubmodule.url = value - elif key == 'os': - currentSubmodule.os = value.split(',') - if currentSubmodule: - submodules.append(currentSubmodule) + submodules = [] + currentSubmodule = None + for line in gitmodules_lines: + if line.find('[submodule') == 0: + if currentSubmodule: + submodules.append(currentSubmodule) + currentSubmodule = Submodule() + tokens = line.split('=') + if len(tokens) >= 2: + key = tokens[0].strip() + value = tokens[1].strip() + if key == 'path': + currentSubmodule.path = value + elif key == 'url': + currentSubmodule.url = value + elif key == 'os': + currentSubmodule.os = value.split(',') + if currentSubmodule: + submodules.append(currentSubmodule) return submodules + def readDeps(self): + parser = DEPSParser() + return parser.parseFile('.DEPS.git') + def initSubmodules(self): - oldCwd = os.getcwd() - os.chdir(self.path) + # try reading submodules from .gitmodules. submodules = self.readSubmodules() for submodule in submodules: submodule.initialize() - if self.ref: - subprocessCall(['git', 'commit', '-a', '-m', 'initialize submodules']) - os.chdir(oldCwd) + if submodules: + return + # if we could not find any submodules in .gitmodules, try .DEPS.git + submodules = self.readDeps() + + if not submodules: + return + + print 'DEPS file provides the following submodules:' + for submodule in submodules: + print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule.shasum + for submodule in submodules: + submodule.initialize() + subprocessCall(['git', 'commit', '-a', '-m', '"initialize submodules"']) diff --git a/tools/scripts/version_resolver.py b/tools/scripts/version_resolver.py deleted file mode 100644 index 94ba9e1ab..000000000 --- a/tools/scripts/version_resolver.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python - -############################################################################# -# -# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). -# Contact: http://www.qt-project.org/legal -# -# This file is part of the QtWebEngine module of the Qt Toolkit. -# -# $QT_BEGIN_LICENSE:LGPL$ -# Commercial License Usage -# Licensees holding valid commercial Qt licenses may use this file in -# accordance with the commercial license agreement provided with the -# Software or, alternatively, in accordance with the terms contained in -# a written agreement between you and Digia. For licensing terms and -# conditions see http://qt.digia.com/licensing. For further information -# use the contact form at http://qt.digia.com/contact-us. -# -# GNU Lesser General Public License Usage -# Alternatively, this file may be used under the terms of the GNU Lesser -# General Public License version 2.1 as published by the Free Software -# Foundation and appearing in the file LICENSE.LGPL included in the -# packaging of this file. Please review the following information to -# ensure the GNU Lesser General Public License version 2.1 requirements -# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -# -# In addition, as a special exception, Digia gives you certain additional -# rights. These rights are described in the Digia Qt LGPL Exception -# version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -# -# GNU General Public License Usage -# Alternatively, this file may be used under the terms of the GNU -# General Public License version 3.0 as published by the Free Software -# Foundation and appearing in the file LICENSE.GPL included in the -# packaging of this file. Please review the following information to -# ensure the GNU General Public License version 3.0 requirements will be -# met: http://www.gnu.org/copyleft/gpl.html. -# -# -# $QT_END_LICENSE$ -# -############################################################################# - -import os -import sys -import json -import urllib2 -import git_submodule as GitSubmodule - -chromium_version = '31.0.1650.63' -chromium_branch = '1650' - -json_url = 'http://omahaproxy.appspot.com/all.json' -git_deps_url = 'http://src.chromium.org/chrome/branches/' + chromium_branch + '/src/.DEPS.git' -base_deps_url = 'http://src.chromium.org/chrome/releases/' - -def currentVersion(): - return chromium_version - -def currentBranch(): - return chromium_branch - -def readReleaseChannels(): - response = urllib2.urlopen(json_url) - raw_json = response.read().strip() - data = json.loads(raw_json) - channels = {} - - for obj in data: - os = obj['os'] - channels[os] = [] - for ver in obj['versions']: - channels[os].append({ 'channel': ver['channel'], 'version': ver['version'], 'branch': ver['true_branch'] }) - return channels - -def readSubmodules(): - response = urllib2.urlopen(base_deps_url + chromium_version + '/DEPS') - svn_deps = response.read().strip() - - response = urllib2.urlopen(git_deps_url) - git_deps = response.read().strip() - - parser = GitSubmodule.DEPSParser() - svn_submodules = parser.parse(svn_deps) - git_submodules = parser.parse(git_deps) - - submodule_dict = {} - git_dict = {} - - for sub in git_submodules: - git_dict[sub.path] = sub - - for sub in svn_submodules: - if 'reference_build' not in sub.path and (sub.revision or sub.shasum) and sub.path in git_dict: - submodule_dict[sub.path] = sub - - for git in git_submodules: - if git.path in submodule_dict: - # We'll use the git repository instead of svn. - module = submodule_dict[git.path] - module.url = git.url - if not module.shasum: - # We use the git shasum as fallback. - module.shasum = git.shasum - - return list(submodule_dict.values()) -- cgit v1.2.3