summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJocelyn Turcotte <jocelyn.turcotte@digia.com>2013-12-16 10:41:28 +0100
committerThe Qt Project <gerrit-noreply@qt-project.org>2013-12-16 10:49:05 +0100
commit25355b4210284591661e22643321588749277f6d (patch)
treeaa0311f87ed29af03c1cfb3b07b2c5926b2875a1
parentdf8eff9b1e457fe0783cc6022bf7c21f5381afbb (diff)
Revert "Improve init-repository.py to fetch up-to-date dependency list"
Let's make sure that the submodule integration succeeds first. This reverts commit 584a8c3c637997fbb6772ab75aea1bd587b82a37. Change-Id: I972e5540031636489f0995ea864c4dcfc8e5cd7a Reviewed-by: Andras Becsi <andras.becsi@digia.com>
-rwxr-xr-xinit-repository.py9
-rwxr-xr-xtools/scripts/get_version.py55
-rw-r--r--tools/scripts/git_submodule.py225
-rw-r--r--tools/scripts/version_resolver.py106
4 files changed, 89 insertions, 306 deletions
diff --git a/init-repository.py b/init-repository.py
index 3f33b77cd..cd11a4f96 100755
--- a/init-repository.py
+++ b/init-repository.py
@@ -52,7 +52,6 @@ qtwebengine_root = os.path.abspath(os.path.join(os.path.dirname(__file__)))
sys.path.append(os.path.join(qtwebengine_root, 'tools', 'scripts'))
import git_submodule as GitSubmodule
-import version_resolver as resolver
chromium_src = os.environ.get('CHROMIUM_SRC_DIR')
ninja_src = os.path.join(qtwebengine_root, 'src/3rdparty_upstream/ninja')
@@ -119,7 +118,11 @@ def initUpstreamSubmodules():
ninja_url = 'https://github.com/martine/ninja.git'
chromium_url = 'https://chromium.googlesource.com/chromium/src.git'
ninja_shasum = '40b51a0b986b8675e15b0cd1b10c272bf51fdb84'
- chromium_ref = 'refs/branch-heads/' + resolver.currentBranch()
+ chromium_shasum = '29d2d710e0e7961dff032ad4ab73887cc33122bb'
+ # Do not define a branch for now.
+ # We will turn this on, once we actually switch to using the release branch.
+ #chromium_ref = 'refs/branch-heads/1599'
+ chromium_ref = ''
os.chdir(qtwebengine_root)
current_submodules = subprocess.check_output(['git', 'submodule'])
@@ -138,11 +141,11 @@ def initUpstreamSubmodules():
if not use_external_chromium:
chromiumSubmodule = GitSubmodule.Submodule()
chromiumSubmodule.path = 'src/3rdparty_upstream/chromium'
+ chromiumSubmodule.shasum = chromium_shasum
chromiumSubmodule.ref = chromium_ref
chromiumSubmodule.url = chromium_url
chromiumSubmodule.os = 'all'
chromiumSubmodule.initialize()
- chromiumSubmodule.initSubmodules()
def initSnapshot():
snapshot = GitSubmodule.Submodule()
diff --git a/tools/scripts/get_version.py b/tools/scripts/get_version.py
deleted file mode 100755
index c9f4534bc..000000000
--- a/tools/scripts/get_version.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env python
-
-#############################################################################
-#
-# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
-# Contact: http://www.qt-project.org/legal
-#
-# This file is part of the QtWebEngine module of the Qt Toolkit.
-#
-# $QT_BEGIN_LICENSE:LGPL$
-# Commercial License Usage
-# Licensees holding valid commercial Qt licenses may use this file in
-# accordance with the commercial license agreement provided with the
-# Software or, alternatively, in accordance with the terms contained in
-# a written agreement between you and Digia. For licensing terms and
-# conditions see http://qt.digia.com/licensing. For further information
-# use the contact form at http://qt.digia.com/contact-us.
-#
-# GNU Lesser General Public License Usage
-# Alternatively, this file may be used under the terms of the GNU Lesser
-# General Public License version 2.1 as published by the Free Software
-# Foundation and appearing in the file LICENSE.LGPL included in the
-# packaging of this file. Please review the following information to
-# ensure the GNU Lesser General Public License version 2.1 requirements
-# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
-#
-# In addition, as a special exception, Digia gives you certain additional
-# rights. These rights are described in the Digia Qt LGPL Exception
-# version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
-#
-# GNU General Public License Usage
-# Alternatively, this file may be used under the terms of the GNU
-# General Public License version 3.0 as published by the Free Software
-# Foundation and appearing in the file LICENSE.GPL included in the
-# packaging of this file. Please review the following information to
-# ensure the GNU General Public License version 3.0 requirements will be
-# met: http://www.gnu.org/copyleft/gpl.html.
-#
-#
-# $QT_END_LICENSE$
-#
-#############################################################################
-
-import sys
-import json
-import version_resolver as resolver
-
-channels = resolver.readReleaseChannels()
-
-if len(sys.argv) == 1 or sys.argv[1] == 'all':
- print(json.dumps(channels, sort_keys=True, indent=2))
-
-for platform in sys.argv:
- if platform in channels:
- print('"' + platform + '": ' + json.dumps(channels[platform], sort_keys=True, indent=2))
diff --git a/tools/scripts/git_submodule.py b/tools/scripts/git_submodule.py
index 4599379aa..f45f08cae 100644
--- a/tools/scripts/git_submodule.py
+++ b/tools/scripts/git_submodule.py
@@ -43,7 +43,6 @@ import glob
import os
import subprocess
import sys
-import version_resolver as resolver
extra_os = ['android', 'mac']
@@ -76,65 +75,25 @@ class DEPSParser:
subdir = dep
if subdir.startswith('src/'):
subdir = subdir[4:]
- if subdir.startswith('src'):
- # Ignore the information about chromium since we get that from git.
- continue
- shasum = ''
- if len(rev) == 40: # Length of a git shasum
- shasum = rev
- submodule = Submodule(subdir, repo)
- submodule.os = os
- submodule.shasum = shasum
- if not submodule.shasum:
- # We need to parse the svn branch and revision number.
- ref = repo
- # Try to find out the branch.
- ref_path = repo.split('branches/')
- if len(ref_path) > 1:
- ref = ref_path[1]
- if 'trunk' in ref or 'svn' in ref:
- # Trunk is used, so we can use the remote master.
- ref = 'master'
- name_path = subdir.split('/')
- if len(name_path) > 1:
- # At this point some svn repository paths still include the repo name
- # after the actual branch so we have to strip it together with the leading /
- name = name_path[-1]
- if ref.endswith(name):
- branch = ref[:-(len(name) + 1)]
- ref = 'refs/branch-heads/' + branch
- if name in ref:
- # At this point the last partition in the path
- # is the branch name, so compose the git ref.
- branch = ref.split('/')[-1]
- ref = 'refs/branch-heads/' + branch
- if 'master' not in ref and 'refs/branch-heads' not in ref:
- # Finally compose the refs that did not mach previous rules.
- ref = 'refs/branch-heads/' + ref
- submodule.ref = ref
- submodule.revision = int(rev)
+ submodule = Submodule(subdir, repo, rev, os)
+ submodule.deps = True
submodules.append(submodule)
return submodules
def sanityCheckModules(self, submodules):
- submodule_dict = {}
+ subdirectories = []
for submodule in submodules:
- if not submodule.matchesOS():
- print '-- skipping ' + submodule.path + ' for this operating system. --'
- continue
- if submodule.path in submodule_dict:
- prev_module = submodule_dict[submodule.path]
- # We might have to create our own DEPS file if different platforms use different branches,
- # but for now it should be safe to select the latest revision from the requirements.
- if submodule.shasum or prev_module.revision >= submodule.revision:
- continue
- if prev_module.ref != submodule.ref:
- sys.exit('ERROR: branch mismatch for ' + submodule.path + '(' + prev_module.ref + ' vs ' + submodule.ref + ')')
- print('Duplicate submodule ' + submodule.path + '. Using latest revison ' + str(submodule.revision) + '.')
- submodule_dict[submodule.path] = submodule
- return list(submodule_dict.values())
+ if submodule.path in subdirectories:
+ print 'SUBMODULE WARNING: duplicate for submodule' + submodule.path
+ subdirectories.append(submodule.path)
- def parse(self, deps_content):
+ def parseFile(self, deps_file_name):
+ currentDir = os.getcwd()
+ if not os.path.isfile(deps_file_name):
+ return []
+ deps_file = open(deps_file_name)
+ deps_content = deps_file.read().decode('utf-8')
+ deps_file.close()
exec(deps_content, self.global_scope, self.local_scope)
submodules = []
@@ -142,16 +101,9 @@ class DEPSParser:
for os_dep in self.local_scope['deps_os']:
submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps_os'][os_dep], os_dep))
- return self.sanityCheckModules(submodules)
+ self.sanityCheckModules(submodules)
- def parseFile(self, deps_file_name):
- currentDir = os.getcwd()
- if not os.path.isfile(deps_file_name):
- return []
- deps_file = open(deps_file_name)
- deps_content = deps_file.read().decode('utf-8')
- deps_file.close()
- return self.parse(deps_content)
+ return submodules
@@ -162,7 +114,7 @@ class Submodule:
self.shasum = shasum
self.os = os
self.ref = ''
- self.revision = None
+ self.deps = False
def matchesOS(self):
if not self.os:
@@ -180,45 +132,18 @@ class Submodule:
return True
return False
- def findShaAndCheckout(self):
- oldCwd = os.getcwd()
- os.chdir(self.path)
- error = 0
- if self.ref:
- # Fetch the ref we parsed from the DEPS file.
- val = subprocessCall(['git', 'fetch', 'origin', self.ref])
- if val != 0:
- sys.exit("Could not fetch branch from upstream " + self.ref)
- error = subprocessCall(['git', 'checkout', 'FETCH_HEAD']);
-
- search_string = ''
- if self.path.endswith('/chromium'):
- search_string = resolver.currentVersion()
- elif self.revision:
- search_string = '@' + str(self.revision) + ' '
- if search_string:
- line = subprocessCheckOutput(['git', 'log', '-n1', '--pretty=oneline', '--grep=' + search_string])
- if line:
- self.shasum = line.split(' ')[0]
- else: # No revision set, use the submodule shasum
- os.chdir(oldCwd)
- line = subprocessCheckOutput(['git', 'submodule', 'status', self.path])
- os.chdir(self.path)
- line = line.lstrip(' -')
- self.shasum = line.split(' ')[0]
-
- current_shasum = subprocessCheckOutput(['git', 'show', '-s', '--oneline']).split(' ')[0]
- if not self.shasum.startswith(current_shasum):
- # In case HEAD differs check out the actual shasum we require.
- error = subprocessCall(['git', 'checkout', self.shasum])
- os.chdir(oldCwd)
- return error
+ def findSha(self):
+ if self.shasum != '':
+ return
+ line = subprocessCheckOutput(['git', 'submodule', 'status', self.path])
+ line = line.lstrip(' -')
+ self.shasum = line.split(' ')[0]
def findGitDir(self):
try:
return subprocessCheckOutput(['git', 'rev-parse', '--git-dir']).strip()
except subprocess.CalledProcessError, e:
- sys.exit("git dir could not be determined! - Initialization failed! " + e.output)
+ sys.exit("git dir could not be determined! - Initialization failed!" + e.output)
def reset(self):
currentDir = os.getcwd()
@@ -242,14 +167,26 @@ class Submodule:
print '-- initializing ' + self.path + ' --'
if os.path.isdir(self.path):
self.reset()
-
- if self.url:
+ if self.deps:
subprocessCall(['git', 'submodule', 'add', '-f', self.url, self.path])
subprocessCall(['git', 'submodule', 'init', self.path])
subprocessCall(['git', 'submodule', 'update', self.path])
-
- if self.findShaAndCheckout() != 0:
- sys.exit("!!! initialization failed !!!")
+ self.findSha()
+ currentDir = os.getcwd()
+ os.chdir(self.path)
+ # Make sure we have checked out the right shasum.
+ # In case there were other patches applied before.
+ if self.ref:
+ val = subprocessCall(['git', 'fetch', 'origin', self.ref]);
+ if val != 0:
+ sys.exit("Could not fetch branch from upstream.")
+ subprocessCall(['git', 'checkout', 'FETCH_HEAD']);
+ else:
+ val = subprocessCall(['git', 'checkout', self.shasum])
+ if val != 0:
+ sys.exit("!!! initialization failed !!!")
+ self.initSubmodules()
+ os.chdir(currentDir)
else:
print '-- skipping ' + self.path + ' for this operating system. --'
@@ -266,49 +203,53 @@ class Submodule:
def readSubmodules(self):
- submodules = []
- if self.ref:
- submodules = resolver.readSubmodules()
- print 'DEPS file provides the following submodules:'
- for submodule in submodules:
- submodule_ref = submodule.shasum
- if submodule.revision:
- submodule_ref = submodule.ref + '@' + str(submodule.revision)
- print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule_ref
- else: # Try .gitmodules since no ref has been specified
- if not os.path.isfile('.gitmodules'):
- return []
- gitmodules_file = open('.gitmodules')
- gitmodules_lines = gitmodules_file.readlines()
- gitmodules_file.close()
+ if not os.path.isfile('.gitmodules'):
+ return []
+ gitmodules_file = open('.gitmodules')
+ gitmodules_lines = gitmodules_file.readlines()
+ gitmodules_file.close()
- submodules = []
- currentSubmodule = None
- for line in gitmodules_lines:
- if line.find('[submodule') == 0:
- if currentSubmodule:
- submodules.append(currentSubmodule)
- currentSubmodule = Submodule()
- tokens = line.split('=')
- if len(tokens) >= 2:
- key = tokens[0].strip()
- value = tokens[1].strip()
- if key == 'path':
- currentSubmodule.path = value
- elif key == 'url':
- currentSubmodule.url = value
- elif key == 'os':
- currentSubmodule.os = value.split(',')
- if currentSubmodule:
- submodules.append(currentSubmodule)
+ submodules = []
+ currentSubmodule = None
+ for line in gitmodules_lines:
+ if line.find('[submodule') == 0:
+ if currentSubmodule:
+ submodules.append(currentSubmodule)
+ currentSubmodule = Submodule()
+ tokens = line.split('=')
+ if len(tokens) >= 2:
+ key = tokens[0].strip()
+ value = tokens[1].strip()
+ if key == 'path':
+ currentSubmodule.path = value
+ elif key == 'url':
+ currentSubmodule.url = value
+ elif key == 'os':
+ currentSubmodule.os = value.split(',')
+ if currentSubmodule:
+ submodules.append(currentSubmodule)
return submodules
+ def readDeps(self):
+ parser = DEPSParser()
+ return parser.parseFile('.DEPS.git')
+
def initSubmodules(self):
- oldCwd = os.getcwd()
- os.chdir(self.path)
+ # try reading submodules from .gitmodules.
submodules = self.readSubmodules()
for submodule in submodules:
submodule.initialize()
- if self.ref:
- subprocessCall(['git', 'commit', '-a', '-m', 'initialize submodules'])
- os.chdir(oldCwd)
+ if submodules:
+ return
+ # if we could not find any submodules in .gitmodules, try .DEPS.git
+ submodules = self.readDeps()
+
+ if not submodules:
+ return
+
+ print 'DEPS file provides the following submodules:'
+ for submodule in submodules:
+ print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule.shasum
+ for submodule in submodules:
+ submodule.initialize()
+ subprocessCall(['git', 'commit', '-a', '-m', '"initialize submodules"'])
diff --git a/tools/scripts/version_resolver.py b/tools/scripts/version_resolver.py
deleted file mode 100644
index 94ba9e1ab..000000000
--- a/tools/scripts/version_resolver.py
+++ /dev/null
@@ -1,106 +0,0 @@
-#!/usr/bin/env python
-
-#############################################################################
-#
-# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
-# Contact: http://www.qt-project.org/legal
-#
-# This file is part of the QtWebEngine module of the Qt Toolkit.
-#
-# $QT_BEGIN_LICENSE:LGPL$
-# Commercial License Usage
-# Licensees holding valid commercial Qt licenses may use this file in
-# accordance with the commercial license agreement provided with the
-# Software or, alternatively, in accordance with the terms contained in
-# a written agreement between you and Digia. For licensing terms and
-# conditions see http://qt.digia.com/licensing. For further information
-# use the contact form at http://qt.digia.com/contact-us.
-#
-# GNU Lesser General Public License Usage
-# Alternatively, this file may be used under the terms of the GNU Lesser
-# General Public License version 2.1 as published by the Free Software
-# Foundation and appearing in the file LICENSE.LGPL included in the
-# packaging of this file. Please review the following information to
-# ensure the GNU Lesser General Public License version 2.1 requirements
-# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
-#
-# In addition, as a special exception, Digia gives you certain additional
-# rights. These rights are described in the Digia Qt LGPL Exception
-# version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
-#
-# GNU General Public License Usage
-# Alternatively, this file may be used under the terms of the GNU
-# General Public License version 3.0 as published by the Free Software
-# Foundation and appearing in the file LICENSE.GPL included in the
-# packaging of this file. Please review the following information to
-# ensure the GNU General Public License version 3.0 requirements will be
-# met: http://www.gnu.org/copyleft/gpl.html.
-#
-#
-# $QT_END_LICENSE$
-#
-#############################################################################
-
-import os
-import sys
-import json
-import urllib2
-import git_submodule as GitSubmodule
-
-chromium_version = '31.0.1650.63'
-chromium_branch = '1650'
-
-json_url = 'http://omahaproxy.appspot.com/all.json'
-git_deps_url = 'http://src.chromium.org/chrome/branches/' + chromium_branch + '/src/.DEPS.git'
-base_deps_url = 'http://src.chromium.org/chrome/releases/'
-
-def currentVersion():
- return chromium_version
-
-def currentBranch():
- return chromium_branch
-
-def readReleaseChannels():
- response = urllib2.urlopen(json_url)
- raw_json = response.read().strip()
- data = json.loads(raw_json)
- channels = {}
-
- for obj in data:
- os = obj['os']
- channels[os] = []
- for ver in obj['versions']:
- channels[os].append({ 'channel': ver['channel'], 'version': ver['version'], 'branch': ver['true_branch'] })
- return channels
-
-def readSubmodules():
- response = urllib2.urlopen(base_deps_url + chromium_version + '/DEPS')
- svn_deps = response.read().strip()
-
- response = urllib2.urlopen(git_deps_url)
- git_deps = response.read().strip()
-
- parser = GitSubmodule.DEPSParser()
- svn_submodules = parser.parse(svn_deps)
- git_submodules = parser.parse(git_deps)
-
- submodule_dict = {}
- git_dict = {}
-
- for sub in git_submodules:
- git_dict[sub.path] = sub
-
- for sub in svn_submodules:
- if 'reference_build' not in sub.path and (sub.revision or sub.shasum) and sub.path in git_dict:
- submodule_dict[sub.path] = sub
-
- for git in git_submodules:
- if git.path in submodule_dict:
- # We'll use the git repository instead of svn.
- module = submodule_dict[git.path]
- module.url = git.url
- if not module.shasum:
- # We use the git shasum as fallback.
- module.shasum = git.shasum
-
- return list(submodule_dict.values())