From fd61d752e313bf91a09c85020b3fb50067c610c8 Mon Sep 17 00:00:00 2001 From: Jocelyn Turcotte Date: Tue, 19 Nov 2013 17:55:49 +0100 Subject: Moving sources to src part 1: Move files. This only move files without adjusting any paths. This moves: - lib/quick -> src/webengine/api (API files) lib/quick -> src/webengine (other files) This contains the main QtWebEngine module library since . - lib/widgets -> src/webenginewidgets Also rename this directory to match its module name and rename Api to api. - lib -> src/core - process -> src/process - resources -> src/core/resources - tools/* -> tools/scripts/ The build directory is spread as follow: - build/build.pro -> src/core/gyp_run.pro - build/qmake_extras/* -> src/core/ (for the host and target .pro files) - build/qmake -> tools/qmake - Build related scripts -> tools/buildscripts Change-Id: I0cded1de772c99c0c1da6536c9afea353236b4a1 Reviewed-by: Zeno Albisser Reviewed-by: Pierre Rossi Reviewed-by: Andras Becsi --- tools/scripts/git_submodule.py | 255 +++++++++++++++++++++++++++++++++++++++++ tools/scripts/take_snapshot.py | 189 ++++++++++++++++++++++++++++++ 2 files changed, 444 insertions(+) create mode 100644 tools/scripts/git_submodule.py create mode 100755 tools/scripts/take_snapshot.py (limited to 'tools/scripts') diff --git a/tools/scripts/git_submodule.py b/tools/scripts/git_submodule.py new file mode 100644 index 000000000..084bd2b1f --- /dev/null +++ b/tools/scripts/git_submodule.py @@ -0,0 +1,255 @@ +############################################################################# +# +# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). +# Contact: http://www.qt-project.org/legal +# +# This file is part of the QtWebEngine module of the Qt Toolkit. +# +# $QT_BEGIN_LICENSE:LGPL$ +# Commercial License Usage +# Licensees holding valid commercial Qt licenses may use this file in +# accordance with the commercial license agreement provided with the +# Software or, alternatively, in accordance with the terms contained in +# a written agreement between you and Digia. For licensing terms and +# conditions see http://qt.digia.com/licensing. For further information +# use the contact form at http://qt.digia.com/contact-us. +# +# GNU Lesser General Public License Usage +# Alternatively, this file may be used under the terms of the GNU Lesser +# General Public License version 2.1 as published by the Free Software +# Foundation and appearing in the file LICENSE.LGPL included in the +# packaging of this file. Please review the following information to +# ensure the GNU Lesser General Public License version 2.1 requirements +# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +# +# In addition, as a special exception, Digia gives you certain additional +# rights. These rights are described in the Digia Qt LGPL Exception +# version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +# +# GNU General Public License Usage +# Alternatively, this file may be used under the terms of the GNU +# General Public License version 3.0 as published by the Free Software +# Foundation and appearing in the file LICENSE.GPL included in the +# packaging of this file. Please review the following information to +# ensure the GNU General Public License version 3.0 requirements will be +# met: http://www.gnu.org/copyleft/gpl.html. +# +# +# $QT_END_LICENSE$ +# +############################################################################# + +import glob +import os +import subprocess +import sys + +extra_os = [] + +def subprocessCall(args): + print args + return subprocess.call(args) + +def subprocessCheckOutput(args): + print args + return subprocess.check_output(args) + +class DEPSParser: + def __init__(self): + self.global_scope = { + 'Var': self.Lookup, + 'deps_os': {}, + } + self.local_scope = {} + + def Lookup(self, var_name): + return self.local_scope["vars"][var_name] + + def createSubmodulesFromScope(self, scope, os): + submodules = [] + for dep in scope: + if (type(scope[dep]) == str): + repo_rev = scope[dep].split('@') + repo = repo_rev[0] + rev = repo_rev[1] + subdir = dep + if subdir.startswith('src/'): + subdir = subdir[4:] + submodule = Submodule(subdir, repo, rev, os) + submodule.deps = True + submodules.append(submodule) + return submodules + + def sanityCheckModules(self, submodules): + subdirectories = [] + for submodule in submodules: + if submodule.path in subdirectories: + print 'SUBMODULE WARNING: duplicate for submodule' + submodule.path + subdirectories.append(submodule.path) + + def parseFile(self, deps_file_name): + currentDir = os.getcwd() + if not os.path.isfile(deps_file_name): + return [] + deps_file = open(deps_file_name) + deps_content = deps_file.read().decode('utf-8') + deps_file.close() + exec(deps_content, self.global_scope, self.local_scope) + + submodules = [] + submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps'], 'all')) + for os_dep in self.local_scope['deps_os']: + submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps_os'][os_dep], os_dep)) + + self.sanityCheckModules(submodules) + + return submodules + + + +class Submodule: + def __init__(self, path='', url='', shasum='', os=[], ref=''): + self.path = path + self.url = url + self.shasum = shasum + self.os = os + self.ref = '' + self.deps = False + + def matchesOS(self): + if not self.os: + return True + if 'all' in self.os: + return True + if sys.platform.startswith('win32') and 'win' in self.os: + return True + if sys.platform.startswith('linux') and 'unix' in self.os: + return True + if sys.platform.startswith('darwin') and ('unix' in self.os or 'mac' in self.os): + return True + for os in extra_os: + if os in self.os: + return True + return False + + def findSha(self): + if self.shasum != '': + return + line = subprocessCheckOutput(['git', 'submodule', 'status', self.path]) + line = line.lstrip(' -') + self.shasum = line.split(' ')[0] + + def findGitDir(self): + try: + return subprocessCheckOutput(['git', 'rev-parse', '--git-dir']).strip() + except subprocess.CalledProcessError, e: + sys.exit("git dir could not be determined! - Initialization failed!" + e.output) + + def reset(self): + currentDir = os.getcwd() + os.chdir(self.path) + gitdir = self.findGitDir() + if os.path.isdir(os.path.join(gitdir, 'rebase-merge')): + if os.path.isfile(os.path.join(gitdir, 'MERGE_HEAD')): + print 'merge in progress... aborting merge.' + subprocessCall(['git', 'merge', '--abort']) + else: + print 'rebase in progress... aborting merge.' + subprocessCall(['git', 'rebase', '--abort']) + if os.path.isdir(os.path.join(gitdir, 'rebase-apply')): + print 'am in progress... aborting am.' + subprocessCall(['git', 'am', '--abort']) + subprocessCall(['git', 'reset', '--hard']) + os.chdir(currentDir) + + def initialize(self): + if self.matchesOS(): + print '-- initializing ' + self.path + ' --' + if os.path.isdir(self.path): + self.reset() + if self.deps: + subprocessCall(['git', 'submodule', 'add', '-f', self.url, self.path]) + subprocessCall(['git', 'submodule', 'init', self.path]) + subprocessCall(['git', 'submodule', 'update', self.path]) + self.findSha() + currentDir = os.getcwd() + os.chdir(self.path) + # Make sure we have checked out the right shasum. + # In case there were other patches applied before. + if self.ref: + val = subprocessCall(['git', 'fetch', 'origin', self.ref]); + if val != 0: + sys.exit("Could not fetch branch from upstream.") + subprocessCall(['git', 'checkout', 'FETCH_HEAD']); + else: + val = subprocessCall(['git', 'checkout', self.shasum]) + if val != 0: + sys.exit("!!! initialization failed !!!") + self.initSubmodules() + os.chdir(currentDir) + else: + print '-- skipping ' + self.path + ' for this operating system. --' + + def listFiles(self): + if self.matchesOS(): + currentDir = os.getcwd() + os.chdir(self.path) + files = subprocessCheckOutput(['git', 'ls-files']).splitlines() + os.chdir(currentDir) + return files + else: + print '-- skipping ' + self.path + ' for this operating system. --' + return [] + + + def readSubmodules(self): + if not os.path.isfile('.gitmodules'): + return [] + gitmodules_file = open('.gitmodules') + gitmodules_lines = gitmodules_file.readlines() + gitmodules_file.close() + + submodules = [] + currentSubmodule = None + for line in gitmodules_lines: + if line.find('[submodule') == 0: + if currentSubmodule: + submodules.append(currentSubmodule) + currentSubmodule = Submodule() + tokens = line.split('=') + if len(tokens) >= 2: + key = tokens[0].strip() + value = tokens[1].strip() + if key == 'path': + currentSubmodule.path = value + elif key == 'url': + currentSubmodule.url = value + elif key == 'os': + currentSubmodule.os = value.split(',') + if currentSubmodule: + submodules.append(currentSubmodule) + return submodules + + def readDeps(self): + parser = DEPSParser() + return parser.parseFile('.DEPS.git') + + def initSubmodules(self): + # try reading submodules from .gitmodules. + submodules = self.readSubmodules() + for submodule in submodules: + submodule.initialize() + if submodules: + return + # if we could not find any submodules in .gitmodules, try .DEPS.git + submodules = self.readDeps() + + if not submodules: + return + + print 'DEPS file provides the following submodules:' + for submodule in submodules: + print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule.shasum + for submodule in submodules: + submodule.initialize() + subprocessCall(['git', 'commit', '-a', '-m', '"initialize submodules"']) diff --git a/tools/scripts/take_snapshot.py b/tools/scripts/take_snapshot.py new file mode 100755 index 000000000..1a6976902 --- /dev/null +++ b/tools/scripts/take_snapshot.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python + +############################################################################# +# +# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). +# Contact: http://www.qt-project.org/legal +# +# This file is part of the QtWebEngine module of the Qt Toolkit. +# +# $QT_BEGIN_LICENSE:LGPL$ +# Commercial License Usage +# Licensees holding valid commercial Qt licenses may use this file in +# accordance with the commercial license agreement provided with the +# Software or, alternatively, in accordance with the terms contained in +# a written agreement between you and Digia. For licensing terms and +# conditions see http://qt.digia.com/licensing. For further information +# use the contact form at http://qt.digia.com/contact-us. +# +# GNU Lesser General Public License Usage +# Alternatively, this file may be used under the terms of the GNU Lesser +# General Public License version 2.1 as published by the Free Software +# Foundation and appearing in the file LICENSE.LGPL included in the +# packaging of this file. Please review the following information to +# ensure the GNU Lesser General Public License version 2.1 requirements +# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +# +# In addition, as a special exception, Digia gives you certain additional +# rights. These rights are described in the Digia Qt LGPL Exception +# version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +# +# GNU General Public License Usage +# Alternatively, this file may be used under the terms of the GNU +# General Public License version 3.0 as published by the Free Software +# Foundation and appearing in the file LICENSE.GPL included in the +# packaging of this file. Please review the following information to +# ensure the GNU General Public License version 3.0 requirements will be +# met: http://www.gnu.org/copyleft/gpl.html. +# +# +# $QT_END_LICENSE$ +# +############################################################################# + +import glob +import os +import subprocess +import sys +import imp +import errno +import shutil + +import git_submodule as GitSubmodule + +qtwebengine_src = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +os.chdir(qtwebengine_src) + +def isInGitBlacklist(file_path): + # We do need all the gyp files. + if file_path.endswith('.gyp') or file_path.endswith('.gypi'): + False + if ( '.gitignore' in file_path + or '.gitmodules' in file_path + or '.DEPS' in file_path ): + return True + +def isInChromiumBlacklist(file_path): + # Filter out empty submodule directories. + if (os.path.isdir(file_path)): + return True + # We do need all the gyp files. + if file_path.endswith('.gyp') or file_path.endswith('.gypi'): + return False + if ( '/tests/' in file_path + or ('/test/' in file_path and + not '/webrtc/test/testsupport/' in file_path and + not file_path.startswith('net/test/') and + not file_path.endswith('mock_chrome_application_mac.h')) + or file_path.startswith('third_party/WebKit/LayoutTests') + or file_path.startswith('third_party/WebKit/PerformanceTests') + or file_path.startswith('third_party/WebKit/ManualTests') + or file_path.startswith('android_webview') + or file_path.startswith('apps/') + or file_path.startswith('build/android/') + or (file_path.startswith('chrome/') and + not 'repack_locales' in file_path and + not file_path.endswith('version.py')) + or file_path.startswith('chrome_frame') + or file_path.startswith('chromeos') + or file_path.startswith('breakpad') + or file_path.startswith('third_party/GTM') + or file_path.startswith('third_party/chromite') + or file_path.startswith('third_party/webgl_conformance') + or file_path.startswith('third_party/hunspell_dictionaries') + or file_path.startswith('cloud_print') + or file_path.startswith('ios') + or file_path.startswith('google_update') + or file_path.startswith('courgette') + or file_path.startswith('native_client') + or (file_path.startswith('third_party/trace-viewer') and + not file_path.endswith('.template') and + not file_path.endswith('.html') and + not file_path.endswith('.js') and + not file_path.endswith('.css') and + not file_path.endswith('.png') and + not '/build/' in file_path) + or file_path.startswith('remoting') + or file_path.startswith('win8') ): + return True + return False + + +def createHardLinkForFile(src, dst): + src = os.path.abspath(src) + dst = os.path.abspath(dst) + dst_dir = os.path.dirname(dst) + + if not os.path.isdir(dst_dir): + os.makedirs(dst_dir) + + if os.path.exists(dst): + os.remove(dst) + + try: + os.link(src, dst) + except OSError as exception: + if exception.errno == errno.ENOENT: + print 'file does not exist:' + src + else: + raise + + +third_party_upstream = os.path.join(qtwebengine_src, '3rdparty_upstream') +third_party = os.path.join(qtwebengine_src, '3rdparty') + +def clearDirectory(directory): + currentDir = os.getcwd() + os.chdir(directory) + print 'clearing the directory:' + directory + for direntry in os.listdir(directory): + if not direntry == '.git': + print 'clearing:' + direntry + shutil.rmtree(direntry) + os.chdir(currentDir) + +def listFilesInCurrentRepository(): + currentRepo = GitSubmodule.Submodule(os.getcwd()) + files = subprocess.check_output(['git', 'ls-files']).splitlines() + submodules = currentRepo.readSubmodules() + for submodule in submodules: + submodule_files = submodule.listFiles() + for submodule_file in submodule_files: + files.append(os.path.join(submodule.path, submodule_file)) + return files + +def exportNinja(): + third_party_upstream_ninja = os.path.join(third_party_upstream, 'ninja') + third_party_ninja = os.path.join(third_party, 'ninja') + os.makedirs(third_party_ninja); + print 'exporting contents of:' + third_party_upstream_ninja + os.chdir(third_party_upstream_ninja) + files = listFilesInCurrentRepository() + print 'creating hardlinks in ' + third_party_ninja + for f in files: + if not isInGitBlacklist(f): + createHardLinkForFile(f, os.path.join(third_party_ninja, f)) + +def exportChromium(): + third_party_upstream_chromium = os.path.join(third_party_upstream, 'chromium') + third_party_chromium = os.path.join(third_party, 'chromium') + os.makedirs(third_party_chromium); + print 'exporting contents of:' + third_party_upstream_chromium + os.chdir(third_party_upstream_chromium) + files = listFilesInCurrentRepository() + # Add LASTCHANGE files which are not tracked by git. + files.append('build/util/LASTCHANGE') + files.append('build/util/LASTCHANGE.blink') + print 'creating hardlinks in ' + third_party_chromium + for f in files: + if not isInChromiumBlacklist(f) and not isInGitBlacklist(f): + createHardLinkForFile(f, os.path.join(third_party_chromium, f)) + + +clearDirectory(third_party) + +exportNinja() +exportChromium() + +print 'done.' + -- cgit v1.2.3