From 662d721dbca23809e821b4d5945187c2969db9e2 Mon Sep 17 00:00:00 2001 From: Alexandru Croitor Date: Fri, 8 Apr 2016 10:49:39 +0200 Subject: Get additional sanitizer dependencies from upstream chromium. The buildtools directory contains custom libc++ libraries, which are a second-level git submodule (the first-level being the buildtools submodule, and the supermodule being chromium itself). The init_repository.py script does not support checking out submodules recursively, thus leading to the mentioned libraries being skipped. This change adds a hardcoded addition to make sure the buildtools submodules are checked out when init_repository is run, as well as support for copying all nested files in the submodule tree, when the take_snapshot script is executed. Thus it copies all necessary sanitizer dependencies. Change-Id: Icb5cf5b52c3d0a83c4690fb7d641cee4f5dc1132 Reviewed-by: Allan Sandfeld Jensen --- tools/scripts/git_submodule.py | 124 +++++++++++++++++++++++++++----------- tools/scripts/take_snapshot.py | 9 ++- tools/scripts/version_resolver.py | 12 ++++ 3 files changed, 108 insertions(+), 37 deletions(-) (limited to 'tools/scripts') diff --git a/tools/scripts/git_submodule.py b/tools/scripts/git_submodule.py index 9252a9424..fcf2af37a 100644 --- a/tools/scripts/git_submodule.py +++ b/tools/scripts/git_submodule.py @@ -50,6 +50,7 @@ class DEPSParser: 'deps_os': {}, } self.local_scope = {} + self.topmost_supermodule_path_prefix = '' def Lookup(self, var_name): return self.local_scope["vars"][var_name] @@ -64,16 +65,18 @@ class DEPSParser: subdir = dep if subdir.startswith('src/'): subdir = subdir[4:] - else: + # Don't skip submodules that have a supermodule path prefix set (at the moment these + # are 2nd level deep submodules). + elif not self.topmost_supermodule_path_prefix: # Ignore the information about chromium itself since we get that from git, # also ignore anything outside src/ (e.g. depot_tools) continue - submodule = Submodule(subdir, repo) + submodule = Submodule(subdir, repo, sp=self.topmost_supermodule_path_prefix) submodule.os = os if not submodule.matchesOS(): - print '-- skipping ' + submodule.path + ' for this operating system. --' + print '-- skipping ' + submodule.pathRelativeToTopMostSupermodule() + ' for this operating system. --' continue if len(rev) == 40: # Length of a git shasum @@ -88,16 +91,36 @@ class DEPSParser: submodules = [] submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps'], 'all')) - for os_dep in self.local_scope['deps_os']: - submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps_os'][os_dep], os_dep)) + if 'deps_os' in self.local_scope: + for os_dep in self.local_scope['deps_os']: + submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps_os'][os_dep], os_dep)) return submodules +# Strips suffix from end of text. +def strip_end(text, suffix): + if not text.endswith(suffix): + return text + return text[:len(text)-len(suffix)] + +# Given supermodule_path = /chromium +# current directory = /chromium/buildtools +# submodule_path = third_party/foo/bar +# returns = buildtools +def computeRelativePathPrefixToTopMostSupermodule(submodule_path, supermodule_path): + relpath = os.path.relpath(submodule_path, supermodule_path) + topmost_supermodule_path_prefix = strip_end(relpath, submodule_path) + return topmost_supermodule_path_prefix + class Submodule: - def __init__(self, path='', url='', ref='', os=[]): + def __init__(self, path='', url='', ref='', os=[], sp=''): self.path = path self.url = url self.ref = ref self.os = os + self.topmost_supermodule_path_prefix = sp + + def pathRelativeToTopMostSupermodule(self): + return os.path.normpath(os.path.join(self.topmost_supermodule_path_prefix, self.path)) def matchesOS(self): if not self.os: @@ -178,8 +201,14 @@ class Submodule: def initialize(self): if self.matchesOS(): - print '\n\n-- initializing ' + self.path + ' --' + print '\n\n-- initializing ' + self.pathRelativeToTopMostSupermodule() + ' --' oldCwd = os.getcwd() + + # The submodule operations should be done relative to the current submodule's + # supermodule. + if self.topmost_supermodule_path_prefix: + os.chdir(self.topmost_supermodule_path_prefix) + if os.path.isdir(self.path): self.reset() @@ -203,9 +232,9 @@ class Submodule: print '-- skipping ' + self.path + ' for this operating system. --' def listFiles(self): - if self.matchesOS() and os.path.isdir(self.path): + if self.matchesOS() and os.path.isdir(self.pathRelativeToTopMostSupermodule()): currentDir = os.getcwd() - os.chdir(self.path) + os.chdir(self.pathRelativeToTopMostSupermodule()) files = subprocessCheckOutput(['git', 'ls-files']).splitlines() os.chdir(currentDir) return files @@ -213,6 +242,54 @@ class Submodule: print '-- skipping ' + self.path + ' for this operating system. --' return [] + def parseGitModulesFileContents(self, gitmodules_lines): + submodules = [] + currentSubmodule = None + for line in gitmodules_lines: + if line.find('[submodule') == 0: + if currentSubmodule: + submodules.append(currentSubmodule) + currentSubmodule = Submodule() + tokens = line.split('=') + if len(tokens) >= 2: + key = tokens[0].strip() + value = tokens[1].strip() + if key == 'path': + currentSubmodule.path = value + elif key == 'url': + currentSubmodule.url = value + elif key == 'os': + currentSubmodule.os = value.split(',') + if currentSubmodule: + submodules.append(currentSubmodule) + return submodules + + # Return a flattened list of submodules starting from module, and recursively collecting child + # submodules. + def readSubmodulesFromGitModules(self, module, gitmodules_file_name, top_level_path): + flattened_submodules = [] + oldCwd = os.getcwd() + os.chdir(module.path) + + if os.path.isfile(gitmodules_file_name): + gitmodules_file = open(gitmodules_file_name) + gitmodules_lines = gitmodules_file.readlines() + gitmodules_file.close() + submodules = self.parseGitModulesFileContents(gitmodules_lines) + + # When inside a 2nd level submodule or deeper, store the path relative to the topmost + # module. + for submodule in submodules: + submodule.topmost_supermodule_path_prefix = computeRelativePathPrefixToTopMostSupermodule(submodule.path, top_level_path) + + flattened_submodules.extend(submodules) + + # Recurse into deeper submodules. + for submodule in submodules: + flattened_submodules.extend(self.readSubmodulesFromGitModules(submodule, gitmodules_file_name, top_level_path)) + + os.chdir(oldCwd) + return flattened_submodules def readSubmodules(self): submodules = [] @@ -220,33 +297,10 @@ class Submodule: submodules = resolver.readSubmodules() print 'DEPS file provides the following submodules:' for submodule in submodules: - print '{:<80}'.format(submodule.path) + '{:<120}'.format(submodule.url) + submodule.ref + print '{:<80}'.format(submodule.pathRelativeToTopMostSupermodule()) + '{:<120}'.format(submodule.url) + submodule.ref else: # Try .gitmodules since no ref has been specified - if not os.path.isfile('.gitmodules'): - return [] - gitmodules_file = open('.gitmodules') - gitmodules_lines = gitmodules_file.readlines() - gitmodules_file.close() - - submodules = [] - currentSubmodule = None - for line in gitmodules_lines: - if line.find('[submodule') == 0: - if currentSubmodule: - submodules.append(currentSubmodule) - currentSubmodule = Submodule() - tokens = line.split('=') - if len(tokens) >= 2: - key = tokens[0].strip() - value = tokens[1].strip() - if key == 'path': - currentSubmodule.path = value - elif key == 'url': - currentSubmodule.url = value - elif key == 'os': - currentSubmodule.os = value.split(',') - if currentSubmodule: - submodules.append(currentSubmodule) + gitmodules_file_name = '.gitmodules' + submodules = self.readSubmodulesFromGitModules(self, gitmodules_file_name, self.path) return submodules def initSubmodules(self): diff --git a/tools/scripts/take_snapshot.py b/tools/scripts/take_snapshot.py index f5a1ed9d9..25b3dc15e 100755 --- a/tools/scripts/take_snapshot.py +++ b/tools/scripts/take_snapshot.py @@ -78,6 +78,7 @@ def isInChromiumBlacklist(file_path): or file_path.startswith('base/android/java') or file_path.startswith('breakpad') or file_path.startswith('build/android/') + or file_path.startswith('buildtools/clang_format/script') or (file_path.startswith('chrome/') and not file_path.startswith('chrome/VERSION') and not file_path.startswith('chrome/browser/chrome_notification_types.h') and @@ -204,6 +205,7 @@ def isInChromiumBlacklist(file_path): or file_path.startswith('third_party/trace-viewer') or file_path.startswith('third_party/undoview') or file_path.startswith('third_party/webgl') + or file_path.startswith('tools/memory_inspector') or (file_path.startswith('tools') and not file_path.startswith('tools/clang') and not file_path.startswith('tools/compile_test') and @@ -215,7 +217,10 @@ def isInChromiumBlacklist(file_path): not file_path.startswith('tools/json_comment_eater') and not file_path.startswith('tools/json_schema_compiler') and not file_path.startswith('tools/idl_parser') and - not file_path.startswith('tools/protoc_wrapper')) + not file_path.startswith('tools/memory') and + not file_path.startswith('tools/msan') and + not file_path.startswith('tools/protoc_wrapper') and + not file_path.startswith('tools/ubsan')) or file_path.startswith('ui/android/java') or file_path.startswith('ui/app_list') or file_path.startswith('ui/base/ime/chromeos') @@ -274,7 +279,7 @@ def listFilesInCurrentRepository(): for submodule in submodules: submodule_files = submodule.listFiles() for submodule_file in submodule_files: - files.append(os.path.join(submodule.path, submodule_file)) + files.append(os.path.join(submodule.pathRelativeToTopMostSupermodule(), submodule_file)) return files def exportNinja(): diff --git a/tools/scripts/version_resolver.py b/tools/scripts/version_resolver.py index 634fec23e..6029bbdc9 100644 --- a/tools/scripts/version_resolver.py +++ b/tools/scripts/version_resolver.py @@ -89,6 +89,18 @@ def readSubmodules(): for sub in git_submodules: submodule_dict[sub.path] = sub + # Add buildtools submodules + buildtools_deps_file_path = "buildtools/DEPS" + if (os.path.isfile(buildtools_deps_file_path)): + with open(buildtools_deps_file_path, 'r') as buildtools_deps_file: + buildtools_deps = buildtools_deps_file.read() + if buildtools_deps: + buildtools_parser = GitSubmodule.DEPSParser() + buildtools_parser.topmost_supermodule_path_prefix = './buildtools/' + buildtools_submodules = buildtools_parser.parse(buildtools_deps) + for sub in buildtools_submodules: + submodule_dict[sub.path] = sub + # Remove unwanted upstream submodules for path in submodule_blacklist: if path in submodule_dict: -- cgit v1.2.3