From f987b713b9f4e363056bf5174a762acb5b77ed79 Mon Sep 17 00:00:00 2001 From: Jocelyn Turcotte Date: Mon, 28 Jul 2014 17:28:51 +0200 Subject: Update the scripts for the snapshot of 37 - The svn repo string would be something like: ...branches/2062/path_suffix and I want only the branch number in this case. Updated git_submodule.py to support that case and use regexp instead to keep thing a bit simpler. - .isolate are gyp files, moved the exception at the top - Changed some inclusions/exclusions - tools/ is now all excluded by default with exception instead of all included. We don't need most of it. Change-Id: I6d0666d9af7fc6a0f3c4fe866adf2744f13f7c35 Reviewed-by: Andras Becsi --- tools/scripts/git_submodule.py | 45 ++++++++----------------- tools/scripts/take_snapshot.py | 70 ++++++++++++++++++++++++--------------- tools/scripts/version_resolver.py | 18 +++++++--- 3 files changed, 72 insertions(+), 61 deletions(-) (limited to 'tools') diff --git a/tools/scripts/git_submodule.py b/tools/scripts/git_submodule.py index 5376e877d..a9002ffe8 100644 --- a/tools/scripts/git_submodule.py +++ b/tools/scripts/git_submodule.py @@ -41,6 +41,7 @@ import glob import os +import re import subprocess import sys import version_resolver as resolver @@ -79,42 +80,24 @@ class DEPSParser: if subdir.startswith('src'): # Ignore the information about chromium since we get that from git. continue - shasum = '' - if len(rev) == 40: # Length of a git shasum - shasum = rev + submodule = Submodule(subdir, repo) submodule.os = os - submodule.shasum = shasum + if not submodule.matchesOS(): print '-- skipping ' + submodule.path + ' for this operating system. --' continue - if not submodule.shasum: - # We need to parse the svn branch and revision number. - ref = repo - # Try to find out the branch. - ref_path = repo.split('branches/') - if len(ref_path) > 1: - ref = ref_path[1] - if 'trunk' in ref or 'svn' in ref: - # Trunk is used, so we can use the remote master. - ref = 'master' - name_path = subdir.split('/') - if len(name_path) > 1: - # At this point some svn repository paths still include the repo name - # after the actual branch so we have to strip it together with the leading / - name = name_path[-1] - if ref.endswith(name): - branch = ref[:-(len(name) + 1)] - ref = 'refs/branch-heads/' + branch - if name in ref: - # At this point the last partition in the path - # is the branch name, so compose the git ref. - branch = ref.split('/')[-1] - ref = 'refs/branch-heads/' + branch - if 'master' not in ref and 'refs/branch-heads' not in ref: - # Finally compose the refs that did not mach previous rules. - ref = 'refs/branch-heads/' + ref - submodule.ref = ref + + if len(rev) == 40: # Length of a git shasum + submodule.shasum = rev + else: + # Try to find out the git branch using the svn path. + branchMatch = re.search('/branches/((chromium/)?[^/]+)', repo) + trunkMatch = re.search('/(trunk|svn)/', repo) + if branchMatch: + submodule.ref = 'refs/branch-heads/' + branchMatch.group(1) + elif trunkMatch: + submodule.ref = 'refs/heads/master' submodule.revision = int(rev) submodules.append(submodule) return submodules diff --git a/tools/scripts/take_snapshot.py b/tools/scripts/take_snapshot.py index 7498a4d76..a2608fb1d 100755 --- a/tools/scripts/take_snapshot.py +++ b/tools/scripts/take_snapshot.py @@ -56,7 +56,7 @@ os.chdir(qtwebengine_root) def isInGitBlacklist(file_path): # We do need all the gyp files. - if file_path.endswith('.gyp') or file_path.endswith('.gypi'): + if file_path.endswith('.gyp') or file_path.endswith('.gypi') or file_path.endswith('.isolate'): False if ( '.gitignore' in file_path or '.gitmodules' in file_path @@ -68,7 +68,7 @@ def isInChromiumBlacklist(file_path): if (os.path.isdir(file_path)): return True # We do need all the gyp files. - if file_path.endswith('.gyp') or file_path.endswith('.gypi'): + if file_path.endswith('.gyp') or file_path.endswith('.gypi') or file_path.endswith('.isolate'): return False if ( '_jni' in file_path or 'jni_' in file_path @@ -77,7 +77,6 @@ def isInChromiumBlacklist(file_path): not 'android/cpufeatures' in file_path) or '/tests/' in file_path or ('/test/' in file_path and - not file_path.endswith('isolate') and not '/webrtc/test/testsupport/' in file_path and not file_path.startswith('net/test/') and not file_path.endswith('mock_chrome_application_mac.h') and @@ -86,6 +85,8 @@ def isInChromiumBlacklist(file_path): or file_path.endswith('.java') or file_path.startswith('android_webview') or file_path.startswith('apps/') + or file_path.startswith('ash/') + or file_path.startswith('athena') or file_path.startswith('base/android/java') or file_path.startswith('breakpad') or file_path.startswith('build/android/') @@ -102,7 +103,6 @@ def isInChromiumBlacklist(file_path): not 'media/desktop_streams_registry.h' in file_path and not 'net/net_error_info' in file_path and not 'common/localized_error' in file_path and - not file_path.endswith('.isolate') and not file_path.endswith('cf_resources.rc') and not file_path.endswith('version.py') and not file_path.endswith('.grd') and @@ -111,86 +111,104 @@ def isInChromiumBlacklist(file_path): or file_path.startswith('chrome_frame') or file_path.startswith('chromeos') or file_path.startswith('cloud_print') + or (file_path.startswith('components') and + not file_path.startswith('components/tracing') and + not file_path.startswith('components/visitedlink')) or file_path.startswith('content/public/android/java') - or file_path.startswith('content/shell/android/java') + or file_path.startswith('content/shell') or file_path.startswith('courgette') - or file_path.startswith('extensions') + or (file_path.startswith('extensions') and + # Included by generated sources of ui/accessibility/ax_enums.idl + not 'browser/extension_function_registry.h' in file_path and + not 'browser/extension_function_histogram_value.h' in file_path) or file_path.startswith('google_update') or file_path.startswith('ios') or file_path.startswith('media/base/android/java') - or file_path.startswith('mojo') or file_path.startswith('native_client') or file_path.startswith('net/android/java') + or file_path.startswith('pdf') or file_path.startswith('remoting') or file_path.startswith('rlz') - or file_path.startswith('sync/android') + or file_path.startswith('sync') or file_path.startswith('testing/android') + or file_path.startswith('testing/buildbot') or file_path.startswith('third_party/accessibility-developer-tools') or file_path.startswith('third_party/GTM') or file_path.startswith('third_party/WebKit/LayoutTests') or file_path.startswith('third_party/WebKit/ManualTests') or file_path.startswith('third_party/WebKit/PerformanceTests') or file_path.startswith('third_party/active_doc') + or file_path.startswith('third_party/android_crazy_linker') or file_path.startswith('third_party/android_platform') or file_path.startswith('third_party/android_testrunner') or file_path.startswith('third_party/aosp') or file_path.startswith('third_party/apache-mime4j') + or file_path.startswith('third_party/apache-win32') or file_path.startswith('third_party/apple_sample_code') + or file_path.startswith('third_party/binutils') or file_path.startswith('third_party/bison') - or file_path.startswith('third_party/brotli') or (file_path.startswith('third_party/cacheinvalidation') and not file_path.endswith('isolate')) or file_path.startswith('third_party/chromite') or file_path.startswith('third_party/cld_2') or file_path.startswith('third_party/codesighs') + or file_path.startswith('third_party/colorama') or file_path.startswith('third_party/cros_dbus_cplusplus') or file_path.startswith('third_party/cros_system_api') + or file_path.startswith('third_party/cygwin') + or file_path.startswith('third_party/elfutils') or file_path.startswith('third_party/eyesfree') or file_path.startswith('third_party/findbugs') or file_path.startswith('third_party/gperf') or file_path.startswith('third_party/gnu_binutils') or file_path.startswith('third_party/gtk+') + or file_path.startswith('third_party/google_appengine_cloudstorage') + or file_path.startswith('third_party/google_toolbox_for_mac') or file_path.startswith('third_party/guava/src') or file_path.startswith('third_party/httpcomponents-client') or file_path.startswith('third_party/httpcomponents-core') or file_path.startswith('third_party/hunspell') or file_path.startswith('third_party/hunspell_dictionaries') + or file_path.startswith('third_party/instrumented_libraries') or file_path.startswith('third_party/jarjar') or file_path.startswith('third_party/jsr-305/src') or file_path.startswith('third_party/libphonenumber') or file_path.startswith('third_party/libaddressinput') + or file_path.startswith('third_party/libc++') + or file_path.startswith('third_party/libc++abi') or file_path.startswith('third_party/liblouis') or file_path.startswith('third_party/lighttpd') + or file_path.startswith('third_party/markdown') or file_path.startswith('third_party/mingw-w64') or file_path.startswith('third_party/nacl_sdk_binaries') or file_path.startswith('third_party/pdfsqueeze') or file_path.startswith('third_party/pefile') or file_path.startswith('third_party/perl') + or file_path.startswith('third_party/pdfium') or file_path.startswith('third_party/psyco_win32') or file_path.startswith('third_party/python_26') or file_path.startswith('third_party/scons-2.0.1') or file_path.startswith('third_party/syzygy') or file_path.startswith('third_party/swig') or file_path.startswith('third_party/webgl') - or (file_path.startswith('third_party/trace-viewer') and - not file_path.endswith('.template') and - not file_path.endswith('.html') and - not file_path.endswith('.js') and - not file_path.endswith('.css') and - not file_path.endswith('.png') and - not '/build/' in file_path) + or file_path.startswith('third_party/trace-viewer') or file_path.startswith('third_party/xulrunner-sdk') - or file_path.startswith('tools/android') - or file_path.startswith('tools/deep_memory_profiler') - or file_path.startswith('tools/gn') - or file_path.startswith('tools/page_cycler') - or file_path.startswith('tools/perf') - or file_path.startswith('tools/swarming_client') - or file_path.startswith('tools/swarm_client') - or file_path.startswith('tools/telemetry') - or (file_path.startswith('tools/win') and - not 'toolchain' in file_path) + or (file_path.startswith('tools') and + not file_path.startswith('tools/clang') and + not file_path.startswith('tools/generate_library_loader') and + not file_path.startswith('tools/generate_shim_headers') and + not file_path.startswith('tools/generate_stubs') and + not file_path.startswith('tools/grit') and + not file_path.startswith('tools/gyp') and + not file_path.startswith('tools/json_comment_eater') and + not file_path.startswith('tools/json_schema_compiler') and + not file_path.startswith('tools/idl_parser') and + not file_path.startswith('tools/protoc_wrapper')) or file_path.startswith('ui/android/java') + or file_path.startswith('ui/app_list') + or file_path.startswith('ui/chromeos') + or file_path.startswith('ui/display/chromeos') + or file_path.startswith('ui/file_manager') ): return True diff --git a/tools/scripts/version_resolver.py b/tools/scripts/version_resolver.py index aec1bf6fa..396fc626a 100644 --- a/tools/scripts/version_resolver.py +++ b/tools/scripts/version_resolver.py @@ -62,14 +62,19 @@ qtwebengine_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', snapshot_src_dir = os.path.abspath(os.path.join(qtwebengine_root, 'src/3rdparty')) upstream_src_dir = os.path.abspath(snapshot_src_dir + '_upstream') +submodule_blacklist = [ + 'third_party/WebKit/LayoutTests/w3c/csswg-test' + , 'third_party/WebKit/LayoutTests/w3c/web-platform-tests' + , 'chrome/tools/test/reference_build/chrome_mac' + , 'chrome/tools/test/reference_build/chrome_linux' + , 'chrome/tools/test/reference_build/chrome_win' + ] + sys.path.append(os.path.join(qtwebengine_root, 'tools', 'scripts')) def currentVersion(): return chromium_version -def currentBranch(): - return chromium_branch - def readReleaseChannels(): response = urllib2.urlopen(json_url) raw_json = response.read().strip() @@ -139,7 +144,7 @@ def readSubmodules(): git_dict[sub.path] = sub for sub in svn_submodules: - if 'reference_build' not in sub.path and (sub.revision or sub.shasum) and sub.path in git_dict: + if (sub.revision or sub.shasum) and sub.path in git_dict: submodule_dict[sub.path] = sub for git in git_submodules: @@ -151,6 +156,11 @@ def readSubmodules(): # We use the git shasum as fallback. module.shasum = git.shasum + # Remove unwanted upstream submodules + for path in submodule_blacklist: + if path in submodule_dict: + del submodule_dict[path] + return sanityCheckModules(submodule_dict.values()) def findSnapshotBaselineSha1(): -- cgit v1.2.3