Update the scripts for the snapshot of 37

- The svn repo string would be something like:
  ...branches/2062/path_suffix
  and I want only the branch number in this case.
  Updated git_submodule.py to support that case and use regexp
  instead to keep thing a bit simpler.
- .isolate are gyp files, moved the exception at the top
- Changed some inclusions/exclusions
- tools/ is now all excluded by default with exception instead
  of all included. We don't need most of it.

Change-Id: I6d0666d9af7fc6a0f3c4fe866adf2744f13f7c35
Reviewed-by: Andras Becsi <andras.becsi@digia.com>

3 files changed, 72 insertions, 61 deletions

diff --git a/tools/scripts/git_submodule.py b/tools/scripts/git_submodule.py
index 5376e877d..a9002ffe8 100644
--- a/tools/scripts/git_submodule.py
+++ b/tools/scripts/git_submodule.py
@@ -41,6 +41,7 @@
 
 import glob
 import os
+import re
 import subprocess
 import sys
 import version_resolver as resolver
@@ -79,42 +80,24 @@ class DEPSParser:
                 if subdir.startswith('src'):
                     # Ignore the information about chromium since we get that from git.
                     continue
-                shasum = ''
-                if len(rev) == 40: # Length of a git shasum
-                    shasum = rev
+
                 submodule = Submodule(subdir, repo)
                 submodule.os = os
-                submodule.shasum = shasum
+
                 if not submodule.matchesOS():
                     print '-- skipping ' + submodule.path + ' for this operating system. --'
                     continue
-                if not submodule.shasum:
-                    # We need to parse the svn branch and revision number.
-                    ref = repo
-                    # Try to find out the branch.
-                    ref_path = repo.split('branches/')
-                    if len(ref_path) > 1:
-                        ref = ref_path[1]
-                    if 'trunk' in ref or 'svn' in ref:
-                        # Trunk is used, so we can use the remote master.
-                        ref = 'master'
-                    name_path = subdir.split('/')
-                    if len(name_path) > 1:
-                        # At this point some svn repository paths still include the repo name
-                        # after the actual branch so we have to strip it together with the leading /
-                        name = name_path[-1]
-                        if ref.endswith(name):
-                            branch = ref[:-(len(name) + 1)]
-                            ref = 'refs/branch-heads/' + branch
-                        if name in ref:
-                            # At this point the last partition in the path
-                            # is the branch name, so compose the git ref.
-                            branch = ref.split('/')[-1]
-                            ref = 'refs/branch-heads/' + branch
-                    if 'master' not in ref and 'refs/branch-heads' not in ref:
-                        # Finally compose the refs that did not mach previous rules.
-                        ref = 'refs/branch-heads/' + ref
-                    submodule.ref = ref
+
+                if len(rev) == 40: # Length of a git shasum
+                    submodule.shasum = rev
+                else:
+                    # Try to find out the git branch using the svn path.
+                    branchMatch = re.search('/branches/((chromium/)?[^/]+)', repo)
+                    trunkMatch = re.search('/(trunk|svn)/', repo)
+                    if branchMatch:
+                        submodule.ref = 'refs/branch-heads/' + branchMatch.group(1)
+                    elif trunkMatch:
+                        submodule.ref = 'refs/heads/master'
                     submodule.revision = int(rev)
                 submodules.append(submodule)
         return submodules
diff --git a/tools/scripts/take_snapshot.py b/tools/scripts/take_snapshot.py
index 7498a4d76..a2608fb1d 100755
--- a/tools/scripts/take_snapshot.py
+++ b/tools/scripts/take_snapshot.py
@@ -56,7 +56,7 @@ os.chdir(qtwebengine_root)
 
 def isInGitBlacklist(file_path):
     # We do need all the gyp files.
-    if file_path.endswith('.gyp') or file_path.endswith('.gypi'):
+    if file_path.endswith('.gyp') or file_path.endswith('.gypi') or file_path.endswith('.isolate'):
         False
     if ( '.gitignore' in file_path
         or '.gitmodules' in file_path
@@ -68,7 +68,7 @@ def isInChromiumBlacklist(file_path):
     if (os.path.isdir(file_path)):
         return True
     # We do need all the gyp files.
-    if file_path.endswith('.gyp') or file_path.endswith('.gypi'):
+    if file_path.endswith('.gyp') or file_path.endswith('.gypi') or file_path.endswith('.isolate'):
         return False
     if ( '_jni' in file_path
         or 'jni_' in file_path
@@ -77,7 +77,6 @@ def isInChromiumBlacklist(file_path):
             not 'android/cpufeatures' in file_path)
         or '/tests/' in file_path
         or ('/test/' in file_path and
-            not file_path.endswith('isolate') and
             not '/webrtc/test/testsupport/' in file_path and
             not file_path.startswith('net/test/') and
             not file_path.endswith('mock_chrome_application_mac.h') and
@@ -86,6 +85,8 @@ def isInChromiumBlacklist(file_path):
         or file_path.endswith('.java')
         or file_path.startswith('android_webview')
         or file_path.startswith('apps/')
+        or file_path.startswith('ash/')
+        or file_path.startswith('athena')
         or file_path.startswith('base/android/java')
         or file_path.startswith('breakpad')
         or file_path.startswith('build/android/')
@@ -102,7 +103,6 @@ def isInChromiumBlacklist(file_path):
             not 'media/desktop_streams_registry.h' in file_path and
             not 'net/net_error_info' in file_path and
             not 'common/localized_error' in file_path and
-            not file_path.endswith('.isolate') and
             not file_path.endswith('cf_resources.rc') and
             not file_path.endswith('version.py') and
             not file_path.endswith('.grd') and
@@ -111,86 +111,104 @@ def isInChromiumBlacklist(file_path):
         or file_path.startswith('chrome_frame')
         or file_path.startswith('chromeos')
         or file_path.startswith('cloud_print')
+        or (file_path.startswith('components') and
+            not file_path.startswith('components/tracing') and
+            not file_path.startswith('components/visitedlink'))
         or file_path.startswith('content/public/android/java')
-        or file_path.startswith('content/shell/android/java')
+        or file_path.startswith('content/shell')
         or file_path.startswith('courgette')
-        or file_path.startswith('extensions')
+        or (file_path.startswith('extensions') and
+            # Included by generated sources of ui/accessibility/ax_enums.idl
+            not 'browser/extension_function_registry.h' in file_path and
+            not 'browser/extension_function_histogram_value.h' in file_path)
         or file_path.startswith('google_update')
         or file_path.startswith('ios')
         or file_path.startswith('media/base/android/java')
-        or file_path.startswith('mojo')
         or file_path.startswith('native_client')
         or file_path.startswith('net/android/java')
+        or file_path.startswith('pdf')
         or file_path.startswith('remoting')
         or file_path.startswith('rlz')
-        or file_path.startswith('sync/android')
+        or file_path.startswith('sync')
         or file_path.startswith('testing/android')
+        or file_path.startswith('testing/buildbot')
         or file_path.startswith('third_party/accessibility-developer-tools')
         or file_path.startswith('third_party/GTM')
         or file_path.startswith('third_party/WebKit/LayoutTests')
         or file_path.startswith('third_party/WebKit/ManualTests')
         or file_path.startswith('third_party/WebKit/PerformanceTests')
         or file_path.startswith('third_party/active_doc')
+        or file_path.startswith('third_party/android_crazy_linker')
         or file_path.startswith('third_party/android_platform')
         or file_path.startswith('third_party/android_testrunner')
         or file_path.startswith('third_party/aosp')
         or file_path.startswith('third_party/apache-mime4j')
+        or file_path.startswith('third_party/apache-win32')
         or file_path.startswith('third_party/apple_sample_code')
+        or file_path.startswith('third_party/binutils')
         or file_path.startswith('third_party/bison')
-        or file_path.startswith('third_party/brotli')
         or (file_path.startswith('third_party/cacheinvalidation') and
             not file_path.endswith('isolate'))
         or file_path.startswith('third_party/chromite')
         or file_path.startswith('third_party/cld_2')
         or file_path.startswith('third_party/codesighs')
+        or file_path.startswith('third_party/colorama')
         or file_path.startswith('third_party/cros_dbus_cplusplus')
         or file_path.startswith('third_party/cros_system_api')
+        or file_path.startswith('third_party/cygwin')
+        or file_path.startswith('third_party/elfutils')
         or file_path.startswith('third_party/eyesfree')
         or file_path.startswith('third_party/findbugs')
         or file_path.startswith('third_party/gperf')
         or file_path.startswith('third_party/gnu_binutils')
         or file_path.startswith('third_party/gtk+')
+        or file_path.startswith('third_party/google_appengine_cloudstorage')
+        or file_path.startswith('third_party/google_toolbox_for_mac')
         or file_path.startswith('third_party/guava/src')
         or file_path.startswith('third_party/httpcomponents-client')
         or file_path.startswith('third_party/httpcomponents-core')
         or file_path.startswith('third_party/hunspell')
         or file_path.startswith('third_party/hunspell_dictionaries')
+        or file_path.startswith('third_party/instrumented_libraries')
         or file_path.startswith('third_party/jarjar')
         or file_path.startswith('third_party/jsr-305/src')
         or file_path.startswith('third_party/libphonenumber')
         or file_path.startswith('third_party/libaddressinput')
+        or file_path.startswith('third_party/libc++')
+        or file_path.startswith('third_party/libc++abi')
         or file_path.startswith('third_party/liblouis')
         or file_path.startswith('third_party/lighttpd')
+        or file_path.startswith('third_party/markdown')
         or file_path.startswith('third_party/mingw-w64')
         or file_path.startswith('third_party/nacl_sdk_binaries')
         or file_path.startswith('third_party/pdfsqueeze')
         or file_path.startswith('third_party/pefile')
         or file_path.startswith('third_party/perl')
+        or file_path.startswith('third_party/pdfium')
         or file_path.startswith('third_party/psyco_win32')
         or file_path.startswith('third_party/python_26')
         or file_path.startswith('third_party/scons-2.0.1')
         or file_path.startswith('third_party/syzygy')
         or file_path.startswith('third_party/swig')
         or file_path.startswith('third_party/webgl')
-        or (file_path.startswith('third_party/trace-viewer') and
-            not file_path.endswith('.template') and
-            not file_path.endswith('.html') and
-            not file_path.endswith('.js') and
-            not file_path.endswith('.css') and
-            not file_path.endswith('.png') and
-            not '/build/' in file_path)
+        or file_path.startswith('third_party/trace-viewer')
         or file_path.startswith('third_party/xulrunner-sdk')
-        or file_path.startswith('tools/android')
-        or file_path.startswith('tools/deep_memory_profiler')
-        or file_path.startswith('tools/gn')
-        or file_path.startswith('tools/page_cycler')
-        or file_path.startswith('tools/perf')
-        or file_path.startswith('tools/swarming_client')
-        or file_path.startswith('tools/swarm_client')
-        or file_path.startswith('tools/telemetry')
-        or (file_path.startswith('tools/win') and
-           not 'toolchain' in file_path)
+        or (file_path.startswith('tools') and
+           not file_path.startswith('tools/clang') and
+           not file_path.startswith('tools/generate_library_loader') and
+           not file_path.startswith('tools/generate_shim_headers') and
+           not file_path.startswith('tools/generate_stubs') and
+           not file_path.startswith('tools/grit') and
+           not file_path.startswith('tools/gyp') and
+           not file_path.startswith('tools/json_comment_eater') and
+           not file_path.startswith('tools/json_schema_compiler') and
+           not file_path.startswith('tools/idl_parser') and
+           not file_path.startswith('tools/protoc_wrapper'))
         or file_path.startswith('ui/android/java')
+        or file_path.startswith('ui/app_list')
+        or file_path.startswith('ui/chromeos')
+        or file_path.startswith('ui/display/chromeos')
+        or file_path.startswith('ui/file_manager')
 
         ):
             return True
diff --git a/tools/scripts/version_resolver.py b/tools/scripts/version_resolver.py
index aec1bf6fa..396fc626a 100644
--- a/tools/scripts/version_resolver.py
+++ b/tools/scripts/version_resolver.py
@@ -62,14 +62,19 @@ qtwebengine_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..',
 snapshot_src_dir = os.path.abspath(os.path.join(qtwebengine_root, 'src/3rdparty'))
 upstream_src_dir = os.path.abspath(snapshot_src_dir + '_upstream')
 
+submodule_blacklist = [
+    'third_party/WebKit/LayoutTests/w3c/csswg-test'
+    , 'third_party/WebKit/LayoutTests/w3c/web-platform-tests'
+    , 'chrome/tools/test/reference_build/chrome_mac'
+    , 'chrome/tools/test/reference_build/chrome_linux'
+    , 'chrome/tools/test/reference_build/chrome_win'
+    ]
+
 sys.path.append(os.path.join(qtwebengine_root, 'tools', 'scripts'))
 
 def currentVersion():
     return chromium_version
 
-def currentBranch():
-    return chromium_branch
-
 def readReleaseChannels():
     response = urllib2.urlopen(json_url)
     raw_json = response.read().strip()
@@ -139,7 +144,7 @@ def readSubmodules():
         git_dict[sub.path] = sub
 
     for sub in svn_submodules:
-        if 'reference_build' not in sub.path and (sub.revision or sub.shasum) and sub.path in git_dict:
+        if (sub.revision or sub.shasum) and sub.path in git_dict:
             submodule_dict[sub.path] = sub
 
     for git in git_submodules:
@@ -151,6 +156,11 @@ def readSubmodules():
                 # We use the git shasum as fallback.
                 module.shasum = git.shasum
 
+    # Remove unwanted upstream submodules
+    for path in submodule_blacklist:
+        if path in submodule_dict:
+            del submodule_dict[path]
+
     return sanityCheckModules(submodule_dict.values())
 
 def findSnapshotBaselineSha1():