Add export_from_git.py to take snapshots from chromium and ninja.

Split out the Submodule class and related functions from
init-repository.py into a separate python module git_submodule.py.
This is necessary in order to reuse the code for the export script.

The export_from_git.py script can be used to export files
from a git repository into an arbitrary directory.
It spiders submodules and applies some pattern matching to remove
unwanted files.
This script can be used to create flattened chromium snapshots
to be used in our CI system.

Change-Id: Iade8126fae6c28b5347c9d6e08941e28d3e0e7be
Reviewed-by: Pierre Rossi <pierre.rossi@gmail.com>

3 files changed, 337 insertions, 105 deletions

diff --git a/init-repository.py b/init-repository.py
index 050fc4a21..7abacccb8 100755
--- a/init-repository.py
+++ b/init-repository.py
@@ -49,6 +49,9 @@ import string
 
 qtwebengine_src = os.path.abspath(os.path.join(os.path.dirname(__file__)))
 
+sys.path.append(os.path.join(qtwebengine_src, 'tools'))
+import git_submodule as GitSubmodule
+
 chrome_src = os.environ.get('CHROMIUM_SRC_DIR')
 if chrome_src:
   chrome_src = os.path.abspath(chrome_src)
@@ -64,110 +67,6 @@ def which(tool_name):
             return entry
     return ''
 
-
-class Submodule:
-    def __init__(self):
-        self.path = ''
-        self.url = ''
-        self.shasum = ''
-        self.os = []
-
-    def matchesOS(self):
-        if not self.os:
-            return True
-        if 'all' in self.os:
-            return True
-        if sys.platform.startswith('win32') and 'win' in self.os:
-            return True
-        if sys.platform.startswith('linux') and 'unix' in self.os:
-            return True
-        if sys.platform.startswith('darwin') and ('unix' in self.os or 'mac' in self.os):
-            return True
-        return False
-
-    def findSha(self):
-        line = subprocess.check_output(['git', 'submodule', 'status', self.path])
-        line = line.lstrip(' -')
-        self.shasum = line.split(' ')[0]
-
-    def findGitDir(self):
-        try:
-            return subprocess.check_output(['git', 'rev-parse', '--git-dir']).strip()
-        except subprocess.CalledProcessError, e:
-            sys.exit("git dir could not be determined! - Initialization failed!" + e.output)
-
-    def reset(self):
-        currentDir = os.getcwd()
-        os.chdir(self.path)
-        gitdir = self.findGitDir()
-        if os.path.isdir(os.path.join(gitdir, 'rebase-merge')):
-            if os.path.isfile(os.path.join(gitdir, 'MERGE_HEAD')):
-                print 'merge in progress... aborting merge.'
-                subprocess.call(['git', 'merge', '--abort'])
-            else:
-                print 'rebase in progress... aborting merge.'
-                subprocess.call(['git', 'rebase', '--abort'])
-        if os.path.isdir(os.path.join(gitdir, 'rebase-apply')):
-            print 'am in progress... aborting am.'
-            subprocess.call(['git', 'am', '--abort'])
-        subprocess.call(['git', 'reset', '--hard'])
-        os.chdir(currentDir)
-
-    def initialize(self):
-        if self.matchesOS():
-            self.reset()
-            print '-- initializing ' + self.path + ' --'
-            subprocess.call(['git', 'submodule', 'init', self.path])
-            subprocess.call(['git', 'submodule', 'update', self.path])
-            self.findSha()
-            currentDir = os.getcwd()
-            os.chdir(self.path)
-            # Make sure we have checked out the right shasum.
-            # In case there were other patches applied before.
-            val = subprocess.call(['git', 'checkout', self.shasum])
-            if val != 0:
-                sys.exit("!!! initialization failed !!!")
-            initSubmodules()
-            os.chdir(currentDir)
-        else:
-            print '-- skipping ' + self.path + ' for this operating system. --'
-
-
-def readSubmodules():
-    currentDir = os.getcwd()
-    if not os.path.isfile('.gitmodules'):
-        return []
-    gitmodules_file = open('.gitmodules')
-    gitmodules_lines = gitmodules_file.readlines()
-    gitmodules_file.close()
-
-    submodules = []
-    currentSubmodule = None
-    for line in gitmodules_lines:
-        if line.find('[submodule') == 0:
-            if currentSubmodule:
-                submodules.append(currentSubmodule)
-            currentSubmodule = Submodule()
-        tokens = line.split('=')
-        if len(tokens) >= 2:
-            key = tokens[0].strip()
-            value = tokens[1].strip()
-            if key == 'path':
-                currentSubmodule.path = value
-            elif key == 'url':
-                currentSubmodule.url = value
-            elif key == 'os':
-                currentSubmodule.os = value.split(',')
-    if currentSubmodule:
-        submodules.append(currentSubmodule)
-    return submodules
-
-def initSubmodules():
-    submodules = readSubmodules()
-    for submodule in submodules:
-        submodule.initialize()
-
-
 def updateLastChange():
     currentDir = os.getcwd()
     os.chdir(chrome_src)
@@ -205,7 +104,7 @@ def applyPatches():
 os.chdir(qtwebengine_src)
 addGerritRemote()
 installGitHooks()
-initSubmodules()
+GitSubmodule.initSubmodules()
 applyPatches()
 updateLastChange()
 buildNinja()
diff --git a/tools/export_from_git.py b/tools/export_from_git.py
new file mode 100755
index 000000000..97c2187fd
--- /dev/null
+++ b/tools/export_from_git.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+
+#############################################################################
+#
+# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
+# Contact: http://www.qt-project.org/legal
+#
+# This file is part of the QtWebEngine module of the Qt Toolkit.
+#
+# $QT_BEGIN_LICENSE:LGPL$
+# Commercial License Usage
+# Licensees holding valid commercial Qt licenses may use this file in
+# accordance with the commercial license agreement provided with the
+# Software or, alternatively, in accordance with the terms contained in
+# a written agreement between you and Digia.  For licensing terms and
+# conditions see http://qt.digia.com/licensing.  For further information
+# use the contact form at http://qt.digia.com/contact-us.
+#
+# GNU Lesser General Public License Usage
+# Alternatively, this file may be used under the terms of the GNU Lesser
+# General Public License version 2.1 as published by the Free Software
+# Foundation and appearing in the file LICENSE.LGPL included in the
+# packaging of this file.  Please review the following information to
+# ensure the GNU Lesser General Public License version 2.1 requirements
+# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+#
+# In addition, as a special exception, Digia gives you certain additional
+# rights.  These rights are described in the Digia Qt LGPL Exception
+# version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+#
+# GNU General Public License Usage
+# Alternatively, this file may be used under the terms of the GNU
+# General Public License version 3.0 as published by the Free Software
+# Foundation and appearing in the file LICENSE.GPL included in the
+# packaging of this file.  Please review the following information to
+# ensure the GNU General Public License version 3.0 requirements will be
+# met: http://www.gnu.org/copyleft/gpl.html.
+#
+#
+# $QT_END_LICENSE$
+#
+#############################################################################
+
+import glob
+import os
+import subprocess
+import sys
+import imp
+import errno
+
+import git_submodule as GitSubmodule
+
+def isBlacklisted(file_path):
+    # Filter out empty submodule directories.
+    if (os.path.isdir(file_path)):
+        return True
+    # We do need all the gyp files.
+    if file_path.endswith('.gyp') or file_path.endswith('.gypi'):
+        return False
+    if '/tests/' in file_path:
+        return True
+    if ('/test/' in file_path and
+        not '/webrtc/test/testsupport/' in file_path and
+        not file_path.startswith('net/test/') and
+        not file_path.endswith('mock_chrome_application_mac.h')):
+        return True
+    if file_path.startswith('third_party/WebKit/LayoutTests'):
+        return True
+    if file_path.startswith('third_party/WebKit/PerformanceTests'):
+        return True
+    if file_path.startswith('third_party/WebKit/ManualTests'):
+        return True
+    if file_path.startswith('android_webview'):
+        return True
+    if file_path.startswith('apps/'):
+        return True
+    if file_path.startswith('build/android/'):
+        return True
+    if (file_path.startswith('chrome/') and
+        not 'repack_locales' in file_path and
+        not file_path.endswith('version.py')):
+        return True
+    if file_path.startswith('chrome_frame'):
+        return True
+    if file_path.startswith('chromeos'):
+        return True
+    if file_path.startswith('breakpad'):
+        return True
+    if file_path.startswith('third_party/GTM'):
+        return True
+    if file_path.startswith('third_party/chromite'):
+        return True
+    if file_path.startswith('third_party/webgl_conformance'):
+        return True
+    if file_path.startswith('third_party/hunspell_dictionaries'):
+        return True
+    if file_path.startswith('cloud_print'):
+        return True
+    if file_path.startswith('ios'):
+        return True
+    if file_path.startswith('google_update'):
+        return True
+    if file_path.startswith('courgette'):
+        return True
+    if file_path.startswith('native_client'):
+        return True
+    if (file_path.startswith('third_party/trace-viewer') and
+        not file_path.endswith('.template') and
+        not file_path.endswith('.html') and
+        not file_path.endswith('.js') and
+        not file_path.endswith('.css') and
+        not file_path.endswith('.png') and
+        not '/build/' in file_path):
+        return True
+    if file_path.startswith('remoting'):
+        return True
+    if file_path.startswith('win8'):
+        return True
+    if '.gitignore' in file_path:
+        return True
+    if '.gitmodules' in file_path:
+        return True
+    if '.DEPS' in file_path:
+        return True
+    return False
+
+
+def createHardLinkForFile(src, dst):
+    src = os.path.abspath(src)
+    dst = os.path.abspath(dst)
+    dst_dir = os.path.dirname(dst)
+
+    if not os.path.isdir(dst_dir):
+        os.makedirs(dst_dir)
+
+    if os.path.exists(dst):
+        os.remove(dst)
+
+    try:
+        os.link(src, dst)
+    except OSError as exception:
+        if exception.errno == errno.ENOENT:
+            print 'file does not exist:' + src
+        else:
+            raise
+
+
+if len(sys.argv) != 2:
+    print 'usage: ' + sys.argv[0] + ' [output directory]'
+    exit(1)
+
+
+output_directory = sys.argv[1]
+files = subprocess.check_output(['git', 'ls-files']).splitlines()
+submodules = GitSubmodule.readSubmodules()
+for submodule in submodules:
+    submodule_files = submodule.listFiles()
+    for submodule_file in submodule_files:
+        files.append(os.path.join(submodule.path, submodule_file))
+
+# Add LASTCHANGE files which are not tracked by git.
+files.append('build/util/LASTCHANGE')
+files.append('build/util/LASTCHANGE.blink')
+
+print 'creating hardlinks...'
+file_list = open('/tmp/file_list', 'w')
+for f in files:
+    if not isBlacklisted(f):
+        createHardLinkForFile(f, os.path.join(output_directory, f))
+        file_list.write(f + '\n');
+
+file_list.close()
+
+print 'done.'
+
diff --git a/tools/git_submodule.py b/tools/git_submodule.py
new file mode 100644
index 000000000..d158af635
--- /dev/null
+++ b/tools/git_submodule.py
@@ -0,0 +1,158 @@
+#############################################################################
+#
+# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
+# Contact: http://www.qt-project.org/legal
+#
+# This file is part of the QtWebEngine module of the Qt Toolkit.
+#
+# $QT_BEGIN_LICENSE:LGPL$
+# Commercial License Usage
+# Licensees holding valid commercial Qt licenses may use this file in
+# accordance with the commercial license agreement provided with the
+# Software or, alternatively, in accordance with the terms contained in
+# a written agreement between you and Digia.  For licensing terms and
+# conditions see http://qt.digia.com/licensing.  For further information
+# use the contact form at http://qt.digia.com/contact-us.
+#
+# GNU Lesser General Public License Usage
+# Alternatively, this file may be used under the terms of the GNU Lesser
+# General Public License version 2.1 as published by the Free Software
+# Foundation and appearing in the file LICENSE.LGPL included in the
+# packaging of this file.  Please review the following information to
+# ensure the GNU Lesser General Public License version 2.1 requirements
+# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+#
+# In addition, as a special exception, Digia gives you certain additional
+# rights.  These rights are described in the Digia Qt LGPL Exception
+# version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+#
+# GNU General Public License Usage
+# Alternatively, this file may be used under the terms of the GNU
+# General Public License version 3.0 as published by the Free Software
+# Foundation and appearing in the file LICENSE.GPL included in the
+# packaging of this file.  Please review the following information to
+# ensure the GNU General Public License version 3.0 requirements will be
+# met: http://www.gnu.org/copyleft/gpl.html.
+#
+#
+# $QT_END_LICENSE$
+#
+#############################################################################
+
+import glob
+import os
+import subprocess
+import sys
+
+class Submodule:
+    def __init__(self):
+        self.path = ''
+        self.url = ''
+        self.shasum = ''
+        self.os = []
+
+    def matchesOS(self):
+        if not self.os:
+            return True
+        if 'all' in self.os:
+            return True
+        if sys.platform.startswith('win32') and 'win' in self.os:
+            return True
+        if sys.platform.startswith('linux') and 'unix' in self.os:
+            return True
+        if sys.platform.startswith('darwin') and ('unix' in self.os or 'mac' in self.os):
+            return True
+        return False
+
+    def findSha(self):
+        line = subprocess.check_output(['git', 'submodule', 'status', self.path])
+        line = line.lstrip(' -')
+        self.shasum = line.split(' ')[0]
+
+    def findGitDir(self):
+        try:
+            return subprocess.check_output(['git', 'rev-parse', '--git-dir']).strip()
+        except subprocess.CalledProcessError, e:
+            sys.exit("git dir could not be determined! - Initialization failed!" + e.output)
+
+    def reset(self):
+        currentDir = os.getcwd()
+        os.chdir(self.path)
+        gitdir = self.findGitDir()
+        if os.path.isdir(os.path.join(gitdir, 'rebase-merge')):
+            if os.path.isfile(os.path.join(gitdir, 'MERGE_HEAD')):
+                print 'merge in progress... aborting merge.'
+                subprocess.call(['git', 'merge', '--abort'])
+            else:
+                print 'rebase in progress... aborting merge.'
+                subprocess.call(['git', 'rebase', '--abort'])
+        if os.path.isdir(os.path.join(gitdir, 'rebase-apply')):
+            print 'am in progress... aborting am.'
+            subprocess.call(['git', 'am', '--abort'])
+        subprocess.call(['git', 'reset', '--hard'])
+        os.chdir(currentDir)
+
+    def initialize(self):
+        if self.matchesOS():
+            self.reset()
+            print '-- initializing ' + self.path + ' --'
+            subprocess.call(['git', 'submodule', 'init', self.path])
+            subprocess.call(['git', 'submodule', 'update', self.path])
+            self.findSha()
+            currentDir = os.getcwd()
+            os.chdir(self.path)
+            # Make sure we have checked out the right shasum.
+            # In case there were other patches applied before.
+            val = subprocess.call(['git', 'checkout', self.shasum])
+            if val != 0:
+                sys.exit("!!! initialization failed !!!")
+            initSubmodules()
+            os.chdir(currentDir)
+        else:
+            print '-- skipping ' + self.path + ' for this operating system. --'
+
+    def listFiles(self):
+        if self.matchesOS():
+            currentDir = os.getcwd()
+            os.chdir(self.path)
+            files = subprocess.check_output(['git', 'ls-files']).splitlines()
+            os.chdir(currentDir)
+            return files
+        else:
+            print '-- skipping ' + self.path + ' for this operating system. --'
+            return []
+
+
+def readSubmodules():
+    currentDir = os.getcwd()
+    if not os.path.isfile('.gitmodules'):
+        return []
+    gitmodules_file = open('.gitmodules')
+    gitmodules_lines = gitmodules_file.readlines()
+    gitmodules_file.close()
+
+    submodules = []
+    currentSubmodule = None
+    for line in gitmodules_lines:
+        if line.find('[submodule') == 0:
+            if currentSubmodule:
+                submodules.append(currentSubmodule)
+            currentSubmodule = Submodule()
+        tokens = line.split('=')
+        if len(tokens) >= 2:
+            key = tokens[0].strip()
+            value = tokens[1].strip()
+            if key == 'path':
+                currentSubmodule.path = value
+            elif key == 'url':
+                currentSubmodule.url = value
+            elif key == 'os':
+                currentSubmodule.os = value.split(',')
+    if currentSubmodule:
+        submodules.append(currentSubmodule)
+    return submodules
+
+def initSubmodules():
+    submodules = readSubmodules()
+    for submodule in submodules:
+        submodule.initialize()