path: root/tools/scripts/git_submodule.py

#############################################################################
##
## Copyright (C) 2016 The Qt Company Ltd.
## Contact: https://www.qt.io/licensing/
##
## This file is part of the QtWebEngine module of the Qt Toolkit.
##
## $QT_BEGIN_LICENSE:GPL-EXCEPT$
## Commercial License Usage
## Licensees holding valid commercial Qt licenses may use this file in
## accordance with the commercial license agreement provided with the
## Software or, alternatively, in accordance with the terms contained in
## a written agreement between you and The Qt Company. For licensing terms
## and conditions see https://www.qt.io/terms-conditions. For further
## information use the contact form at https://www.qt.io/contact-us.
##
## GNU General Public License Usage
## Alternatively, this file may be used under the terms of the GNU
## General Public License version 3 as published by the Free Software
## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
## included in the packaging of this file. Please review the following
## information to ensure the GNU General Public License requirements will
## be met: https://www.gnu.org/licenses/gpl-3.0.html.
##
## $QT_END_LICENSE$
##
#############################################################################

import glob
import os
import re
import subprocess
import sys
import version_resolver as resolver

extra_os = ['mac', 'win']

def subprocessCall(args):
    print args
    return subprocess.call(args)

def subprocessCheckOutput(args):
    print args
    return subprocess.check_output(args)

class DEPSParser:
    def __init__(self):
        self.global_scope = {
          'Var': self.Lookup,
          'deps_os': {},
        }
        self.local_scope = {}
        self.topmost_supermodule_path_prefix = ''

    def Lookup(self, var_name):
        return self.local_scope["vars"][var_name]

    def createSubmodulesFromScope(self, scope, os):
        submodules = []
        for dep in scope:
            if (type(scope[dep]) == str):
                repo_rev = scope[dep].split('@')
                repo = repo_rev[0]
                rev = repo_rev[1]
                subdir = dep
                if subdir.startswith('src/'):
                    subdir = subdir[4:]
                # Don't skip submodules that have a supermodule path prefix set (at the moment these
                # are 2nd level deep submodules).
                elif not self.topmost_supermodule_path_prefix:
                    # Ignore the information about chromium itself since we get that from git,
                    # also ignore anything outside src/ (e.g. depot_tools)
                    continue

                submodule = Submodule(subdir, repo, sp=self.topmost_supermodule_path_prefix)
                submodule.os = os

                if not submodule.matchesOS():
                    print '-- skipping ' + submodule.pathRelativeToTopMostSupermodule() + ' for this operating system. --'
                    continue

                if len(rev) == 40: # Length of a git shasum
                    submodule.ref = rev
                else:
                    sys.exit("Invalid shasum: " + str(rev))
                submodules.append(submodule)
        return submodules

    def parse(self, deps_content):
        exec(deps_content, self.global_scope, self.local_scope)

        submodules = []
        submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps'], 'all'))
        if 'deps_os' in self.local_scope:
            for os_dep in self.local_scope['deps_os']:
                submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps_os'][os_dep], os_dep))
        return submodules

# Strips suffix from end of text.
def strip_end(text, suffix):
    if not text.endswith(suffix):
        return text
    return text[:len(text)-len(suffix)]

# Given supermodule_path = /chromium
#      current directory = /chromium/buildtools
#         submodule_path = third_party/foo/bar
# returns                = buildtools
def computeRelativePathPrefixToTopMostSupermodule(submodule_path, supermodule_path):
    relpath = os.path.relpath(submodule_path, supermodule_path)
    topmost_supermodule_path_prefix = strip_end(relpath, submodule_path)
    return topmost_supermodule_path_prefix

class Submodule:
    def __init__(self, path='', url='', ref='', os=[], sp=''):
        self.path = path
        self.url = url
        self.ref = ref
        self.os = os
        self.topmost_supermodule_path_prefix = sp

    def pathRelativeToTopMostSupermodule(self):
        return os.path.normpath(os.path.join(self.topmost_supermodule_path_prefix, self.path))

    def matchesOS(self):
        if not self.os:
            return True
        if 'all' in self.os:
            return True
        if sys.platform.startswith('linux') and 'unix' in self.os:
            return True
        if sys.platform.startswith('darwin') and ('unix' in self.os or 'mac' in self.os):
            return True
        if sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
            if 'win' in self.os:
                return True
            else:
                # Skipping all dependecies of the extra_os on Windows platform, because it caused confict.
                return False
        for os in extra_os:
            if os in self.os:
                return True
        return False

    def findShaAndCheckout(self):
        oldCwd = os.getcwd()
        os.chdir(self.path)

        # Fetch the shasum we parsed from the DEPS file.
        error = subprocessCall(['git', 'fetch', 'origin', self.ref])
        if error != 0:
            print('ERROR: Could not fetch ' + self.ref + ' from upstream origin.')
            return error

        error = subprocessCall(['git', 'checkout', 'FETCH_HEAD']);

        current_shasum = subprocessCheckOutput(['git', 'rev-parse', 'HEAD']).strip()
        current_tag = subprocessCheckOutput(['git', 'name-rev', '--tags', '--name-only', current_shasum]).strip()

        if current_tag == resolver.currentVersion():
            # We checked out a tagged version of chromium.
            self.ref = current_shasum

        if not self.ref:
            # No shasum could be deduced, use the submodule shasum.
            os.chdir(oldCwd)
            line = subprocessCheckOutput(['git', 'submodule', 'status', self.path])
            os.chdir(self.path)
            line = line.lstrip(' -')
            self.ref = line.split(' ')[0]

        if not self.ref.startswith(current_shasum):
            # In case HEAD differs check out the actual shasum we require.
            subprocessCall(['git', 'fetch'])
            error = subprocessCall(['git', 'checkout', self.ref])
        os.chdir(oldCwd)
        return error

    def findGitDir(self):
        try:
            return subprocessCheckOutput(['git', 'rev-parse', '--git-dir']).strip()
        except subprocess.CalledProcessError, e:
            sys.exit("git dir could not be determined! - Initialization failed! " + e.output)

    def reset(self):
        currentDir = os.getcwd()
        os.chdir(self.path)
        gitdir = self.findGitDir()
        if os.path.isdir(os.path.join(gitdir, 'rebase-merge')):
            if os.path.isfile(os.path.join(gitdir, 'MERGE_HEAD')):
                print 'merge in progress... aborting merge.'
                subprocessCall(['git', 'merge', '--abort'])
            else:
                print 'rebase in progress... aborting merge.'
                subprocessCall(['git', 'rebase', '--abort'])
        if os.path.isdir(os.path.join(gitdir, 'rebase-apply')):
            print 'am in progress... aborting am.'
            subprocessCall(['git', 'am', '--abort'])
        subprocessCall(['git', 'reset', '--hard'])
        os.chdir(currentDir)

    def initialize(self):
        if self.matchesOS():
            print '\n\n-- initializing ' + self.pathRelativeToTopMostSupermodule() + ' --'
            oldCwd = os.getcwd()

            # The submodule operations should be done relative to the current submodule's
            # supermodule.
            if self.topmost_supermodule_path_prefix:
                os.chdir(self.topmost_supermodule_path_prefix)

            if os.path.isdir(self.path):
                self.reset()

            if self.url:
                subprocessCall(['git', 'submodule', 'add', '-f', self.url, self.path])
            subprocessCall(['git', 'submodule', 'sync', '--', self.path])
            subprocessCall(['git', 'submodule', 'init', self.path])
            subprocessCall(['git', 'submodule', 'update', self.path])

            if '3rdparty_upstream' in os.path.abspath(self.path):
                if self.findShaAndCheckout() != 0:
                    sys.exit("!!! initialization failed !!!")

                # Add baseline commit for upstream repository to be able to reset.
                os.chdir(self.path)
                commit = subprocessCheckOutput(['git', 'rev-list', '--max-count=1', 'HEAD'])
                subprocessCall(['git', 'commit', '-a', '--allow-empty', '-m', '-- QtWebEngine baseline --\n\ncommit ' + commit])

            os.chdir(oldCwd)
        else:
            print '-- skipping ' + self.path + ' for this operating system. --'

    def listFiles(self):
        if self.matchesOS() and os.path.isdir(self.pathRelativeToTopMostSupermodule()):
            currentDir = os.getcwd()
            os.chdir(self.pathRelativeToTopMostSupermodule())
            files = subprocessCheckOutput(['git', 'ls-files']).splitlines()
            os.chdir(currentDir)
            return files
        else:
            print '-- skipping ' + self.path + ' for this operating system. --'
            return []

    def parseGitModulesFileContents(self, gitmodules_lines):
        submodules = []
        currentSubmodule = None
        for line in gitmodules_lines:
            if line.find('[submodule') == 0:
                if currentSubmodule:
                    submodules.append(currentSubmodule)
                currentSubmodule = Submodule()
            tokens = line.split('=')
            if len(tokens) >= 2:
                key = tokens[0].strip()
                value = tokens[1].strip()
                if key == 'path':
                    currentSubmodule.path = value
                elif key == 'url':
                    currentSubmodule.url = value
                elif key == 'os':
                    currentSubmodule.os = value.split(',')
        if currentSubmodule:
            submodules.append(currentSubmodule)
        return submodules

    # Return a flattened list of submodules starting from module, and recursively collecting child
    # submodules.
    def readSubmodulesFromGitModules(self, module, gitmodules_file_name, top_level_path):
        flattened_submodules = []
        oldCwd = os.getcwd()
        os.chdir(module.path)

        if os.path.isfile(gitmodules_file_name):
            gitmodules_file = open(gitmodules_file_name)
            gitmodules_lines = gitmodules_file.readlines()
            gitmodules_file.close()
            submodules = self.parseGitModulesFileContents(gitmodules_lines)

            # When inside a 2nd level submodule or deeper, store the path relative to the topmost
            # module.
            for submodule in submodules:
                submodule.topmost_supermodule_path_prefix = computeRelativePathPrefixToTopMostSupermodule(submodule.path, top_level_path)

            flattened_submodules.extend(submodules)

            # Recurse into deeper submodules.
            for submodule in submodules:
                flattened_submodules.extend(self.readSubmodulesFromGitModules(submodule, gitmodules_file_name, top_level_path))

        os.chdir(oldCwd)
        return flattened_submodules

    def readSubmodules(self):
        submodules = []
        if self.ref:
            submodules = resolver.readSubmodules()
            print 'DEPS file provides the following submodules:'
            for submodule in submodules:
                print '{:<80}'.format(submodule.pathRelativeToTopMostSupermodule()) + '{:<120}'.format(submodule.url) + submodule.ref
        else: # Try .gitmodules since no ref has been specified
            gitmodules_file_name = '.gitmodules'
            submodules = self.readSubmodulesFromGitModules(self, gitmodules_file_name, self.path)
        return submodules

    def initSubmodules(self):
        oldCwd = os.getcwd()
        os.chdir(self.path)
        submodules = self.readSubmodules()
        for submodule in submodules:
            submodule.initialize()
        subprocessCall(['git', 'commit', '-a', '--amend', '--no-edit'])
        os.chdir(oldCwd)