path: root/tools/scripts/git_submodule.py

# Copyright (C) 2016 The Qt Company Ltd.
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0

import glob
import os
import re
import string
import subprocess
import sys
import version_resolver as resolver

extra_os = ['mac', 'win']

def subprocessCall(args):
    print(args)
    return subprocess.call(args)

def subprocessCheckOutput(args):
    print(args)
    return subprocess.check_output(args).decode()

# Special string formatter that support flat key names with '.' in them
class DepsFormatter(string.Formatter):
    def get_field(self, field_name, args, kwargs):
        return (self.get_value(field_name, args, kwargs), field_name)

class SubmoduleDEPSParser(resolver.DEPSParser):
    def __init__(self):
        super().__init__()

    def get_vars(self):
        """Returns a dictionary of effective variable values
        (DEPS file contents with applied custom_vars overrides)."""
        result = {}
        # Variables defined in DEPS file override built-in ones.
        result.update(self.local_scope["vars"])
        #result.update(self.custom_vars or {})
        return result

    def createSubmodulesFromScope(self, scope, os, module_whitelist = []):
        submodules = []
        for dep in scope:
            url = ''
            if (type(scope[dep]) == str):
                url = scope[dep]
            elif (type(scope[dep]) == dict and 'url' in scope[dep]):
                url = scope[dep]['url']

                if ('condition' in scope[dep]) and \
                (not 'checkout_linux' in scope[dep]['condition']) and \
                (not dep in module_whitelist):
                    continue
            if url:
                url = DepsFormatter().vformat(url, [], self.get_vars())
                repo_rev = url.split('@')
                repo = repo_rev[0]
                rev = repo_rev[1]
                subdir = self.subdir(dep)
                if subdir is None:
                    continue
                submodule = Submodule(subdir, repo, sp=self.topmost_supermodule_path_prefix)
                submodule.os = os

                if not submodule.matchesOS():
                    print('-- skipping ' + submodule.pathRelativeToTopMostSupermodule() + ' for this operating system. --')
                    continue

                if len(rev) == 40: # Length of a git shasum
                    submodule.ref = rev
                elif len(rev) == 0:
                    submodule.ref = 'master'
                else:
                    sys.exit("Invalid shasum: " + str(url))
                submodules.append(submodule)
        return submodules

    def parse(self, deps_content, module_whitelist = []):
        exec(deps_content, self.global_scope, self.local_scope)

        submodules = []
        submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps'], 'all', module_whitelist))
        if 'deps_os' in self.local_scope:
            for os_dep in self.local_scope['deps_os']:
                submodules.extend(self.createSubmodulesFromScope(self.local_scope['deps_os'][os_dep], os_dep))
        return submodules

# Strips suffix from end of text.
def strip_end(text, suffix):
    if not text.endswith(suffix):
        return text
    return text[:len(text)-len(suffix)]

# Given supermodule_path = /chromium
#      current directory = /chromium/buildtools
#         submodule_path = third_party/foo/bar
# returns                = buildtools
def computeRelativePathPrefixToTopMostSupermodule(submodule_path, supermodule_path):
    relpath = os.path.relpath(submodule_path, supermodule_path)
    topmost_supermodule_path_prefix = strip_end(relpath, submodule_path)
    return topmost_supermodule_path_prefix

class Submodule:
    def __init__(self, path='', url='', ref='', os=[], sp=''):
        self.path = path
        self.url = url
        self.ref = ref
        self.os = os
        self.topmost_supermodule_path_prefix = sp

    def pathRelativeToTopMostSupermodule(self):
        return os.path.normpath(os.path.join(self.topmost_supermodule_path_prefix, self.path))

    def matchesOS(self):
        if not self.os:
            return True
        if 'all' in self.os:
            return True
        if sys.platform.startswith('linux') and 'unix' in self.os:
            return True
        if sys.platform.startswith('darwin') and ('unix' in self.os or 'mac' in self.os):
            return True
        if sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
            if 'win' in self.os:
                return True
            else:
                # Skipping all dependecies of the extra_os on Windows platform, because it caused confict.
                return False
        for os in extra_os:
            if os in self.os:
                return True
        return False

    def findShaAndCheckout(self):
        oldCwd = os.getcwd()
        os.chdir(self.path)

        # Fetch the shasum we parsed from the DEPS file.
        error = subprocessCall(['git', 'fetch', 'origin', self.ref])
        if error != 0:
            print('ERROR: Could not fetch ' + self.ref + ' from upstream origin.')
            return error

        error = subprocessCall(['git', 'checkout', 'FETCH_HEAD']);

        current_shasum = subprocessCheckOutput(['git', 'rev-parse', 'HEAD']).strip()
        current_tag = subprocessCheckOutput(['git', 'name-rev', '--tags', '--name-only', current_shasum]).strip()

        if current_tag == resolver.currentVersion():
            # We checked out a tagged version of chromium.
            self.ref = current_shasum

        if not self.ref:
            # No shasum could be deduced, use the submodule shasum.
            os.chdir(oldCwd)
            line = subprocessCheckOutput(['git', 'submodule', 'status', self.path])
            os.chdir(self.path)
            line = line.lstrip(' -')
            self.ref = line.split(' ')[0]

        if not self.ref.startswith(current_shasum):
            # In case HEAD differs check out the actual shasum we require.
            subprocessCall(['git', 'fetch'])
            error = subprocessCall(['git', 'checkout', self.ref])
        os.chdir(oldCwd)
        return error

    def findGitDir(self):
        try:
            return subprocessCheckOutput(['git', 'rev-parse', '--git-dir']).strip()
        except subprocess.CalledProcessError as e:
            sys.exit("git dir could not be determined! - Initialization failed! " + e.output)

    def reset(self):
        currentDir = os.getcwd()
        os.chdir(self.path)
        gitdir = self.findGitDir()
        if os.path.isdir(os.path.join(gitdir, 'rebase-merge')):
            if os.path.isfile(os.path.join(gitdir, 'MERGE_HEAD')):
                print('merge in progress... aborting merge.')
                subprocessCall(['git', 'merge', '--abort'])
            else:
                print('rebase in progress... aborting merge.')
                subprocessCall(['git', 'rebase', '--abort'])
        if os.path.isdir(os.path.join(gitdir, 'rebase-apply')):
            print('am in progress... aborting am.')
            subprocessCall(['git', 'am', '--abort'])
        subprocessCall(['git', 'reset', '--hard'])
        os.chdir(currentDir)

    def initialize(self):
        if self.matchesOS():
            print('\n\n-- initializing ' + self.pathRelativeToTopMostSupermodule() + ' --')
            oldCwd = os.getcwd()

            # The submodule operations should be done relative to the current submodule's
            # supermodule.
            if self.topmost_supermodule_path_prefix:
                if not os.path.isdir(self.path):
                    print('-- creating ' + self.path + ' as dir is missing. --')
                    os.makedirs(self.path)
                os.chdir(self.topmost_supermodule_path_prefix)

            if os.path.isdir(self.path):
                self.reset()

            if self.url:
                subprocessCall(['git', 'submodule', 'add', '-f', self.url, self.path])
            subprocessCall(['git', 'submodule', 'sync', '--', self.path])
            subprocessCall(['git', 'submodule', 'init', self.path])
            subprocessCall(['git', 'submodule', 'update', self.path])

            if '3rdparty_upstream' in os.path.abspath(self.path):
                if self.findShaAndCheckout() != 0:
                    sys.exit("!!! initialization failed !!!")

                # Add baseline commit for upstream repository to be able to reset.
                os.chdir(self.path)
                commit = subprocessCheckOutput(['git', 'rev-list', '--max-count=1', 'HEAD'])
                subprocessCall(['git', 'commit', '-a', '--allow-empty', '-m', '-- QtWebEngine baseline --\n\ncommit ' + commit])

            os.chdir(oldCwd)
        else:
            print('-- skipping ' + self.path + ' for this operating system. --')

    def listFiles(self):
        if self.matchesOS() and os.path.isdir(self.pathRelativeToTopMostSupermodule()):
            currentDir = os.getcwd()
            os.chdir(self.pathRelativeToTopMostSupermodule())
            files = subprocessCheckOutput(['git', 'ls-files']).splitlines()
            os.chdir(currentDir)
            return files
        else:
            print('-- skipping ' + self.path + ' for this operating system. --')
            return []

    def parseGitModulesFileContents(self, gitmodules_lines):
        submodules = []
        currentSubmodule = None
        for line in gitmodules_lines:
            if line.find('[submodule') == 0:
                if currentSubmodule:
                    submodules.append(currentSubmodule)
                currentSubmodule = Submodule()
            tokens = line.split('=')
            if len(tokens) >= 2:
                key = tokens[0].strip()
                value = tokens[1].strip()
                if key == 'path':
                    currentSubmodule.path = value
                elif key == 'url':
                    currentSubmodule.url = value
                elif key == 'os':
                    currentSubmodule.os = value.split(',')
        if currentSubmodule:
            submodules.append(currentSubmodule)
        return submodules

    # Return a flattened list of submodules starting from module, and recursively collecting child
    # submodules.
    def readSubmodulesFromGitModules(self, module, gitmodules_file_name, top_level_path):
        flattened_submodules = []
        oldCwd = os.getcwd()
        os.chdir(module.path)

        if os.path.isfile(gitmodules_file_name):
            gitmodules_file = open(gitmodules_file_name)
            gitmodules_lines = gitmodules_file.readlines()
            gitmodules_file.close()
            submodules = self.parseGitModulesFileContents(gitmodules_lines)

            # When inside a 2nd level submodule or deeper, store the path relative to the topmost
            # module.
            for submodule in submodules:
                submodule.topmost_supermodule_path_prefix = computeRelativePathPrefixToTopMostSupermodule(submodule.path, top_level_path)

            flattened_submodules.extend(submodules)

            # Recurse into deeper submodules.
            for submodule in submodules:
                flattened_submodules.extend(self.readSubmodulesFromGitModules(submodule, gitmodules_file_name, top_level_path))

        os.chdir(oldCwd)
        return flattened_submodules

    def readSubmodules(self, use_deps=False):
        submodules = []
        if use_deps:
            submodules = resolver.read(SubmoduleDEPSParser)
            print('DEPS file provides the following submodules:')
            for submodule in submodules:
                print('{:<80}'.format(submodule.pathRelativeToTopMostSupermodule()) + '{:<120}'.format(submodule.url) + submodule.ref)
        else: # Try .gitmodules instead
            gitmodules_file_name = '.gitmodules'
            submodules = self.readSubmodulesFromGitModules(self, gitmodules_file_name, self.path)
        return submodules

    def initSubmodules(self):
        oldCwd = os.getcwd()
        os.chdir(self.path)
        submodules = self.readSubmodules(True)
        for submodule in submodules:
            submodule.initialize()
        subprocessCall(['git', 'commit', '-a', '--amend', '--no-edit'])
        os.chdir(oldCwd)