path: root/bin/git-qt-merge-mainlines

#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (C) 2015 The Qt Company Ltd.
# Contact: http://www.qt.io/licensing/
#
# You may use this file under the terms of the 3-clause BSD license.
# See the file LICENSE from this package for details.
#

from __future__ import print_function

import os
import sys
import logging
import subprocess
import re
import warnings

fnull = open(os.devnull, "w")

class colors:
    HEADER = '\033[95m'
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

class BlameParser:
    __begin = ('<' * 7, '=' * 7)
    __end_map = {'merge': '=', 'diff3': '|'}
    # One example line to parse (result from git blame -c --date=short):
    # "ddd8ab31	(Andras Becsi	2014-06-13 18:17:07 +0200	414)bool GLSurface::InitializeOneOffInternal()"
    __matcher = re.compile(r"([0-9a-f]+)\s+\((.*?)\s+\d+\)(.*)").match

    def __init__(self):
        self.__end = (self.__from_git_config('merge.conflictstyle', 'merge', self.__end_map) * 7, '>' * 7)

    def __from_git_config(self, parameter, default, mapping):
        try:
            value = subprocess.check_output(['git', 'config', parameter]).decode('utf-8').strip()
        except subprocess.CalledProcessError:
            # When the value is unset
            value = ''
        try:
            return mapping[value or default]
        except KeyError:
            raise ValueError('Unknown git config', parameter, value)

    def __parse_blame_line(self, codeline):
        # Match a "git blame -c --date=short" output line
        m = self.__matcher(codeline)
        if m:
            return m.group(1, 2, 3)
        raise ValueError("Failed to parse blame line: %s" % (codeline))

    def __get_blame_results(self, conflict):
        cmd_git_blame = ["git", "blame", "-c", "--date=short", conflict]
        blame_output = subprocess.check_output(cmd_git_blame).decode('utf-8')
        return blame_output.split('\n')

    def parse(self, conflict):
        results, scan = {}, False
        for line in self.__get_blame_results(conflict):
            if not line.strip():
                continue
            sha1, info, code = self.__parse_blame_line(line)
            if any(code.startswith(b) for b in self.__begin):
                scan = True
                continue
            if any(code.startswith(e) for e in self.__end):
                scan = False
            if scan:
                results[sha1] = info

        return results.items()

# Try to find related commits for the conflicts
def check_conflicts(module, config, from_ref, conflicts):
    results = ''
    blameparser = BlameParser()
    for conflict in conflicts:
        changes = blameparser.parse(conflict)
        results += conflict + ':\n\t' + '\n\t'.join('%s %s' %c for c in changes) + '\n'
    return results

# Replace all occurrences of searchExp in one file
def replaceInFile(file, searchExp, replaceExp):
    import fileinput
    for line in fileinput.input(file, inplace=1):
        line = re.sub(searchExp, replaceExp, line)
        sys.stdout.write(line)

# Reset module to destination branch
def reset_module(module, config):
    opts = {}
    opts['to'] = config.branch_to
    try:
        git_checkout = "git checkout %(to)s" %opts
        subprocess.check_call(git_checkout.split(), stdout=fnull)
        git_reset = "git reset --hard origin/%(to)s" %opts
        subprocess.check_call(git_reset.split(), stdout=fnull, stderr=fnull)
    except Exception as e:
        logging.error("Git reset failed: %s", str(e))
        return False
    return True

# Run git merge for one module
def do_merge(module, config, from_ref):
    try:
        cmd_git_merge = ["git", "merge", "-m", \
                         "Merge remote-tracking branch \'origin/" + config.branch_from + "\' into " + config.branch_to, \
                         from_ref, "--no-edit", "--no-ff"]
        ret = subprocess.call(cmd_git_merge, stdout=fnull)
        if ret != 0 and config.mergetool:
            print("Starting mergetool")
            ret = subprocess.call(['git', 'mergetool', '-y'])
            if ret == 0:
                ret = subprocess.call(['git', 'commit', '--no-edit'], stdout=fnull)
        if ret != 0:
            print("Module %s failed to merge, manual merge needed." % module)
            return False

        # Make sure we get a change-id as git merge doesn't generate it for us
        git_amend = "git commit --amend --no-edit"
        subprocess.check_call(git_amend.split(), stdout=fnull)

    except Exception as e:
        logging.error("Merge failed: %s", str(e))
        return False

    return True

def wip_merge(module, config, from_ref):
    conflicts = []
    try:
        cmd_git_status = ["git", "status", "--porcelain"]
        status_output = subprocess.check_output(cmd_git_status).decode('utf-8').strip()
        status_count = 0
        if status_output:
            lines = status_output.split('\n')
            status_count = len(lines)

            for line in lines:
                try: mode, file = line.split()
                except (ValueError, TypeError): pass # not exactly two tokens
                else:
                    if 'U' in mode:
                        print("conflict: ", file)
                        conflicts.append(file)

        if not conflicts:
            print("Didn't find any conflict. Need to check carefully.")
            return False

        cmd_git_commit = ["git", "commit", "-a", "-m",
                          "WIP: Merge remote-tracking branch \'origin/" + config.branch_from + "\' into " + config.branch_to + "\n"
                          + '\nConflicts:\n\t' + '\n\t'.join(conflicts) + '\n']
        ret = subprocess.call(cmd_git_commit, stdout=fnull)
        if ret != 0:
            print("Failed to do \"git commit -a\".")
            return False

        # Make sure we get a change-id as git merge doesn't generate it for us
        git_amend = "git commit --amend --no-edit"
        subprocess.check_call(git_amend.split(), stdout=fnull)

        related = check_conflicts(module, config, from_ref, conflicts)
        print("Related: ", related)

    except Exception as e:
        logging.error("Create WIP change failed: %s", str(e))
        return False

    return True

# Change the version number in the .qmake.conf file to a given string
def update_qmake_conf(module, version):
    fileName = ".qmake.conf"
    replaceInFile(fileName, "5\..\..", version)
    cmd_git_status = "git status".split()

    status = subprocess.check_output(cmd_git_status).decode('utf-8')
    print(status)
    if status.find(fileName) < 0:
        return False

    cmd_git_add = "git add .qmake.conf".split()
    subprocess.check_call(cmd_git_add, stdout=fnull)
    cmd_git_commit = ['git', 'commit', '-m', 'Update module version to %s' % version]
    subprocess.check_call(cmd_git_commit, stdout=fnull)
    return True

# Run git push to the destination branch for one module
def push_gerrit(module, config):
    subprocess.call(["git", "show"])
    print('\n"%s" was successfully updated and merged' % module)
    # fix for python 2.x compatibility
    try: input = raw_input
    except NameError: pass
    confirm = input('Push merge? [Y/n] ')
    if not confirm or confirm.startswith(('y', 'Y')):
        output = subprocess.check_output(('git push gerrit HEAD:refs/for/%s' % config.branch_to).split()).decode('utf-8')
        #url = re.search("https.*", output).group(0).strip()
        #print(url)

        #GERRIT_CHANGE_ID=$(echo "$GIT_PUSH" | sed 's/.*https:\/\/codereview.qt-project.org\/\(.*\)/\;tx;d;:x')
        #echo "ChangeId: $GERRIT_CHANGE_ID"
        # if we had gerrit 2.4, we could do this:
        #for reviewer in
        #   sh codereview.qt-project.org gerrit set-reviewers --add jedrzej.nowacki@digia.com 49938

    else:
        print(colors.WARNING + "Merge not pushed." + colors.ENDC)

# Perform merge and push it for one module
def merge(module, config, from_ref):
    push_required = False
    if config.merge or config.wip:
        if not do_merge(module, config, from_ref):
            return False
        push_required = True

    if config.version:
        print('Updating .qmake.conf for', module)
        if update_qmake_conf(module, config.version):
            push_required = True

    if push_required:
        push_gerrit(module, config)
    return True

# Create WIP change when having conflicts
def create_wip(module, config, from_ref):
    if config.wip:
        if not wip_merge(module, config, from_ref):
            return False

        push_gerrit(module, config)

    return True

def get_module_sha_from_super(module, branch):
    result = ""
    cmd_git_lstree = ["git", "ls-tree", "origin/" + branch, module]
    lstree_output = subprocess.check_output(cmd_git_lstree).decode('utf-8').strip()
    lstree_outputs = lstree_output.split()
    if len(lstree_outputs) == 4 and lstree_outputs[3] == module:
        result = lstree_outputs[2].strip()
    return result

# Iterate over all modules (either default or passed with -m option)
def process_modules(config):
    if config.list_modules:
        print("Modules: ", config.modules)
        return

    cmd_git_fetch = ["git", "fetch"]
    subprocess.check_call(cmd_git_fetch, stdout=fnull, stderr=fnull)

    manual_merges = []

    for module in config.modules.split():
        print("\nModule: " + colors.GREEN + module + colors.ENDC)
        if not os.path.isdir(module):
            print("Directory '%s' does not exist. Skipping..." % module)
            continue
        try:
            os.chdir(module)
            try:
                cmd_git_fetch = ["git", "fetch"]
                subprocess.check_call(cmd_git_fetch, stdout=fnull, stderr=fnull)

                to_string = "origin/" + config.branch_to
                from_string = "origin/" + config.branch_from

                if config.super:
                    os.chdir("..")
                    from_sha1 = get_module_sha_from_super(module, config.branch_from)
                    if from_sha1:
                        from_string = from_sha1
                    else:
                        print("Unable to obtain sha1 from super module for module ", module, "\n")
                    os.chdir(module)

                print("Merge from", from_string, "to", to_string, ":")
                cmd_git_cherry = ["git", "cherry", to_string, from_string, "-v"]
                cherry_output = subprocess.check_output(cmd_git_cherry).decode('utf-8').strip()

                change_count = 0
                if cherry_output:
                    change_count = len(cherry_output.split('\n'))

                if config.status:
                    print(cherry_output)
                    print(colors.GREEN, change_count, colors.ENDC, "patches to be merged in", module, "\n")

                if config.reset or config.merge or config.version or config.wip:
                    reset_module(module, config)

                if change_count > 0:
                    if not merge(module, config, from_string):
                        manual_merges.append(module)
                        if config.wip:
                            create_wip(module, config, from_string)

            except Exception as e:
                logging.error("Command execution failed: %s", str(e))
                import traceback
                traceback.print_exc(file=sys.stdout)
            finally:
                os.chdir("..")

        except Exception as e:
            logging.error("Changing current dir failed: %s", str(e))
            import traceback
            traceback.print_exc(file=sys.stdout)

    if manual_merges:
        print("Modules failed to merge: ", manual_merges)

# get a list of submodules and a list of submodules that are not checked out
def get_submodules():
    git_submodule_status = subprocess.check_output('git submodule status'.split(' ')).strip()
    modules = []
    modules_not_checked_out = []
    for line in git_submodule_status.split('\n'):
        module = line.strip().split(' ')[1]
        if line.startswith('-'):
            modules_not_checked_out.append(module)
            print('WARNING:', module, 'is not checked out')
        elif module in ('qtqa', 'qtrepotools'):
            print('skipping', module)
        else:
            modules.append(module)
    return modules, modules_not_checked_out


if __name__== "__main__":
    import argparse
    parser = argparse.ArgumentParser(prog="git-qt-merge-branches",
        description="Merge branches for the Qt Project",
        epilog="""

Run from within a work tree of the qt5 super-module, with all relevant
sub-modules checked out (e.g. using qt5/init-repository).
""")
    group = parser.add_argument_group("Primary action", "Chose one of:")
    group.add_argument('-s', '--status', action="store_true",
                       help='show the status (which patches will be merged)')
    group.add_argument('-d', '--merge', action="store_true",
                       help='do the merge')
    parser.add_argument('-u', '--super', action="store_true",
                        help='obtain sha1s from supermodule instead of using branch tips')
    parser.add_argument('-m', '--modules',
                        help='override the list of modules (eg. -m "qtbase qtdeclarative")')
    group.add_argument('-l', '--list-modules', action="store_true",
                       help='list the modules to be merged and exit')
    group.add_argument('--reset', action="store_true",
                       help='reset to origin/to_branch. this is implicit in the merge command')
    parser.add_argument('-f', '--branch-from', default='stable',
                        help='from which branch to merge')
    parser.add_argument('-t', '--branch-to', default='dev',
                        help='the target branch')
    group.add_argument('-v', '--version', default='',
                       help='set version in .qmake.conf to given version string')
    parser.add_argument('--mergetool', action="store_true",
                        help='run mergetool for conflicts')
    parser.add_argument('-w', '--wip', action="store_true",
                        help='create WIP change when conflicts arise')
    config = parser.parse_args()

    print("Qt Project merge tool\n")
    default_modules, modules_not_checked_out = get_submodules()
    if config.modules is None:
        config.modules = ' '.join(default_modules)
    print("Submodules: ", default_modules)
    print("Ignored submodules: ", modules_not_checked_out)

    logging.basicConfig(format='%(levelname)s: %(message)s')

    if config.status or config.merge or config.reset or config.list_modules or config.version or config.wip:
        process_modules(config)
    else:
        parser.print_help()
    print("Submodules that are not checked out and are ignored:\n",
          ' '.join(modules_not_checked_out))