path: root/scripts/api-review/api-review-gen

#!/bin/sh
usage () { echo Usage: `basename $0` "[-h|-u] [-v] [--] prior soon"; }
# Copyright (C) 2020 The Qt Company Ltd.
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

help () {
    usage
    cat <<EOF

Prepare a commit for pushing to Gerrit, that expresses the interesting
changes in API between a prior release (tag or branch) and an imminent
release (branch).  Certain boring changes are excluded; you can review
them after this has run with git diff -D; if necessary, you can amend
the commit to remove any remaining boredom or include anything
mis-classified as boring, before pushing to Gerrit.

Run in the top-level directory of a work tree in a clean state; don't
expect it to be left in a clean state. In particular, any sub-modules
shall be checked out on their versions from the prior release.

Depends on the dulwich package for python; see your local package
manager or 'pip install dulwich' if managing python packages with pip.

Options:

 -u
 --usage   Print simple usage summary (first line above) and exit.

 -h
 --help    Print this help and exit.

 -t ID
 --task ID
 --task-number ID
           Include a Task-number: in the commit message, with the
           given id, ID, of the bug-tracker issue being used to keep
           track of the API change review.

 -a
 --amend   Amend existing commit rather than extending the review
           branch, if it exists already.

 --replace Replace existing branch, if present, re-using its last
           Change-Id, rather than updating it.  (Conflicts with
           --amend, which should be preferred unless these scripts
           have changed.  If both are used, the last wins.)  If
           --task-number has not been specified, any Task-number:
           footer in the earlier commit will also be preserved.

 -r name
 --release name
           Supply a name for the imminent release, if the branch given
           for it doesn't match (e.g. when preparing a review on dev,
           before the release branch is created).

 -v
 --verbose Say what's happening as it happens.

 -q
 --quiet   Don't mention anything but errors.

 --        End option processing.

Arguments:

 prior     The commit (branch, tag or sha1) of a prior release.
 soon      The branch approaching release.

You should see the usual git output describing the commit created.
You should verify that git diff -D is all boring and the commit is all
interesting before pushing this commit to Gerrit.

Exit status is 1 on error, else 0.  Success may mean no change at all,
only boring changes or some interesting changes have been saved.  If
there are no changes, the repository is restored to its prior state
(including deleting the api-review-* branch, if created rather than
reused), except for any sub-modules.  The other two cases can be
distinguished by comparing the api-review-* branch with the prior
release using git diff --quiet; this succeeds if there is no
difference - i.e. if the API change was all boring so we made no
commit.

After a first pass of review, if prior or soon has changed (and been
fetched), it is possible to rebase the review branch onto prior and
re-run this script to update the review.  If you pass the --amend
option and there was a prior commit on the branch, that commit shall
be amended rather than a new commit added to the branch.  Otherwise,
any change shall be recorded as a fresh commit - which you can always
squash onto an earlier commit later, after reviewing it, if you prefer
that work-flow so left out --amend.  Either way, you can then push the
squashed or amended commit to Gerrit for re-review, as usual.
EOF
}
warn () { echo "$@" >&2; }
die () { warn "$@"; exit 1; }
second () {
    if [ $# -lt 2 ]
    then die "No argument supplied for $1"
    elif [ -z "$2" ]
    then die "Empty argument passed for $1 "
    fi
    echo "$2"
}

modules () {
    # Scan for CMakeLists.txt files defining non-Private, non-plugin Qt modules:
    # Known variants on qt_internal_add_*_module: protobuf, qml
    find src -type d -name .git -prune -o -type f -name CMakeLists.txt -print \
        | xargs grep 'qt_internal_add\(_[a-z][a-z]*\)\?_module(' \
        | grep -wv 'qt_internal_add.*module([A-Za-z0-9]*Private' \
        | grep -wv 'qt_internal_add.*module([A-Za-z0-9]*plugin' \
        | sed -e 's/: *qt_internal_add.*_module(/ /' \
        | while read file module ignore
    do echo $module $file
    done
}

# Parse command-line:
RELNAME=
CHATTY=
AMEND=
TASK=
bad () { usage >&2; die "$@"; }
while [ $# -gt 0 ]
do  case "$1" in
        -u|--usage) usage; exit 0 ;;
        -h|--help) help; exit 0 ;;
        -a|--amend) AMEND=--amend; shift ;;
        --replace) AMEND=--replace; shift ;;
        -r|--release) RELNAME=`second "$@"`; shift 2 ;;
        -t|--task|--task-number) TASK=`second "$@"`; shift 2 ;;
        -v|--verbose) CHATTY=more; shift ;;
        -q|--quiet) CHATTY=less; shift ;;
        --) shift; break ;;
        -*) bad "Unrecognised option: $1" ;;
        *) break ;;
    esac
done

# Check basic expectations of context:
[ -d src ] || \
    die "I expect to be run in the top level directory of a module (see --help)."

GITMODULE=`/bin/pwd`
GITMODULE=`basename "$GITMODULE"`

THERE=`dirname $0`
[ -n "$THERE" -a -x "$THERE/resetboring.py" ] || \
    die "I don't know where resetboring.py is: please run me via an explicit path."
python3 -c 'from dulwich.repo import Repo; from dulwich.index import IndexEntry' || \
    die "I need dulwich installed (for resetboring.py; see --help)."
# dulwich 0.16.3 has been known to work; 0.9.4 is too old.

# Select revisions to compare:
[ $# -eq 2 ] || bad "Expected exactly two arguments, got $#: $@"
for arg
do git rev-parse "$arg" -- >/dev/null || bad "Failed to parse $arg as a git ref"
done
PRIOR="$1"
RELEASE="$2"
[ -n "$RELNAME" ] || RELNAME="$RELEASE"
RESTORE="`git branch | sed -n -e '/^\* (HEAD/ s/.* \([^ ]*\))$/\1/ p' -e '/^\*/ s/.* // p'`"

# Implement --verbose, --quiet:
mutter () { true; }
mention () { warn "$@"; }
# Option to pass to various git commands:
QUIET=
UNQUIET=-q
case "$CHATTY" in
    more) UNQUIET=
        mutter () { warn "$@"; }
        ;;
    less) QUIET=-q
        mention () { true; }
        ;;
    *) ;;
esac
run () { mutter "Running: $@"; eval "$@" || die "Failed ($?): $@"; }
logged () { mention "Running: $@"; eval "$@"; }
checkout () { run git checkout $QUIET "$@"; }

headers () {
    # usage: headers rev file module
    # If the file did not exist at the given revision, output is empty.
    git show "$1:$2" 2>/dev/null | $THERE/cmakescan.py "$3" "$2"
}

# Get API headers of $RELEASE checked out on a branch off $PRIOR:
BRANCH="api-review-$PRIOR-$RELNAME"
mutter "Checking for branch $BRANCH to check out"
case `git branch | grep -wF " $BRANCH" | grep "^[* ] $BRANCH"'$'` in
    '')
        checkout -b "$BRANCH" "$PRIOR"
        NEWBRANCH=yes
        if [ -n "$AMEND" ]
        then
            mention "Ignoring requested $AMEND: no prior $BRANCH"
            AMEND=
        fi
        ;;
    '* '*)
        case "$AMEND" in
            '--replace')
                mutter "On prior branch $BRANCH; shall be removed and recreated"
                ;;
            '--amend')
                mutter "Already on branch $BRANCH; preparing to amend it"
                ;;
            *)
                mutter "Already on branch $BRANCH; preparing to extend it"
                ;;
        esac
        ;;
    '  '*)
        case "$AMEND" in
            '--replace')
                mutter "Replacing existing branch $BRANCH (reusing its Change-Id)"
                ;;
            '--amend')
                mutter "Reusing existing branch $BRANCH; preparing to amend it"
                checkout "$BRANCH"
                ;;
            *)
                mutter "Reusing existing branch $BRANCH; preparing to extend it"
                checkout "$BRANCH"
                ;;
        esac
        ;;
esac

# Implement --replace and --amend:
CHANGEID=
if [ -n "$AMEND" ]
then
    # Suppress --amend or --replace unless we have a prior commit on $BRANCH:
    if git diff --quiet "$BRANCH" "$PRIOR"
    then
        mention "Suppressing requested $AMEND: no prior commit on $BRANCH"
        AMEND=
    else # Read (and validate) last commit's Change-Id:
        CHANGEID=`git show --summary $BRANCH | sed -ne '/Change-Id:/ s/.*: *//p'`
        [ -n "$CHANGEID" ] || die "No prior Change-Id from $BRANCH"
        expr "$CHANGEID" : "^I[0-9a-f]\{40\}$" >/dev/null || \
            die "Bad prior Change-Id ($CHANGEID) from $BRANCH"
        # Also preserve Task-number, if present and not over-ridden:
        [ -n "$TASK" ] || TASK=`git show --summary $BRANCH | sed -ne '/Task-number:/ s/.*: *//p'`
    fi
fi
if [ "$AMEND" = '--replace' ]
then
    AMEND=
    checkout "$RELEASE"
    run git branch -D "$BRANCH"
    checkout -b "$BRANCH" "$PRIOR"
fi
# Even when we do have a prior commit, the headers it reports as
# deleted are not actually deleted as part of that commit; so their
# deletion below shall ensure they're reported in the commit message,
# whether AMENDing or not.  We could filter these when not AMENDing,
# but (doing so would be fiddly and) any restored would then be
# described as deleted in the first commit's message, without
# mentioning that they're restored in the second (albeit any change in
# them shall show up in the diff).

# Make sure any sub-submodules are in their right states:
git submodule update --checkout

# Find a good place to prepare our commit messages
if [ -f .git ]
then GITDIR=`cut -d ' ' -f 2 <.git`
else GITDIR=.git
fi

# Suppress clang-format pre-commit hook and inane post-commit hook:
if [ -e $GITDIR/hooks/pre-commit ]
then
    PRECOMMIT=$GITDIR/hooks/isolated-pre-commit
    mv $GITDIR/hooks/pre-commit $PRECOMMIT

    if [ -e $GITDIR/hooks/post-commit ]
    then
        POSTCOMMIT=$GITDIR/hooks/isolated-post-commit
        mv $GITDIR/hooks/post-commit $POSTCOMMIT

        # But put it back when we're done:
        rehook () {
            mv $PRECOMMIT $GITDIR/hooks/pre-commit;
            mv $POSTCOMMIT $GITDIR/hooks/post-commit;
        }
        trap rehook EXIT
    else
        # But put it back when we're done:
        reclang () {
            mv $PRECOMMIT $GITDIR/hooks/pre-commit;
        }
        trap reclang EXIT
    fi
elif [ -e $GITDIR/hooks/post-commit ]
then
    POSTCOMMIT=$GITDIR/hooks/isolated-post-commit
    mv $GITDIR/hooks/post-commit $POSTCOMMIT

    # But put it back when we're done:
    resane () {
        mv $POSTCOMMIT $GITDIR/hooks/post-commit;
    }
    trap resane EXIT
fi

# Note: the loop is in a sub-shell, so can't change script variables.
# It can, however, write to local temp files:
PROC="$$-$GITMODULE"
COMMITTED="../.committed-$PROC"
BORED="../.bored-$PROC"
# For each module, create a commit:
modules | while read QTMODULE CMakeList
do
    mutter "Purging obsolete headers for $QTMODULE"
    # To ensure renames get detected and handled correctly:
    headers "$PRIOR" $CMakeList $QTMODULE | \
        while read h # Update former API headers, remove them if removed:
        do git checkout $UNQUIET "$RELEASE" -- "$h" || git rm $UNQUIET -f -- "$h"
        done 2>&1 | grep -wv "error: pathspec '.*' did not match any"
    mutter "Checking out $RELNAME's API headers for $QTMODULE"
    headers "$RELEASE" $CMakeList $QTMODULE \
        | tr '\n' '\0' | xargs -0r git checkout "$RELEASE" -- \
        || die "Failed new header checkout"

    if git diff --quiet --cached
    then
        mutter "No changes to $QTMODULE API (not even boring ones)"
        continue
    fi

    mutter "Reverting the boring changes to $QTMODULE"
    run "$THERE/resetboring.py" --disclaim | while read f
    do git checkout $QUIET "$PRIOR" -- "$f" || logged rm -f "$f"
    done

    if git diff --quiet --cached
    then
        echo "$QTMODULE" >> "$BORED"
        continue
    fi

    ( echo "$QTMODULE: API comparison from $PRIOR to $RELNAME in $GITMODULE"
      echo
      git status | grep 'deleted:' | tr '\t' ' '
      git diff | wc | python3 -c 'import sys; \
          row = sys.stdin.readline().split(); \
          print(); \
          print("Excluded {} lines ({} words, {} bytes) of boring changes.".format(*row) \
                if any(int(x) != 0 for x in row) else "Found nothing boring to ignore.")'
      cat <<EOF

NOTE: Please review the appropriate documentation and make sure it is
correct, too!

Note to reviewers: if you know of other changes to API, not covered by
this review, fetch this from Gerrit - see its checkout instructions;
you might want to add -b $BRANCH to those - and you
can add files to the result.  This can be relevant for behavior
changes in the .cpp or, for changes to documented behavior, in .qdoc
files.

Just git checkout $RELEASE each relevant file; you can then git reset
HEAD each and git add -p to select the relevant changes or use git
reset -p HEAD to unstage what git checkout has staged.  Try not to
include extraneous (non-API) changes.  Once you've added what matters,
git commit --amend and push back to gerrit's refs/for/$RELEASE in the
usual way.

Between staging changes from $RELEASE and committing, you can filter
some boring changes by running qtqa/scripts/api-review/resetboring.py
in the top-level directory of your module; this selectively de-stages
things commonly deemed boring in review.  (You'll need the python
dulwich package installed, for this.)  You can git add -p after that,
of course, to restore any changes you think aren't boring.

Remember that the parent of this commit is $PRIOR, not $RELEASE, despite
the review being sent to the latter in Gerrit.
EOF
      [ -z "$CHANGEID$TASK" ] || echo
      [ -z "$TASK" ] || echo "Task-number: $TASK"
      [ -z "$CHANGEID" ] || echo "Change-Id: $CHANGEID"
    ) > "$GITDIR/COMMIT_EDITMSG"
    # The git status in that holds a lock that precludes the git commit;
    # so we can't just pipe the message and use -F - to deliver it.
    mention "Committing change for $QTMODULE"
    run git commit $QUIET $AMEND -F "$GITDIR/COMMIT_EDITMSG"

    echo "$QTMODULE" >> "$COMMITTED"
done

consume () {
    if [ -f "$1" ]
    then
        cat "$1"
        rm "$1"
    fi
}
COMMITS="`consume "$COMMITTED"`"
BORES="`consume "$BORED"`"

if [ -n "$COMMITS" ]
then
   mention "I recommend you review that what git diff -D reports (now) *is* boring."
   mention "(If any of it isn't, you can git add -p it and git commit --amend, during a rebase.)"
   mention "Then you can: git push gerrit $BRANCH:refs/for/$RELEASE%topic=api-change-review-$RELNAME"
   [ -n "$TASK" ] || warn "Warning: no Task-number: footer specified."
elif [ -n "$BORES" ]
then mention "All the change here looks boring: check with git diff -D"
else
    mutter "Clearing away unused branch and restoring $RESTORE"
    git checkout $QUIET "$RESTORE"
    git branch -D "$BRANCH"
    mention "No changes to API"
fi