path: root/modularize

#!/usr/bin/perl -w
####################################################################################################
#
# Modularization script for Qt
#
# Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
# Contact: Nokia Corporation (qt-info@nokia.com)
#
####################################################################################################

use Config;
use File::Basename;
use File::Path;
use Cwd;
use Cwd 'abs_path';
use strict;

use constant {
    WINDOWS => 0,
    UNIX    => 1,
    MAC     => 2,
};

use constant {
    COPY      => 0,
    SYMLINKS  => 1,
    HARDLINKS => 2,
};

# Msys Perl uses unix paths, but $0 is a Windows path
my $fp0 = $0;
if ($Config{'osname'} =~ /msys/i && $fp0 =~ /^.:/) {
    $fp0 =~ s,:,,;
    $fp0 =~ s,\\,/,g;
    $fp0 = "/" . lcfirst($fp0);
}
our $basepath = dirname(abs_path($fp0));
$basepath =~ s,\\,/,g;
push(@INC, "$basepath/scripts");

# These SHA1s represents current "good" SHA1s for the two branches.
# Using option -bleeding will avoid these, and just try to run on the current branch you're in
# if that is successful, you should update the SHA1 below to reflect that.
my $good_47_sha1 = '770d5468a87c84cc435eefaa2a234f8c3b87f8c2';
my $good_master_sha1 = '896db169ea224deb96c59ce8af800d019de63f12';

our @repos = ('qtbase', 'qtsvg', 'qtdeclarative', 'qt3support', 'qtactiveqt',
              'qtscript', 'qtphonon', 'qtmultimedia', 'qttools', 'qtxmlpatterns',
              'qttranslations', 'qtdoc', 'qlalr', 'qtrepotools', 'qtwebkit-examples-and-demos',
              'qtqa'
             );

# Whether we are using qtwebkit or webkit trunk.
our $qtWebKitType = "qtwebkit";
our $qtWebKitCloneUrl;
our $qtWebKitCheckoutRef;

if ($qtWebKitType =~ /^webkit$/) {
    $qtWebKitCloneUrl = "git://git.webkit.org/WebKit.git";
    $qtWebKitCheckoutRef = "f1bfaac6bd42996c6a41ad2174b79ae200652759"; # master
} else {
    $qtWebKitCloneUrl = "git://gitorious.org/webkit/qtwebkit.git";
    $qtWebKitCheckoutRef = "f1bfaac6bd42996c6a41ad2174b79ae200652759"; #"refs/remotes/origin/qt-modularization-base";
}


our $OStype;
our $verbose;
our $run_split = 1;
our $run_rest = 1;
our $run_push = 0;
our $commit_changes = 1;
our $redo = 0;
our $move = 0;
our $move_dest = "";
our $repos_base;
our $good_sha1;
our $link_type = COPY;
our $qtdir = getcwd();
our $qtVersion;
our $isMaster = 1;
our $zeroMoveGraft = 0;

my $regenerate_patches = 0;
my $rebase_patches = 0;
my $no_patches = 0;
my $apply_afterlive = 0;


# -- Functions -------------------------------------------------------------------------------------
sub checkOS {
    my $r = WINDOWS;
    $r = UNIX if ((-f "/bin/uname" && !-f "\\bin\\uname")
                  || (-f "/usr/bin/uname" && !-f "\\usr\\bin\\uname"));
    if($r) {
        $_ = $Config{'osname'};
        $r = WINDOWS if (/cygwin|mswin|msys/i);
        $r = MAC if (/darwin/);
    }
    return $r;
}

# Syntax:  fileContents(filename)
# Returns: String with contents of the file, or empty string if file
#          doens't exist.
sub fileContents {
    my ($filename) = @_;
    my $filecontents = "";
    if (-e $filename) {
        open(I, "< $filename") || die "Could not open $filename for reading, read block?";
        local $/;
        binmode I;
        $filecontents = <I>;
        close I;
    }
    return $filecontents;
}

# Do not call this function directly, use either run() or runNotDie() instead
sub runBare {
    my ($cmd, $file, $line) = @_;

    print ' --> '.basename($file).":$line\t$cmd\n" if $verbose;

    system("$cmd");

    if ($? == -1) {
        printf "'%s' failed to execute ($!), at %s:%s\n", $cmd, $file, $line;
    } elsif ($? & 127) {
        printf "'%s' died with signal %d, %s coredump, at %s:%s\n", $cmd, $? & 127, ($? & 128) ? 'with' : 'without', $file, $line;
    } elsif ($? >> 8) {
        printf "'%s' exited with value %d, at %s:%s\n", $cmd, ($? >> 8), $file, $line;
    }
    return $?;
}

sub runNotDie {
    my ($cmd) = @_;
    my ($file, $line) = (caller)[1,2];
    return runBare($cmd, $file, $line);
}

sub run {
    my ($cmd) = @_;
    my ($file, $line) = (caller)[1,2];
    my $ret = runBare($cmd, $file, $line);
    exit ($ret >> 8) if $ret;
    return 0;
}

sub fsCopy {
    my @args = @_;
    if ($OStype == WINDOWS) {
        foreach (@args) {
            s,/,\\,g;
            s,-R,/s,g;
            s,-r,/s /e /i /c,g;
        }
        run("xcopy /Q " . join(" ", @args));
    } else {
        run("cp " . join(" ", @args));
    }
}

sub fsMove {
    my @args = @_;
    if ($OStype == WINDOWS) {
        foreach (@args) {
            s,/,\\,g;
        }
        run("move " . join(" ", @args));
    } else {
        run("mv " . join(" ", @args));
    }
}

sub fsRm {
    unlink($_[0]) or die("Could not remove $_[0]");
}

sub fsRmdir {
    rmdir($_[0]) or die("Could not remove $_[0]");
}

sub fsRmdirWithParents {
    while ($_[0]) {
        my $arg = shift;
        do {
            rmdir($arg) or die("Could not remove $arg");
        } while ($arg =~ s,/[^/]*$,,);
    }
}

# Makes sure the arguments are directories, and creates them if not.
# Will die if there is an error.
sub ensureDir {
    foreach (@_) {
        if (-e $_) {
            if (-d $_) {
                next;
            } else {
                die ("$_ exists, but is not a directory");
            }
        }
        File::Path::mkpath($_) or die("Could not create $_");
    }
}

# list<string> findFiles(string dir, regexp match, bool descend)
#
# Returns matching filenames. If descend is true, returns relative paths,
# if descend is false, returns plain filenames.
sub findFiles {
    my ($dir,$match,$descend) = @_;
    my ($file,$p,@files);
    local(*D);
    $dir =~ s=\\=/=g;
    ($dir eq "") && ($dir = ".");
    if ( opendir(D,$dir) ) {
        if ( $dir eq "." ) {
            $dir = "";
        } else {
            ($dir =~ /\/$/) || ($dir .= "/");
        }
        foreach $file ( sort readdir(D) ) {
            next if ( $file  =~ /^\.\.?$/ );
            $descend and $p = "$dir$file" or $p = "$file";
            ($file =~ /$match/) && (push @files, $p);
            if ( $descend && -d $p && ! -l $p ) {
                push @files, &findFiles($p,$match,$descend);
            }
        }
        closedir(D);
    }
    return @files;
}

#first argument is a directory, set second argument to true of it has to be ordered
sub createSubdirProfile {
    my $dir = $_[0];
    my $ordered = $_[1];
    die ("$dir is not a directory") if (!-d $dir);
    my @D;
    my $subdirs = "";
    if (opendir(DIR, $dir)) {
        @D = readdir(DIR);
        closedir(DIR);
        foreach my $file (@D) {
            next if ($file  =~ /^\.\.?$/);
            if (-d "$dir/$file") {
                if (-e "$dir/$file/$file.pro") {
                    if ($file =~ /^(src|svg|script|declarative)$/) {
                        $subdirs = "$file $subdirs";
                    } else {
                        $subdirs = "$subdirs $file";
                    }
                }
            }
        }
    }

    my $base = basename($dir);
    open PRO, ">$dir/$base.pro" || die "Could not open $dir/$base.pro for writing!\n";
    print PRO "TEMPLATE = subdirs\n";
    print PRO "CONFIG += ordered\n" if $ordered;

    print PRO "SUBDIRS += $subdirs\n";
    close PRO;
    run("git add $dir/$base.pro");
}

# Figure out which branch of Qt we're on
sub findBranchVersion
{
    my ($qtdir) = @_;
    my $qglobal = fileContents("$qtdir/src/corelib/global/qglobal.h");
    $qglobal = fileContents("$qtdir/qtbase/src/corelib/global/qglobal.h") if (!$qglobal);
    return "" if (!$qglobal);
    $qglobal =~ s/.*define\s+QT_VERSION_STR\s*\"([0-9\.]*)\".*/$1/s;
    return $qglobal;
}

# Find SHA1 of commit of $subject, else the SHA1 of HEAD
sub findSHA1
{
    my ($qtdir, $subject) = @_;
    my @commits = `cd $qtdir && git rev-list --grep="$subject" HEAD`;
    if (@commits) {
        my $commit = $commits[0];
        chomp $commit;
        return $commit;
    }
    my $commit = `git rev-parse HEAD`;
    chomp $commit;
    return $commit;
}

# Find SHA1 of commit before $subject, else the SHA1 of HEAD
sub findSHA1Before
{
    my ($qtdir, $subject) = @_;
    my @commits = `cd $qtdir && git rev-list --grep="$subject" HEAD~300..HEAD`;
    if (@commits) {
        my $commit = $commits[0];
        chomp $commit;
        $commit = `git rev-parse $commit~`;
        chomp $commit;
        return $commit;
    }
    my $commit = `git rev-parse HEAD`;
    chomp $commit;
    return $commit;
}

# Checks if file has relevant changes to it, based on git diff output
# PS. uses CWD!
sub patchHasChanges
{
    my ($filename, $org) = @_;

    my @content;
    if ("$org" eq "") {
        @content = `git diff $filename` if (-e $filename);
    } else {
        @content = `diff -u $org $filename` if (-e $filename && -e $org);
    }
    return 0 if (!@content);

    # remove git version at the end of the patch
    pop(@content);
    pop(@content);
    pop(@content);

    # grep out useless content changes
    @content = grep(!/^[ @]/, @content);
    @content = grep(!/^(\+\+\+|---|(|\+|-)index|diff) /, @content);
    @content = grep(!/^[\+-]From [0-9a-z]{40} \w+ \w+ \d+ \d+:\d+:\d+ \d+/, @content);

    return !!@content;
}

sub applyAfterLivePatches {
    my @modules = sort(findFiles("$basepath/afterlive", '.*', 0));
    print "---\nApplying 'afterlive' patches for modules ". join(", ", @modules). "\n";
    $repos_base = $qtdir . "/qt" if (!defined $repos_base);
    foreach my $module (@modules) {
        chdir($qtdir);
        if (-e "$repos_base/$module") {
            print "Applying patches for '$module' @ '$repos_base/$module'\n";
            chdir("$repos_base/$module");
            my $commit = findSHA1("$repos_base/$module", 'Added dependency information to the sync.profile.');
            run("git reset -q --hard $commit");
            run("git am $basepath/afterlive/$module/*.patch");
        } else {
            print "** Cannot find $repos_base/$module\n";
        }
    }

    print "All 'afterlive' patches applied!\n";
}

sub showUsage {
    print "$0 usage:\n";
    print "  -no-modularize     Don't run the 0xx-8xx script range\n";
    print "  -no-split          Don't run the 9xx script range\n";
    print "  -modularized       Push the repos to server(will not work with -no-split, -hardlinks)\n";
    print "                     Note: need to have following in your ~/.ssh/config\n";
    print "                     SendEnv GIT_FORCE\n";
    print "                     SendEnv GIT_PUSH\n";
    print "  -no-commit         Do not commit the changes\n";
    print "  -move <dest>       Copy the new folder in the given destination \n";
    print "  -symlinks          Uses symlinks for the qt5 repo instead of repo copies\n";
    print "                     (Ignored when -move option is used, and on Windows)\n";
    print "  -hardlinks         Same as -symlinks, but uses hardlinks instead.\n";
    print "                     You MOST LIKELY want to use this option, since symlinks\n";
    print "                     will be resolved by qmake, and the actual location used.\n";
    print "  -zero-move-graft   Apply patches on the modularized repositories instead of the\n";
    print "                     old monolithic one, so that the graft will not see any file\n";
    print "                     moves (only works on Linux).\n";
    print "  -redo              Resets (hard) the repo, and cleans up (-dffx)\n";
    print "                     before running the scripts\n";
    print "  -experimental      Ignore the set \"good\" SHA1 for the current branch, and\n";
    print "                     instead use this branch's HEAD (ignoring potential\n";
    print "                     modularization patches)\n";
    print "  -rebase-patches    Automatically rebase all patches from current good SHA1\n";
    print "                     to current HEAD\n";
    print "  -regenerate-patches Automatically regenerate all the patches for the\n";
    print "                     modularization project (does not copy to the repo!)\n";
    print "  -afterlive         Apply 'afterlive' patches to modularized repos\n";
    print "  -redo-afterlive    Reapply 'afterlive' patches\n";
    print "  -?                 This help\n";
}
# -- End functions ---------------------------------------------------------------------------------

# -- main ------------------------------------------------------------------------------------------
$OStype = checkOS;
$qtVersion = findBranchVersion($qtdir);

while ( @ARGV ) {
    my $arg = shift @ARGV;
    if ($arg eq "-?" || $arg eq "-h" || $arg eq "-help" || $arg eq "?") {
        showUsage();
        exit 0;
    } elsif($arg eq "-no-modularize") {
        $run_rest = 0;
    } elsif($arg eq "-no-split") {
        $run_split = 0;
    } elsif($arg eq "-modularized") {
        $run_push = 1;
    } elsif($arg eq "-no-commit") {
        $commit_changes = 0;
    } elsif($arg eq "-no-patches") {
        # This option is only used internally by the -rebase-patches option
        $no_patches = 1;
    } elsif($arg eq "-redo") {
        $redo = 1;
    } elsif($arg eq "-verbose") {
        $verbose = 1;
    } elsif($arg eq "-symlinks") {
        $link_type = ($OStype == WINDOWS ? COPY : SYMLINKS);
    } elsif($arg eq "-hardlinks") {
        $link_type = HARDLINKS;
    } elsif($arg eq "-move") {
        if (@ARGV) {
            $move = 1;
            $move_dest = shift @ARGV;
            $redo = 1;
            $link_type = COPY;
        }
        else {
            print "Missing destination\n\n";
            showUsage();
            exit 1;
        }
    } elsif ($arg eq "-zero-move-graft") {
        $zeroMoveGraft = 1;
    } elsif ($arg eq "-experimental") {
        $good_sha1 = findSHA1Before($qtdir, "Commit of all changes done by the modularization script");
    } elsif ($arg eq "-rebase-patches") {
        $rebase_patches = 1;
    } elsif ($arg eq "-regenerate-patches") {
        $regenerate_patches = 1;
    } elsif ($arg eq "-afterlive") {
        $apply_afterlive = 1;
    } elsif ($arg eq "-redo-afterlive") {
        applyAfterLivePatches();
        exit 0;
    } else {
        print "Unknown option: $arg\n\n";
        showUsage();
        exit 1;
    }
}

if (!$good_sha1) {
    if ($qtVersion =~ /4.7/) {
        $good_sha1 = $good_47_sha1;
    } else {
        $good_sha1 = $good_master_sha1;
    }
}

if ($rebase_patches) {
    my $new_sha1 = findSHA1Before($qtdir, "Commit of all changes done by the modularization script");
    print "*** Rebasing patches mode **\n";
    print "*** QTDIR: $qtdir\n";
    print "*** Branch: $qtVersion\n";
    print "*** ---\n";
    printf "*** First running modularization for previous good SHA1:%s\n", substr($good_sha1, 0, 8);
    printf "*** Then rebasing patches ontop of new SHA1:%s\n\n", substr($new_sha1, 0, 8);

    # Modularize for old good sha1
    run("$0 -redo -no-split");

    # Modularize based on new HEAD, without patches
    run("git branch -f ${qtVersion}_mod");
    my $patch_start_sha1 = findSHA1Before($qtdir, "Some changes for qtbase");
    print "\n*** Good modularization branched ${qtVersion}_mod\n";
    printf "*** Good patch series start at SHA1:%s\n", substr($patch_start_sha1, 0, 8);
    printf "*** Now modularizing SHA1:%s without patches\n\n", substr($new_sha1, 0, 8);

    run("git reset --hard $new_sha1");
    run("git clean -dffx");
    run("$0 -redo -experimental -no-patches");

    # Rebase old patches ontop of new split
    printf "\n*** Rebasing old good patches (from SHA1:%s to ref:${qtVersion}_mod) onto HEAD\n\n", substr($patch_start_sha1, 0, 8);
    my $new_mod_sha1 = findSHA1Before($qtdir, "Commit of all changes done by the modularization script");
    run("git rebase --onto HEAD $patch_start_sha1 ${qtVersion}_mod");

    print "\n*** Rebasing patches done!\n";
}

if ($regenerate_patches) {
        my $masterPath = "";
        # $masterPath = "/master" if($isMaster);
        chdir("$basepath/patches$masterPath");

        # Check if we have uncommited patches first
        my @modifications = `git status --porcelain .`;
        if (@modifications) {
            print "$basepath/patches contains modifications. Will not clobber!\n";
            exit(1);
        }

        # Regenerate all patches
        my $patchesSHA1 = findSHA1Before($qtdir, "Some changes for qtbase");
        run("git --git-dir=$qtdir/.git format-patch -N $patchesSHA1");

        # Report updated patches, and revert "no change" patches
        chdir("$basepath");
        @modifications = `git status --porcelain patches/$masterPath`;
        foreach my $line (@modifications) {
            chomp $line;
            my $mode = substr($line, 0, 3);
            my $file = substr($line, 3);
            if ($mode eq " M ") {
                if (patchHasChanges($file, "")) {
                    print "Updated patch '$file'\n";
                    run("git add $file");
                } else {
                    runNotDie("git checkout $file");
                }
            } elsif ($mode eq "?? " && "$masterPath" ne "") {
                my $org = $file;
                $org =~ s/master\///;
                if (patchHasChanges($file, $org)) {
                    run("git add $file");
                    print "Added significantly different patch '$file'\n";
                } else {
                    fsRm($file);
                }
            } elsif ($mode eq "?? ") {
                run("git add $file");
                print "Added new patch '$file'\n";
            }
        }
        print "\nPatches regenerated.\n";
}

exit(0) if ($rebase_patches || $regenerate_patches);

print "QTDIR: $qtdir\n";
print "Branch: $qtVersion\n";
print "Scripts from: $basepath\n";
print "Running on: " . ($OStype == WINDOWS ? "Windows" : ($OStype == UNIX ? "Unix" : "Mac")) . "\n";

if ($redo) {
    print "---\n";
    printf "Resetting repository (SHA1:%s), and removing residue files\n", substr($good_sha1, 0, 8);
    run("git reset --hard $good_sha1");
    run("git clean -dffx");
    if ($move) {
        if (-e $move_dest) {
            rmtree($move_dest, 0, 0) || warn "rmtree $move_dest: $!";
        }
    }
    if (-e ".git/rebase-apply") {
        rmtree(".git/rebase-apply", 0, 0) || warn "rmtree .git/rebase-apply: $!";
    }
    print "---\n";
}

my @scripts;
push @scripts, sort(findFiles("$basepath/scripts", '^[0-7]\d+_.*[^~]', 0)) if ($run_rest);
my @scripts_generated_commits;
push @scripts_generated_commits, sort(findFiles("$basepath/scripts", '^8\d+_.*[^~]', 0)) if ($run_rest);
my @scripts_split;
push @scripts_split, sort(findFiles("$basepath/scripts", '^9\d+_.*[^~]', 0)) if ($run_split);
print "Scripts to execute (in order):\n    ".join(", ", (@scripts, @scripts_generated_commits))."\n";

# Remember that each script need to end with a "return 1;" on success, and "return 0;" when unsuccessful
foreach my $script (@scripts) {
    print "--> $script\n";
    chdir($qtdir);
    require("$basepath/scripts/$script");
}
chdir($qtdir);

fsCopy("$basepath/files/qtsvg.pro", "qtsvg/");
fsCopy("$basepath/files/qttools.pro", "qttools/");
fsCopy("$basepath/files/qtdeclarative.pro", "qtdeclarative/");
fsCopy("$basepath/files/qtscript.pro", "qtscript/");
fsCopy("$basepath/files/qt3support.pro", "qt3support/");
fsCopy("$basepath/files/qtmultimedia.pro", "qtmultimedia/");
fsCopy("$basepath/files/qtxmlpatterns.pro", "qtxmlpatterns/");

if ($commit_changes) {
    print "Committing all changes done by the modularization script\n";
    run("git add .");
    if ($zeroMoveGraft) {
        # run("git commit -a -F $basepath/misc/import-commit-msg --author \"Qt by Nokia <qt-info\@nokia.com>\" -q");
        my $GIT_COMMIT;
        open(GIT_COMMIT, "| git commit -a -q --author \"Qt by Nokia <qt-info\@nokia.com>\" -F -") or die "Couldn't fork: $!\n";;
        print GIT_COMMIT fileContents("$basepath/misc/import-commit-msg") . $good_sha1;
        close(GIT_COMMIT) or die "Couldn't close: $!\n";;
    } else {
        run("git commit -a -q -m \"Commit of all changes done by the modularization script" .
            "\n\nBranched from monolithic repo, Qt master branch, at commit\n$good_sha1\"");
    }
}

# Run scripts that generate extra commits.
# Remember to check $commit_changes inside the script.
if ($run_rest) {
    foreach my $script (@scripts_generated_commits) {
        print "--> $script\n";
        chdir($qtdir);
        require("$basepath/scripts/$script");
    }
}

# This is only used internally by the -rebase-patches option
exit(0) if ($no_patches);

if ($commit_changes) {
    # Applying patches
    print "Applying patches\n";
    chdir($qtdir);
    run("git am --ignore-whitespace $basepath/patches/*.patch") if ($run_rest);
} else {
    my @patches = sort(findFiles("$basepath/patches", '^\d+-.*\.patch', 0));
    foreach my $patch (@patches) {
        run("patch -p1 < $basepath/patches/$patch");
    }
}

chdir($qtdir);

#split the repository
if($run_split) {
    foreach my $script (@scripts_split) {
        print "--> $script\n";
        chdir($qtdir);
        require("$basepath/scripts/$script");
    }
}
chdir($qtdir);

applyAfterLivePatches() if ($apply_afterlive);

print "===\n";
print "Modularization done!\n";