jenkins: add log synchronization support

Allow qt-jenkins-ci.pl to put compressed logs on another host when a
build has completed, and allow summarize-jenkins-build.pl to generate
links referring to that host.

This supports a setup where the Jenkins build data may be deleted
without notice, or where the Jenkins server is not publicly accessible.

Change-Id: I48a66866ad56fa7df74b190927b192c9f4cb22df
Reviewed-by: Kalle Lehtonen <kalle.ju.lehtonen@nokia.com>

5 files changed, 675 insertions, 43 deletions

diff --git a/scripts/jenkins/qt-jenkins-ci.pl b/scripts/jenkins/qt-jenkins-ci.pl
index 1cfcfed0..4df30e4e 100755
--- a/scripts/jenkins/qt-jenkins-ci.pl
+++ b/scripts/jenkins/qt-jenkins-ci.pl
@@ -88,6 +88,43 @@ The Jenkins build status determines the gerrit build status (pass or fail).
 
 A summary of the build, and links to build logs, will be pasted as gerrit comment(s).
 
+Options:
+
+=over
+
+=item --log-upload-url <url>
+
+=item --log-download-url <url>
+
+Upload and download URLs for build log synchronization.
+
+If these options are specified, the raw build logs from Jenkins are uploaded to the given
+upload URL, and any links to build logs used in gerrit comments will refer to the given
+download URL.
+
+Currently, the upload URL must be an ssh URL, and the download URL must be an http URL.
+
+For working uploads, the host running this script must have passwordless ssh access to
+the remote host. wget must be installed on the remote host.
+
+The given URLs should refer to a directory under which this script will create a new
+directory for each project and build.  For example:
+
+  --log-upload-url ssh://testresults.qt-project.org/var/www/ci
+  --log-download-url http://testresults.qt-project.org/ci
+
+... would upload logs to /var/www/ci/<project>/build_<build_number>/<cfg>/log.txt.gz
+on the named host, and post HTTP links of the same structure in the gerrit comments.
+
+The purpose of these options is to facilitate a setup where Jenkins builds are considered
+volatile and may be cleaned up regularly, but compressed build/test logs are kept
+"permanently" on some other simple web host.
+
+If these options are omitted, no log uploads occur, and gerrit comments will refer
+directly to Jenkins logs.
+
+=back
+
 =back
 
 =head2 GLOBAL OPTIONS
@@ -339,9 +376,12 @@ package QtQA::App::JenkinsCI;
 use strict;
 use warnings;
 
+use AnyEvent::HTTP;
 use AnyEvent::Util;
 use AnyEvent;
 use Carp qw( confess );
+use Coro;
+use Coro::AnyEvent;
 use Data::Dumper;
 use English qw( -no_match_vars );
 use File::Basename;
@@ -413,25 +453,32 @@ sub fetch_json_data
 #     within a certain amount of time (overridable by 'timeout' option,
 #     in seconds). When this occurs, the exit status is -1
 #
-#   - the $$ option may not be passed into this function, since it
-#     is already used internally.
-#
 sub run_timed_cmd
 {
     my ($cmd, %options) = @_;
 
     my $timeout = delete $options{ timeout } || 60*15;
 
+    # We need to know the pid, but note the caller might have asked for it too.
     my $pid;
+    my $pid_ref = delete( $options{ '$$' } ) || \$pid;
 
     # command may exit normally or via timeout
-    my $cv = run_cmd( $cmd, %options, '$$' => \$pid );
-    my $timer = AnyEvent->timer( after => $timeout, cb => sub {
-        local $LIST_SEPARATOR = '] [';
-        warn "command [@{ $cmd }] timed out after $timeout seconds\n";
+    my $cv = run_cmd( $cmd, %options, '$$' => $pid_ref );
+    my $timer;
+    $timer = AnyEvent->timer( after => $timeout, cb => sub {
+        if (!$cv->ready()) {
+            # timer expired and process is not yet finished
+            local $LIST_SEPARATOR = '] [';
+            warn "command [@{ $cmd }] timed out after $timeout seconds\n";
+
+            kill( 15, $$pid_ref );
+            $cv->send( -1 );
+        }
 
-        kill( 15, $pid );
-        $cv->send( -1 );
+        # doing explicit undef of $timer here ensures that the timer
+        # object is kept alive until it fires
+        undef $timer;
     });
 
     return $cv;
@@ -517,6 +564,105 @@ sub safe_qx
     return $stdout;
 }
 
+# Upload a single log from an HTTP URL via ssh.
+# Expected to be run from within a coro.
+#
+# Arguments:
+#
+#   ssh_command => arrayref, the ssh command to run. Should read log data from stdin.
+#   url => the log url
+#
+# The upload will be retried a few times if either the ssh or http connections
+# experience an error.
+sub upload_http_log_by_ssh
+{
+    my (%args) = @_;
+
+    my @cmd = @{ $args{ ssh_command } };
+    my $url = $args{ url };
+
+    my $retry = 8;
+    my $sleep = 1;
+
+    while ($retry) {
+        my ($r, $w);
+        pipe( $r, $w) || die "pipe: $!";
+
+        my $ssh_pid;
+        my $ssh_cv = run_timed_cmd(
+            \@cmd,
+            '<' => $r,
+            '$$' => \$ssh_pid,
+        );
+
+        # cv receives nothing on success, error type and details on ssh or http error.
+        my $cv = AnyEvent->condvar();
+        $ssh_cv->cb( sub {
+            my $status = $ssh_cv->recv();
+            $cv->send( ($status == 0) ? () : ('ssh', $status) );
+        });
+
+        # check http headers and fail if anything other than '200 OK'
+        my $check_headers = sub {
+            my $h = shift;
+            if ($h->{ Status } != 200) {
+                my $status = $h->{ OrigStatus } || $h->{ Status };
+                my $reason = $h->{ OrigReason } || $h->{ Reason };
+                $cv->send( 'http', "fetching $url: $status $reason" );
+                return 0;
+            }
+            return 1;
+        };
+
+        my $req = http_get(
+            $url,
+            on_body => sub {
+                my ($data, $headers) = @_;
+                return unless $check_headers->( $headers );
+                # all HTTP data is piped to ssh STDIN
+                print $w $data;
+                return 1;
+            },
+            sub {
+                my (undef, $headers) = @_;
+                return unless $check_headers->( $headers );
+                close( $w ) || $cv->send( 'ssh', $! );
+            },
+        );
+
+        my (@error) = $cv->recv();
+        if (!@error) {
+            # all done
+            return;
+        }
+
+        # something bad happened
+        my $type = shift @error;
+        my $error_str = "$type error: @error";
+
+        # if we stopped due to an http error, make sure to kill the ssh
+        if ($type eq 'http') {
+            kill( 15, $ssh_pid ) if $ssh_pid;
+        }
+
+        # we will retry on _any_ http error, or on ssh exit code 255 (network error)
+        if ($type eq 'http' || ($type eq 'ssh' && ($error[0] >> 8) == 255)) {
+            warn "$error_str\n  Trying again in $sleep seconds\n";
+            --$retry;
+            Coro::AnyEvent::sleep( $sleep );
+            $sleep *= 2;
+            next;
+        }
+
+        # any other kind of error is considered fatal
+        die "$error_str\n";
+    }
+
+    # if we get here, we never succeeded.
+    local $LIST_SEPARATOR = '] [';
+    die "HTTP fetch $url to ssh command [@cmd] repeatedly failed, giving up.\n";
+}
+
 # ======================= instance ============================================
 
 # Remove any old qt-ci-.*.properties file from $dir.
@@ -819,6 +965,16 @@ sub jenkins_build_summary
     if (!$self->{ cfg }{ jenkins_build_summary }) {
         my @cmd = ($SUMMARIZE_JENKINS_BUILD, '--url', $self->jenkins_build_url( ));
 
+        if (my $arg = $self->{ log_download_url }) {
+            push @cmd, '--log-url', $arg;
+        }
+        if (my $arg = $self->{ force_jenkins_host }) {
+            push @cmd, '--force-jenkins-host', $arg;
+        }
+        if (my $arg = $self->{ force_jenkins_port }) {
+            push @cmd, '--force-jenkins-port', $arg;
+        }
+
         my $summary;
         my $status = run_timed_cmd(
             \@cmd,
@@ -911,6 +1067,102 @@ sub do_staging_approve
     return;
 }
 
+# Upload all logs to $url base, or die on error.
+sub upload_logs
+{
+    my ($self, $url) = @_;
+
+    my $result = $self->jenkins_build_result( );
+
+    my $parsed_url = URI->new( $url );
+    if ($parsed_url->scheme() ne 'ssh') {
+        confess "unsupported URL, only ssh URLs are supported: $url";
+    }
+
+    # Figure out the list of target and source URLs
+    my $build_number = $self->jenkins_build_number( );
+    my $project_name = $self->jenkins_job_name( );
+    my $build_url = $self->jenkins_build_url( );
+
+    my $dest_project_name = $project_name;
+    $dest_project_name =~ s{ }{_}g;
+
+    my $dest_project_path = catfile( $parsed_url->path(), $dest_project_name );
+    my $dest_build_number = sprintf( 'build_%05d', $build_number );
+    my $dest_build_path = catfile( $dest_project_path, $dest_build_number );
+
+    my %to_upload;
+
+    foreach my $config ($self->jenkins_configurations( )) {
+        # If $config only has one axis (the normal case), just use it directly,
+        # to avoid useless 'cfg=' in URLs.
+        my $dest_config = $config;
+        $dest_config =~ s{\A [^=]+ = ([^=]+) \z}{$1}xms;
+        $dest_config =~ s{ }{_}g;
+
+        my $src_url = "$build_url/$config/consoleText";
+        my $dest_path = catfile( $dest_build_path, $dest_config, 'log.txt.gz' );
+
+        $to_upload{ $src_url } = $dest_path;
+    }
+
+    # And also sync the master log ...
+    $to_upload{ "$build_url/consoleText" } = catfile( $dest_build_path, 'log.txt.gz' );
+
+    my @ssh_base = (
+        'ssh',
+        '-oBatchMode=yes',
+        '-p', $parsed_url->port( ),
+        ($parsed_url->user( )
+            ? ($parsed_url->user( ) . '@' . $parsed_url->host( ))
+            : $parsed_url->host( )
+        )
+    );
+
+    my @coro;
+    while (my ($src, $dest) = each %to_upload) {
+        my $dir = dirname( $dest );
+        my @command = (
+            @ssh_base,
+            qq{mkdir -p "$dir" && cd "$dir" && }
+           .qq{gzip > .incoming.log.txt.gz && }
+           .qq{mv .incoming.log.txt.gz log.txt.gz}
+        );
+        push @coro, async {
+            my $host = $parsed_url->host();
+            my $thing = "Upload $src -> $dest (on $host)";
+            eval {
+                upload_http_log_by_ssh(
+                    ssh_command => \@command,
+                    url => $src,
+                );
+            };
+            if (my $error = $EVAL_ERROR) {
+                die "$thing: $error";
+            }
+            print "$thing: OK!\n";
+        };
+    }
+
+    map { $_->join() } @coro;
+
+    # Create the 'latest' and possibly 'latest-success' links
+    my $cmd = qq{cd "$dest_project_path" && ln -sf "$dest_build_number" latest};
+    if ($result eq 'SUCCESS') {
+        $cmd .= qq{ && ln -sf "$dest_build_number" latest-success};
+    }
+
+    do_robust_cmd(
+        cmd => [
+            @ssh_base,
+            $cmd,
+        ],
+        retry_exitcodes => [255],
+    );
+
+    return;
+}
+
 # Entry point of 'new_build'
 sub command_new_build
 {
@@ -935,7 +1187,19 @@ sub command_new_build
 # Entry point of 'complete_build'
 sub command_complete_build
 {
-    my ($self) = @_;
+    my ($self, @args) = @_;
+
+    my $log_upload_url;
+
+    GetOptionsFromArray(
+        \@args,
+        'log-upload-url=s' => \$log_upload_url,
+        'log-download-url=s' => \$self->{ log_download_url },
+    );
+
+    if ($log_upload_url) {
+        $self->upload_logs( $log_upload_url );
+    }
 
     $self->do_staging_approve();
 
diff --git a/scripts/jenkins/summarize-jenkins-build.pl b/scripts/jenkins/summarize-jenkins-build.pl
index 1667614d..0c5db237 100755
--- a/scripts/jenkins/summarize-jenkins-build.pl
+++ b/scripts/jenkins/summarize-jenkins-build.pl
@@ -77,6 +77,37 @@ URL of the Jenkins build.
 For a multi-configuration build, the URL of the top-level build should be used.
 The script will parse the logs from each configuration.
 
+=item --log-base-url B<base>
+
+Base URL of the build logs.
+
+Optional; if set, build logs will be fetched from URLs under this path, instead
+of directly from Jenkins. This may be used to support a setup where Jenkins
+build logs are volatile, subject to removal without notice, and the logs are
+primarily accessed from another server.
+
+The URLs constructed using B<base> are currently not customizable, and always
+use the following pattern:
+
+  <base>/<project_name>/build_<five_digit_build_number>/<configuration>/log.txt.gz
+
+If build logs cannot be fetched from this URL for any reason, the logs are parsed
+directly from Jenkins. However, any user-visible links will still refer to the
+URL passed in this option.
+
+Note that the build status is still fetched directly from Jenkins.
+
+=item --force-jenkins-host <HOSTNAME>
+
+=item --force-jenkins-port <PORTNUMBER>
+
+When fetching any data from Jenkins, disregard the host and port portion
+of Jenkins URLs and use these instead.
+
+This is useful for network setups (e.g. port forwarding) where the Jenkins
+host cannot access itself using the outward-facing hostname, or simply to
+avoid unnecessary round-trips through a reverse proxy setup.
+
 =item --debug
 
 Print an internal representation of the build to standard error, for debugging
@@ -95,6 +126,7 @@ use Getopt::Long qw(GetOptionsFromArray);
 use JSON;
 use Pod::Usage;
 use Readonly;
+use URI;
 
 # Jenkins status constants
 Readonly my $SUCCESS => 'SUCCESS';
@@ -141,55 +173,135 @@ sub run_parse_build_log
     return system( $PARSE_BUILD_LOG, '--summarize', $url );
 }
 
-# Returns the output of "parse_build_log.pl --summarize $url",
-# or warns and returns nothing on error.
+# Returns the output of parse_build_log.pl on the first
+# working of the given @url, or warns and returns nothing
+# if all URLs are not usable.
 sub get_build_summary_from_log_url
 {
-    my ($url) = @_;
+    my (@url) = @_;
 
-    return unless $url;
+    return unless @url;
+
+    my $stdout;
 
-    my $status;
-    my ($stdout, $stderr) = capture {
-        $status = run_parse_build_log( $url );
-    };
+    while (!$stdout && @url) {
+        my $url = shift @url;
+        my $stderr;
+        my $status;
+        ($stdout, $stderr) = capture {
+            $status = run_parse_build_log( $url );
+        };
 
-    chomp $stdout;
+        chomp $stdout;
 
-    if ($status != 0) {
-        warn "parse_build_log exited with status $status"
-            .($stderr ? ":\n$stderr" : q{})
-            ."\n";
+        if ($status != 0) {
+            warn "for $url, parse_build_log exited with status $status"
+                .($stderr ? ":\n$stderr" : q{})
+                ."\n";
 
-        # Output is not trusted if script didn't succeed
-        undef $stdout;
+            # Output is not trusted if script didn't succeed
+            undef $stdout;
+        }
     }
 
     return $stdout;
 }
 
-# Given a Jenkins build object, returns a "permanent" link
-# to the build log (which may itself not be in jenkins).
-sub get_permanent_url_for_build_log
+# Given a Jenkins build object, returns one or more links
+# to the build logs, in order of preference.
+sub get_url_for_build_log
 {
-    my ($cfg) = @_;
+    my ($self, $cfg, $log_base_url) = @_;
 
-    # FIXME: support testresults.qt-project.org logs
     my $url = $cfg->{ url };
     return unless $url;
 
+    my @out;
+
+    if ($log_base_url) {
+        if ($url =~
+            m{
+                \A
+                .+
+                /job/
+                (?<job>
+                    [^/]+   # job name
+
+                )
+                /
+                (?:
+                    # jenkins sometimes introduces a useless './' into the URLs
+                    \./
+                )*
+                (?:
+                    # configuration part is optional;
+                    # it is not present if the failure was on the master, for instance.
+                    (?<cfg>
+                        [^/]+
+                    )
+                    /
+                )?
+                (?<build>
+                    [0-9]+
+                )
+                /?
+                \z
+            }xms
+        ) {
+            my ($job_name, $cfg_name, $build_number) = @+{ 'job', 'cfg', 'build' };
+            if ($cfg_name) {
+                # If $cfg_name only has one axis (the normal case), just use it directly,
+                # to avoid useless 'cfg=' in URLs.
+                $cfg_name =~ s{\A [^=]+ = ([^=]+) \z}{$1}xms;
+                $cfg_name =~ s{ }{_}g;
+            }
+            push @out, sprintf( '%s/%s/build_%05d%s/log.txt.gz', $log_base_url, $job_name, $build_number, $cfg_name ? "/$cfg_name" : q{});
+        } else {
+            warn "URL '$url' not of expected format, cannot rebase to $log_base_url\n";
+        }
+    }
+
+    # Use direct Jenkins logs
     if ($url !~ m{/\z}) {
         $url .= '/';
     }
 
-    return $url . 'consoleText';
+    push @out, $self->maybe_rewrite_url( $url . 'consoleText' );
+    return @out;
+}
+
+# Returns a version of $url possibly with the host and port replaced, according
+# to the --force-jenkins-host and --force-jenkins-port command-line arguments.
+sub maybe_rewrite_url
+{
+    my ($self, $url) = @_;
+
+    if (!$self->{ force_jenkins_host } && !$self->{ force_jenkins_port }) {
+        return $url;
+    }
+
+    my $parsed = URI->new( $url );
+    if ($self->{ force_jenkins_host }) {
+        $parsed->host( $self->{ force_jenkins_host } );
+    }
+    if ($self->{ force_jenkins_port }) {
+        $parsed->port( $self->{ force_jenkins_port } );
+    }
+
+    return $parsed->as_string();
 }
 
 # Given a jenkins build $url, returns a human-readable summary of
 # the build result.
+#
+# If $log_base_url is set, log URLs are derived under that base
+# URL using a predefined pattern, with Jenkins logs used as a fallback.
+# Otherwise, the logs are used directly from Jenkins.
 sub summarize_jenkins_build
 {
-    my ($self, $url) = @_;
+    my ($self, $url, $log_base_url) = @_;
+
+    $url = $self->maybe_rewrite_url( $url );
 
     my $build = get_build_data_from_url( $url );
 
@@ -230,11 +342,11 @@ sub summarize_jenkins_build
     my @summaries;
 
     foreach my $cfg (@configurations) {
-        my $log_url = get_permanent_url_for_build_log( $cfg );
+        my (@log_url) = $self->get_url_for_build_log( $cfg, $log_base_url );
 
         my $this_out;
 
-        if (my $summary = get_build_summary_from_log_url( $log_url )) {
+        if (my $summary = get_build_summary_from_log_url( @log_url )) {
             # If we can get a sensible summary, just do that.
             # The summary should already mention the tested configuration.
             $this_out = $summary;
@@ -244,11 +356,11 @@ sub summarize_jenkins_build
             $this_out = "$cfg->{ fullDisplayName }: $cfg->{ result }";
         }
 
-        if ($log_url) {
+        if (@log_url) {
             if ($this_out !~ m{\n\z}ms) {
                 $this_out .= "\n";
             }
-            $this_out .= "  Build log: $log_url";
+            $this_out .= "  Build log: $log_url[0]";
         }
 
         push @summaries, $this_out;
@@ -268,17 +380,21 @@ sub run
     my ($self, @args) = @_;
 
     my $url;
+    my $log_base_url;
 
     GetOptionsFromArray(
         \@args,
         'url=s' => \$url,
+        'log-base-url=s' => \$log_base_url,
+        'force-jenkins-host=s' => \$self->{ force_jenkins_host },
+        'force-jenkins-port=i' => \$self->{ force_jenkins_port },
         'h|help' => sub { pod2usage( 2 ) },
         'debug' => \$self->{ debug },
     );
 
     $url || die 'Missing mandatory --url option';
 
-    print $self->summarize_jenkins_build( $url ) . "\n";
+    print $self->summarize_jenkins_build( $url, $log_base_url ) . "\n";
 
     return;
 }
diff --git a/scripts/jenkins/t/05-qt-jenkins-ci.t b/scripts/jenkins/t/05-qt-jenkins-ci.t
index 58d7a92a..260c85f1 100644
--- a/scripts/jenkins/t/05-qt-jenkins-ci.t
+++ b/scripts/jenkins/t/05-qt-jenkins-ci.t
@@ -68,7 +68,7 @@ Readonly my $SUMMARIZE_JENKINS_BUILD => realpath( catfile( $FindBin::Bin, '..',
 
 # Mock the fetching of data from Jenkins and the running of external commands.
 # Returns an object. When that object is destroyed, the mocking is undone.
-sub do_mocks
+sub do_mocks  ## no critic(RequireArgUnpacking) - mocked http_get is incompatible with this policy
 {
     my (%args) = @_;
 
@@ -118,8 +118,29 @@ sub do_mocks
         return $cv;
     });
 
+    my $http_get_override = Sub::Override->new( "${PACKAGE}::http_get", sub($@) {
+        my $url = shift @_;
+        my $done_cb = pop @_;
+        my %http_args = @_;
+
+        if (ok( exists( $args{ url_to_content }{ $url } ), "fetched $url as expected (http_get)" )) {
+            my $content = $args{ url_to_content }{ $url };
+            my $headers = { Status => 200 };
+            if ($http_args{ on_body }) {
+                $http_args{ on_body }->( $content, $headers );
+                undef $content;
+            }
+            $done_cb->( $content, $headers );
+        } else {
+            $done_cb->( undef, { Status => 400, Reason => "(mock) not expected to fetch $url" } );
+        }
+
+        return;
+    });
+
     return {
         fetch_mock => $fetch_override,
+        http_get_mock => $http_get_override,
         cmd_mock => $cmd_override,
     };
 }
@@ -263,6 +284,34 @@ sub test_complete_build
 }
 END_JSON
 
+        my $log_cmd_for_cfg = sub {
+            my ($config_name) = @_;
+
+            # Empty config name means the master, which has no nested directory
+            # (and so no /)
+            if ($config_name) {
+                $config_name = "/$config_name";
+            } else {
+                $config_name = q{};
+            }
+
+            return
+                    '[ssh] [-oBatchMode=yes] [-p] [555] [logs.example.com] '
+                .qq{[mkdir -p "/var/www/ci/Some_Job/build_00005$config_name" && }
+                .qq{cd "/var/www/ci/Some_Job/build_00005$config_name" && }
+                .  'gzip > .incoming.log.txt.gz && mv .incoming.log.txt.gz log.txt.gz]';
+        };
+
+        # command to setup 'latest', 'latest-success' links (successful build only)
+        my $ln_latest_and_latest_success_cmd =
+                '[ssh] [-oBatchMode=yes] [-p] [555] [logs.example.com] '
+               .'[cd "/var/www/ci/Some_Job" && ln -sf "build_00005" latest && ln -sf "build_00005" latest-success]';
+
+        # command to setup 'latest' link (unsuccessful build only - no 'latest-success')
+        my $ln_latest_cmd =
+                '[ssh] [-oBatchMode=yes] [-p] [555] [logs.example.com] '
+               .'[cd "/var/www/ci/Some_Job" && ln -sf "build_00005" latest]';
+
         my $mock = do_mocks(
             url_to_content => {
 
@@ -274,12 +323,18 @@ END_JSON
 
                 "$url_base/5/api/json?tree=result"
                 =>
-                '{"result":"SUCCESS"}',
+                '{"result":"SUCCESS"}'
 
+                ,
+
+                "$url_base/5/config_X/consoleText" => 'fake log X',
+                "$url_base/5/config_Y/consoleText" => 'fake log Y',
+                "$url_base/5/config_Z/consoleText" => 'fake log Z',
+                "$url_base/5/consoleText" => 'fake log master',
             },
 
             cmd => {
-                "[$SUMMARIZE_JENKINS_BUILD] [--url] [http://jenkins.example.com/job/Some_Job/5]"
+                "[$SUMMARIZE_JENKINS_BUILD] [--url] [http://jenkins.example.com/job/Some_Job/5] [--log-url] [http://logs.example.com:8080/ci]"
                 =>
                 { exitcode => 0, '>' => '(fake summary)' }
 
@@ -290,6 +345,28 @@ END_JSON
                .'[--project] [other/project] [--result] [pass] [--message] [-]'
                 =>
                 { exitcode => 0, '<' => '(fake summary)' }
+
+                ,
+
+                $log_cmd_for_cfg->( 'config_X' )
+                =>
+                { exitcode => 0 },
+
+                $log_cmd_for_cfg->( 'config_Y' )
+                =>
+                { exitcode => 0 },
+
+                $log_cmd_for_cfg->( 'config_Z' )
+                =>
+                { exitcode => 0 },
+
+                $log_cmd_for_cfg->( )
+                =>
+                { exitcode => 0 },
+
+                $ln_latest_and_latest_success_cmd
+                =>
+                { exitcode => 0 },
             }
 
         );
@@ -303,6 +380,8 @@ END_JSON
             my $obj = $PACKAGE->new();
             $obj->run(
                 'complete_build',
+                '--log-upload-url', 'ssh://logs.example.com:555/var/www/ci',
+                '--log-download-url', 'http://logs.example.com:8080/ci'
             );
         }
 
@@ -319,12 +398,18 @@ END_JSON
 
                 "$url_base/5/api/json?tree=result"
                 =>
-                '{"result":"FAILURE"}',
+                '{"result":"FAILURE"}'
+
+                ,
 
+                "$url_base/5/config_X/consoleText" => 'fake log X',
+                "$url_base/5/config_Y/consoleText" => 'fake log Y',
+                "$url_base/5/config_Z/consoleText" => 'fake log Z',
+                "$url_base/5/consoleText" => 'fake log master',
             },
 
             cmd => {
-                "[$SUMMARIZE_JENKINS_BUILD] [--url] [http://jenkins.example.com/job/Some_Job/5]"
+                "[$SUMMARIZE_JENKINS_BUILD] [--url] [http://jenkins.example.com/job/Some_Job/5] [--log-url] [http://logs.example.com:8080/ci]"
                 =>
                 { exitcode => 0, '>' => '(fake summary)' }
 
@@ -335,6 +420,28 @@ END_JSON
                .'[--project] [other/project] [--result] [fail] [--message] [-]'
                 =>
                 { exitcode => 0, '<' => '(fake summary)' }
+
+                ,
+
+                $log_cmd_for_cfg->( 'config_X' )
+                =>
+                { exitcode => 0 },
+
+                $log_cmd_for_cfg->( 'config_Y' )
+                =>
+                { exitcode => 0 },
+
+                $log_cmd_for_cfg->( 'config_Z' )
+                =>
+                { exitcode => 0 },
+
+                $log_cmd_for_cfg->( )
+                =>
+                { exitcode => 0 },
+
+                $ln_latest_cmd
+                =>
+                { exitcode => 0 },
             }
 
         );
@@ -343,6 +450,8 @@ END_JSON
             my $obj = $PACKAGE->new();
             $obj->run(
                 'complete_build',
+                '--log-upload-url', 'ssh://logs.example.com:555/var/www/ci',
+                '--log-download-url', 'http://logs.example.com:8080/ci'
             );
         }
     }
@@ -352,6 +461,42 @@ END_JSON
     return;
 }
 
+# basic checks of the run_timed_cmd utility function
+sub test_run_timed_cmd
+{
+    my $run_timed_cmd = $PACKAGE->can('run_timed_cmd');
+
+    ok( $run_timed_cmd, 'run_timed_cmd defined' );
+
+    {
+        my $cv = $run_timed_cmd->( [ 'perl', '-e', '1' ] );
+        is( $cv->recv(), 0, 'run_timed_cmd simple success OK' );
+    }
+
+    {
+        my $cv = $run_timed_cmd->( [ 'perl', '-e', 'exit 3' ] );
+        is( $cv->recv(), (3 << 8), 'run_timed_cmd simple failure OK' );
+    }
+
+    {
+        my $cv = $run_timed_cmd->( [ 'perl', '-e', 'sleep 4' ], timeout => 1 );
+        is( $cv->recv(), -1, 'run_timed_cmd times out' );
+    }
+
+    {
+        # make sure it works when we ask for pid.
+        my $pid;
+        my $cv = $run_timed_cmd->( [ 'perl', '-e', 'sleep 4' ], timeout => 2, '$$' => \$pid );
+        ok( $pid, 'pid is set' );
+        is( kill( 15, $pid ), 1, 'kill OK' ) || diag $!;
+
+        my $status = $cv->recv();
+        is( ($status & 127), 15, 'run_timed_cmd killed OK' );
+    }
+
+    return;
+}
+
 # main entry point
 sub run
 {
@@ -365,6 +510,7 @@ sub run
 
     test_new_build();
     test_complete_build();
+    test_run_timed_cmd();
     done_testing();
 
     return;
diff --git a/scripts/jenkins/t/10-summarize-jenkins-build.t b/scripts/jenkins/t/10-summarize-jenkins-build.t
index 96008575..c59fff21 100644
--- a/scripts/jenkins/t/10-summarize-jenkins-build.t
+++ b/scripts/jenkins/t/10-summarize-jenkins-build.t
@@ -61,6 +61,7 @@ use FindBin;
 use Readonly;
 use Sub::Override;
 use Test::More;
+use Test::Warn;
 
 Readonly my $SCRIPT => catfile( $FindBin::Bin, '..', 'summarize-jenkins-build.pl' );
 Readonly my $PACKAGE => 'QtQA::App::SummarizeJenkinsBuild';
@@ -71,6 +72,9 @@ Readonly my $PACKAGE => 'QtQA::App::SummarizeJenkinsBuild';
 #   name => the human-readable name for this test
 #   object => QtQA::App::SummarizeJenkinsBuild object
 #   url => url to summarize
+#   error_url => arrayref of URLs which, if fetched by parse_build_log.pl, should
+#                generate an error. All other URLs succeed with dummy text.
+#   warnings_like => arrayref of expected warning patterns
 #   fake_json => ref to a hash containing (key,value) pairs, where keys are
 #       URLs and values are the fake JSON text to return for a URL
 #   expected_output => expected output of the summarize_jenkins_build function
@@ -81,6 +85,9 @@ sub do_test
 
     my $o = $args{ object };
     my $name = $args{ name };
+    my @error_url = @{ $args{ error_url } || [] };
+    my $warnings_like = $args{ warnings_like } || [];
+    my $warnings_count = @{ $warnings_like };
 
     my @mock_subs;
     if (my $fake_json = $args{ fake_json }) {
@@ -100,12 +107,22 @@ sub do_test
         "${PACKAGE}::run_parse_build_log",
         sub {
             my ($url) = @_;
+
+            if (grep { $_ eq $url } @error_url) {
+                print STDERR "(parse_build_log.pl error for $url)\n";
+                return 1;
+            }
+
             print "(parse_build_log.pl output for $url)\n";
             return 0;
         },
     );
 
-    my $output = $o->summarize_jenkins_build( $args{ url } );
+    my $output;
+    warnings_like {
+        $output = $o->summarize_jenkins_build( $args{ url }, $args{ log_url } );
+    } $warnings_like, "$name: $warnings_count warning(s) as expected";
+
     is( $output, $args{ expected_output }, "$name: summarize_jenkins_build output as expected" );
 
     return;
@@ -152,6 +169,19 @@ sub run_object_tests
     );
 
     do_test(
+        name => 'failure with rebased master log',
+        object => $o,
+        url => $url,
+        log_url => 'http://testresults.example.com/ci',
+        fake_json => {
+            $url => '{"number":3,"result":"FAILURE","fullDisplayName":"bar build 3","url":"http://example.com/jenkins/job/Some_Job/123"}',
+        },
+        expected_output =>
+            "(parse_build_log.pl output for http://testresults.example.com/ci/Some_Job/build_00123/log.txt.gz)\n"
+           ."  Build log: http://testresults.example.com/ci/Some_Job/build_00123/log.txt.gz",
+    );
+
+    do_test(
         name => 'failure with master and configuration logs',
         object => $o,
         url => $url,
@@ -180,6 +210,81 @@ END
     );
 
     do_test(
+        name => 'failure with rebased master and configuration logs',
+        object => $o,
+        url => $url,
+        log_url => 'http://testresults.example.com/ci',
+        fake_json => {
+            $url => <<'END'
+{
+    "number":4,
+    "result":"FAILURE",
+    "fullDisplayName":"bar build 4",
+    "url":"master-url",
+    "runs":[
+        {"number":4,"result":"FAILURE","fullDisplayName":"cfg1","url":"http://example.com/jenkins/job/bar/key1=val1,cfg=cfg1/4/"},
+        {"number":4,"result":"SUCCESS","fullDisplayName":"cfg2","url":"cfg2-url"},
+        {"number":4,"result":"FAILURE","fullDisplayName":"cfg3","url":"http://example.com/jenkins/job/bar/./cfg=cfg3/4"},
+        {"number":5,"result":"FAILURE","fullDisplayName":"not-this","url":"not-this-url"}
+    ]
+}
+END
+        },
+        expected_output =>
+            # note multi-axis config name is left as-is...
+            "(parse_build_log.pl output for http://testresults.example.com/ci/bar/build_00004/key1=val1,cfg=cfg1/log.txt.gz)\n"
+           ."  Build log: http://testresults.example.com/ci/bar/build_00004/key1=val1,cfg=cfg1/log.txt.gz"
+           ."\n\n--\n\n"
+            # ... while a config with a single axis is collapsed, useless cfg= prefix removed
+           ."(parse_build_log.pl output for http://testresults.example.com/ci/bar/build_00004/cfg3/log.txt.gz)\n"
+           ."  Build log: http://testresults.example.com/ci/bar/build_00004/cfg3/log.txt.gz"
+    );
+
+    {
+        # try --force-jenkins-host and --force-jenkins-port
+        local $o->{ force_jenkins_host } = 'forced-host';
+        local $o->{ force_jenkins_port } = 999;
+        do_test(
+            name => 'failure with rebased master and configuration logs, log force and fallback',
+            object => $o,
+            url => $url,
+            log_url => 'http://testresults.example.com/ci',
+            warnings_like => [
+                qr{
+                    \Qhttp://testresults.example.com/ci/bar/build_00004/key1=val1,cfg=cfg1/log.txt.gz\E
+                    .*
+                    \Qparse_build_log exited with status 1\E
+                }xms
+            ],
+            error_url => [ 'http://testresults.example.com/ci/bar/build_00004/key1=val1,cfg=cfg1/log.txt.gz' ],
+            fake_json => {
+                'http://forced-host:999/jenkins/123' => <<'END'
+{
+    "number":4,
+    "result":"FAILURE",
+    "fullDisplayName":"bar build 4",
+    "url":"master-url",
+    "runs":[
+        {"number":4,"result":"FAILURE","fullDisplayName":"cfg1","url":"http://example.com/jenkins/job/bar/key1=val1,cfg=cfg1/4/"},
+        {"number":4,"result":"SUCCESS","fullDisplayName":"cfg2","url":"cfg2-url"},
+        {"number":4,"result":"FAILURE","fullDisplayName":"cfg3","url":"http://example.com/jenkins/job/bar/cfg=cfg3/4"},
+        {"number":5,"result":"FAILURE","fullDisplayName":"not-this","url":"not-this-url"}
+    ]
+}
+END
+            },
+            expected_output =>
+                # we simulated an error on the testresults host here, so parse_build_log was run directly on jenkins,
+                # but the link passed to gerrit is still the testresults link.
+                "(parse_build_log.pl output for http://forced-host:999/jenkins/job/bar/key1=val1,cfg=cfg1/4/consoleText)\n"
+               ."  Build log: http://testresults.example.com/ci/bar/build_00004/key1=val1,cfg=cfg1/log.txt.gz"
+               ."\n\n--\n\n"
+               ."(parse_build_log.pl output for http://testresults.example.com/ci/bar/build_00004/cfg3/log.txt.gz)\n"
+               ."  Build log: http://testresults.example.com/ci/bar/build_00004/cfg3/log.txt.gz"
+        );
+    }
+
+    do_test(
         name => 'aborted ignores failure logs',
         object => $o,
         url => $url,
diff --git a/scripts/setup.pl b/scripts/setup.pl
index d6be08d3..7eabd05c 100755
--- a/scripts/setup.pl
+++ b/scripts/setup.pl
@@ -159,6 +159,7 @@ sub all_required_cpan_modules
         AnyEvent::Util
         Capture::Tiny
         Class::Factory::Util
+        Coro::AnyEvent
         Env::Path
         File::chdir
         File::Copy::Recursive