From ef4ae4acbed531b886d581b12df8247b8b7f1740 Mon Sep 17 00:00:00 2001 From: Rohan McGovern Date: Mon, 9 Jul 2012 14:09:52 +1000 Subject: jenkins: add log synchronization support Allow qt-jenkins-ci.pl to put compressed logs on another host when a build has completed, and allow summarize-jenkins-build.pl to generate links referring to that host. This supports a setup where the Jenkins build data may be deleted without notice, or where the Jenkins server is not publicly accessible. Change-Id: I48a66866ad56fa7df74b190927b192c9f4cb22df Reviewed-by: Kalle Lehtonen --- scripts/jenkins/summarize-jenkins-build.pl | 170 ++++++++++++++++++++++++----- 1 file changed, 143 insertions(+), 27 deletions(-) (limited to 'scripts/jenkins/summarize-jenkins-build.pl') diff --git a/scripts/jenkins/summarize-jenkins-build.pl b/scripts/jenkins/summarize-jenkins-build.pl index 1667614d..0c5db237 100755 --- a/scripts/jenkins/summarize-jenkins-build.pl +++ b/scripts/jenkins/summarize-jenkins-build.pl @@ -77,6 +77,37 @@ URL of the Jenkins build. For a multi-configuration build, the URL of the top-level build should be used. The script will parse the logs from each configuration. +=item --log-base-url B + +Base URL of the build logs. + +Optional; if set, build logs will be fetched from URLs under this path, instead +of directly from Jenkins. This may be used to support a setup where Jenkins +build logs are volatile, subject to removal without notice, and the logs are +primarily accessed from another server. + +The URLs constructed using B are currently not customizable, and always +use the following pattern: + + //build_//log.txt.gz + +If build logs cannot be fetched from this URL for any reason, the logs are parsed +directly from Jenkins. However, any user-visible links will still refer to the +URL passed in this option. + +Note that the build status is still fetched directly from Jenkins. + +=item --force-jenkins-host + +=item --force-jenkins-port + +When fetching any data from Jenkins, disregard the host and port portion +of Jenkins URLs and use these instead. + +This is useful for network setups (e.g. port forwarding) where the Jenkins +host cannot access itself using the outward-facing hostname, or simply to +avoid unnecessary round-trips through a reverse proxy setup. + =item --debug Print an internal representation of the build to standard error, for debugging @@ -95,6 +126,7 @@ use Getopt::Long qw(GetOptionsFromArray); use JSON; use Pod::Usage; use Readonly; +use URI; # Jenkins status constants Readonly my $SUCCESS => 'SUCCESS'; @@ -141,55 +173,135 @@ sub run_parse_build_log return system( $PARSE_BUILD_LOG, '--summarize', $url ); } -# Returns the output of "parse_build_log.pl --summarize $url", -# or warns and returns nothing on error. +# Returns the output of parse_build_log.pl on the first +# working of the given @url, or warns and returns nothing +# if all URLs are not usable. sub get_build_summary_from_log_url { - my ($url) = @_; + my (@url) = @_; - return unless $url; + return unless @url; + + my $stdout; - my $status; - my ($stdout, $stderr) = capture { - $status = run_parse_build_log( $url ); - }; + while (!$stdout && @url) { + my $url = shift @url; + my $stderr; + my $status; + ($stdout, $stderr) = capture { + $status = run_parse_build_log( $url ); + }; - chomp $stdout; + chomp $stdout; - if ($status != 0) { - warn "parse_build_log exited with status $status" - .($stderr ? ":\n$stderr" : q{}) - ."\n"; + if ($status != 0) { + warn "for $url, parse_build_log exited with status $status" + .($stderr ? ":\n$stderr" : q{}) + ."\n"; - # Output is not trusted if script didn't succeed - undef $stdout; + # Output is not trusted if script didn't succeed + undef $stdout; + } } return $stdout; } -# Given a Jenkins build object, returns a "permanent" link -# to the build log (which may itself not be in jenkins). -sub get_permanent_url_for_build_log +# Given a Jenkins build object, returns one or more links +# to the build logs, in order of preference. +sub get_url_for_build_log { - my ($cfg) = @_; + my ($self, $cfg, $log_base_url) = @_; - # FIXME: support testresults.qt-project.org logs my $url = $cfg->{ url }; return unless $url; + my @out; + + if ($log_base_url) { + if ($url =~ + m{ + \A + .+ + /job/ + (? + [^/]+ # job name + + ) + / + (?: + # jenkins sometimes introduces a useless './' into the URLs + \./ + )* + (?: + # configuration part is optional; + # it is not present if the failure was on the master, for instance. + (? + [^/]+ + ) + / + )? + (? + [0-9]+ + ) + /? + \z + }xms + ) { + my ($job_name, $cfg_name, $build_number) = @+{ 'job', 'cfg', 'build' }; + if ($cfg_name) { + # If $cfg_name only has one axis (the normal case), just use it directly, + # to avoid useless 'cfg=' in URLs. + $cfg_name =~ s{\A [^=]+ = ([^=]+) \z}{$1}xms; + $cfg_name =~ s{ }{_}g; + } + push @out, sprintf( '%s/%s/build_%05d%s/log.txt.gz', $log_base_url, $job_name, $build_number, $cfg_name ? "/$cfg_name" : q{}); + } else { + warn "URL '$url' not of expected format, cannot rebase to $log_base_url\n"; + } + } + + # Use direct Jenkins logs if ($url !~ m{/\z}) { $url .= '/'; } - return $url . 'consoleText'; + push @out, $self->maybe_rewrite_url( $url . 'consoleText' ); + return @out; +} + +# Returns a version of $url possibly with the host and port replaced, according +# to the --force-jenkins-host and --force-jenkins-port command-line arguments. +sub maybe_rewrite_url +{ + my ($self, $url) = @_; + + if (!$self->{ force_jenkins_host } && !$self->{ force_jenkins_port }) { + return $url; + } + + my $parsed = URI->new( $url ); + if ($self->{ force_jenkins_host }) { + $parsed->host( $self->{ force_jenkins_host } ); + } + if ($self->{ force_jenkins_port }) { + $parsed->port( $self->{ force_jenkins_port } ); + } + + return $parsed->as_string(); } # Given a jenkins build $url, returns a human-readable summary of # the build result. +# +# If $log_base_url is set, log URLs are derived under that base +# URL using a predefined pattern, with Jenkins logs used as a fallback. +# Otherwise, the logs are used directly from Jenkins. sub summarize_jenkins_build { - my ($self, $url) = @_; + my ($self, $url, $log_base_url) = @_; + + $url = $self->maybe_rewrite_url( $url ); my $build = get_build_data_from_url( $url ); @@ -230,11 +342,11 @@ sub summarize_jenkins_build my @summaries; foreach my $cfg (@configurations) { - my $log_url = get_permanent_url_for_build_log( $cfg ); + my (@log_url) = $self->get_url_for_build_log( $cfg, $log_base_url ); my $this_out; - if (my $summary = get_build_summary_from_log_url( $log_url )) { + if (my $summary = get_build_summary_from_log_url( @log_url )) { # If we can get a sensible summary, just do that. # The summary should already mention the tested configuration. $this_out = $summary; @@ -244,11 +356,11 @@ sub summarize_jenkins_build $this_out = "$cfg->{ fullDisplayName }: $cfg->{ result }"; } - if ($log_url) { + if (@log_url) { if ($this_out !~ m{\n\z}ms) { $this_out .= "\n"; } - $this_out .= " Build log: $log_url"; + $this_out .= " Build log: $log_url[0]"; } push @summaries, $this_out; @@ -268,17 +380,21 @@ sub run my ($self, @args) = @_; my $url; + my $log_base_url; GetOptionsFromArray( \@args, 'url=s' => \$url, + 'log-base-url=s' => \$log_base_url, + 'force-jenkins-host=s' => \$self->{ force_jenkins_host }, + 'force-jenkins-port=i' => \$self->{ force_jenkins_port }, 'h|help' => sub { pod2usage( 2 ) }, 'debug' => \$self->{ debug }, ); $url || die 'Missing mandatory --url option'; - print $self->summarize_jenkins_build( $url ) . "\n"; + print $self->summarize_jenkins_build( $url, $log_base_url ) . "\n"; return; } -- cgit v1.2.3