diff options
| -rwxr-xr-x | scripts/generic/parse_build_log.pl | 309 |
1 files changed, 120 insertions, 189 deletions
diff --git a/scripts/generic/parse_build_log.pl b/scripts/generic/parse_build_log.pl index 5002d07a..e351d780 100755 --- a/scripts/generic/parse_build_log.pl +++ b/scripts/generic/parse_build_log.pl @@ -80,35 +80,6 @@ Print this message. If given, as well as printing out the interesting lines from the log, the script will attempt to print out a human-readable summary of the error(s). -This option has no effect when --yaml is given. - -=item B<--yaml> - -Use YAML instead of plain text output. - -The output will consist of a single YAML document with zero or more of the -following key/value pairs: - -=over - -=item summary - -The human-readable summary of the failure reason (if known). -Usually one or two sentences. - -=item detail - -The extracted text from the build log relating to the failure. - -=item should_retry - -A hint that it might make sense to retry the build/test. - -If set, this indicates that the failure extracted from the log may be unrelated -to the code under test; for example, a temporary network outage. - -=back - =item B<--limit> LINES Limit the amount of extracted lines to the given value. @@ -121,6 +92,12 @@ If omitted, an undefined but reasonable default is used. Enable some debug messages to STDERR. Use this to troubleshoot when some log is not parsed in the expected manner. +=item B<--trim-prefix> REGEX + +Remove any matching content from the specified regular expression before +further analyzing the content. Use this when our log output is filtered +through an intermediate program that adds a prefix such as a time stamp. + =back =head1 CAVEATS @@ -135,35 +112,31 @@ use File::Basename; use File::Slurp qw(); use Getopt::Long qw(GetOptionsFromArray); use IO::Uncompress::AnyInflate qw(anyinflate $AnyInflateError); -use Lingua::EN::Inflect qw(inflect PL WORDLIST); -use Lingua::EN::Numbers qw(num2en); use List::MoreUtils qw(any apply); use Pod::Usage; -use Const::Fast; use Text::Wrap; -use YAML qw(); # Contact details of some CI admins who can deal with problems. # Put a public email address here once we have one! -const my $CI_CONTACT - => q{some CI administrator}; +my $CI_CONTACT + = q{some CI administrator}; # The max amount of lines we're willing to buffer before giving up, # when attempting to identify a related chunk of output (e.g. a single # autotest log). -const my $MAX_CHUNK_LINES => 5000; +my $MAX_CHUNK_LINES = 5000; # The max amount of characters permitted in a line; # any more than this and we will truncate the line. # Longer lines could trigger bad performance in some regexes, and it is # not user-friendly to present such long lines to the reader. -const my $MAX_LINE_LENGTH => 3500; +my $MAX_LINE_LENGTH = 3500; # The max amount of lines to search around any interesting line for # related text (for example, if a compiler failure message is seen for # foo.cpp, look up to $RECENT_MAX lines in the past for other messages # relating to .cpp). -const my $RECENT_MAX => 60; +my $RECENT_MAX = 60; # List of all common error strings returned by strerror(); # This may be generated by: @@ -177,8 +150,8 @@ const my $RECENT_MAX => 60; # Note that these are matched case-insensitive, as certain tools seem to use # messages from this list with slight differences in case. # -const my @POSIX_ERROR_STRINGS - => split /\n/, <<'END_ERROR_STRINGS'; +my @POSIX_ERROR_STRINGS + = split /\n/, <<'END_ERROR_STRINGS'; .lib section in a.out corrupted Accessing a corrupted shared library Address already in use @@ -258,6 +231,7 @@ Link has been severed Link number out of range Machine is not on the network Malformed Mach-o file +Memory page has hardware error Message too long Multihop attempted Name not unique on network @@ -291,6 +265,7 @@ Object is remote Operation already in progress Operation canceled Operation not permitted +Operation not possible due to RF-kill Operation not supported Operation not supported by device Operation not supported on socket @@ -321,6 +296,7 @@ Resource busy Resource deadlock avoided Resource temporarily unavailable Result too large +Stale file handle STREAM ioctl timeout Shared library version mismatch Socket is already connected @@ -355,13 +331,13 @@ END_ERROR_STRINGS # List of any test script contexts where an error indicates that the build may be # able to succeed if we retry. -const my %TESTSCRIPT_RETRY_CONTEXTS => map { $_ => 1 } ( +my %TESTSCRIPT_RETRY_CONTEXTS = map { $_ => 1 } ( 'determining test script configuration', # usually error in qtqa/testconfig 'setting up git repositories', # usually network outage or similar ); # All important regular expressions used to extract errors -const my %RE => ( +my %RE = ( # never matches anything never_match => qr{a\A}ms, @@ -482,6 +458,15 @@ const my %RE => ( ' \s \Qexited with code \E\d }xms, + # configure output. + # + configure_begin => qr{ + Running configuration tests(?: \(phase [12]\))\.\.\. + }xms, + configure_end => qr{ + \QDone running configuration tests.\E + }xms, + # make failed. # # Examples: @@ -514,7 +499,7 @@ const my %RE => ( | [gn]make # GNU make, nmake | - mingw32-make + [Mm]ingw32-make ) (?: \.exe )? # maybe has .exe on the end on Windows @@ -556,16 +541,19 @@ const my %RE => ( | - # This comes when make itself segfaults - \QSegmentation fault: 11\E + # This comes when make itself or a tool segfaults + (?<errortext> + \QSegmentation fault: 11\E + .*? + ) ) | (?<errortext> - \QNo rule to make target `\E - [^']+ - \Q', needed by `\E + \QNo rule to make target \E. + [^']+? + \Q', needed by \E. (?<target> [^']+ ) @@ -633,6 +621,7 @@ const my %RE => ( # foobar.cpp:123: error: quiznux (?<file> + (?:\w:)? [^:]+ ) @@ -642,11 +631,11 @@ const my %RE => ( \d+ ) - (?: # It is possible to have more than one line number in the error, e.g: + (?: # gcc sometimes includes column number after line number, e.g: :\d+ # mapsgl/frustum_p.h:60:27: (...) - )* # We do not capture them at the moment. + )* # We do not capture this at the moment. - : \s + : \s+ (?<error> (?: @@ -672,7 +661,7 @@ const my %RE => ( /bin/sh: \s+ - line \s \d+: + line \s+ \d+: \s+ \d+ # pid @@ -691,7 +680,7 @@ const my %RE => ( .+? # rest of command and arguments ... (?<file> # ...and assume file is the last argument (qmake-specific assumption) - [^\s]+ + \S+ ) | @@ -804,8 +793,8 @@ const my %RE => ( \s+ ) from - \s - [^\s]+:\d+ # some/file.cpp:123 + \s+ + \S+:\d+ # some/file.cpp:123 [,:] # , or : depending on whether it's the last line \s* \z @@ -864,13 +853,13 @@ const my %RE => ( (?: (?<linker> - ld # basename only + (?:[a-z0-9.-]+-)?ld # basename only | - /[^\s]{1,80}/ld # full path + /\S{1,80}/(?:[a-z0-9.-]+-)?ld # full path ) : - \s + \s+ (?<error> (?: @@ -935,14 +924,14 @@ const my %RE => ( .{1,300}? # file part (may be .o, .cpp, or both, with also .text reference) : - \s + \s+ (?: - \Qundefined reference to `\E + \Qundefined reference to \E | - \Qmore undefined references to `\E + \Qmore undefined references to \E | - \Qmultiple definition of `\E + \Qmultiple definition of \E ) .+ @@ -1029,9 +1018,10 @@ const my %RE => ( \A \s* + (?:\w:)? [^:]+\.o: - \s - \QIn function `\E + \s+ + \QIn function \E .+ \z @@ -1077,7 +1067,7 @@ const my %RE => ( .+ -o - \s + \s+ (?<lib> lib [^\.]+ \. # name always starts with libSomething. @@ -1088,7 +1078,7 @@ const my %RE => ( \d # mac: libQtCore.5.0.0.dylib ) - [^\s]+ + \S+ ) | @@ -1111,10 +1101,10 @@ const my %RE => ( # silent mode, linking path/to/libWhatever.so linking - \s + \s+ (?<lib> - [^\s]+ + \S+ (?: \.so @@ -1122,7 +1112,7 @@ const my %RE => ( \.dylib # must contain at least one .so or .dylib to be a library ) - [^\s]+ + \S+ ) \z @@ -1132,9 +1122,9 @@ const my %RE => ( # silent mode, linking path/to/Something.framework/Something linking - \s + \s+ - [^\s]+? + \S+? / (?<lib> \w+ @@ -1143,7 +1133,7 @@ const my %RE => ( ) / - [^\s]+ + \S+ \z ) @@ -1174,6 +1164,7 @@ const my %RE => ( # path/to/file.pro:123: Parse Error (?<file> + (?:\w:)? [^:]+? \.pr[iof] ) @@ -1191,6 +1182,7 @@ const my %RE => ( \QCannot find file: \E ) (?<file> + (?:\w:)? [^:]+? \.pr[iof] ) @@ -1265,6 +1257,23 @@ const my %RE => ( .* (?<tool_objcopy>) + | + # QtPlatformHeaders + + ^QtPlatformHeaders: + \s+ + ERROR: + \s+ + (?<file> + (?:\w:)? + [^:]+? + ) + \s+ + includes private header + \s+ + .* + (?<tool_QtPlatformHeaders>) + # add more as discovered ) @@ -1380,10 +1389,10 @@ const my %RE => ( .*? # all the arguments to testrunner.pl [ ]--[ ] # end of the arguments to testrunner.pl - [^\s]+? # path up to the last directory separator + \S+? # path up to the last directory separator [/\\] # the last directory separator (?<name> - [^\s]+ # basename of the test + \S+ # basename of the test ) ) @@ -1396,7 +1405,7 @@ const my %RE => ( [gn]?make \[ \d+ \] : - \s + \s+ Entering[ ]directory[ ] ` @@ -1454,36 +1463,6 @@ const my %RE => ( \QQtQA::App::TestRunner: the test seems to be flaky\E }xms, - # The line where a QtQA::TestScript YAML message opens. - # - # Captures: - # - # type - the type of message (currently 'error' or 'failure') - # - yaml_begin => qr{ - \A - \Q--- !qtqa.qt-project.org/\E - (?<type> - .{1,50} - ) - \z - }xms, - - # The line where a QtQA::TestScript YAML message ends. - # - # Captures: - # - # type - the type of message (currently 'error' or 'failure') - # - yaml_end => qr{ - \A - \Q... \E\#\Q end qtqa.qt-project.org/\E - (?<type> - .{1,50} - ) - \z - }xms, - # Generic strerror-based pattern. # # This pattern will find any line containing an error string commonly returned by strerror() @@ -1610,7 +1589,7 @@ sub run $self->set_options_from_args( @args ); - my @log_lines = $self->read_file( ); + my @log_lines = $self->read_file( $self->{ trim_prefix } ); # We pass through the log twice. # The first pass determines what caused the build to fail (if anything) ... @@ -1644,13 +1623,10 @@ sub set_options_from_args 'help' => sub { pod2usage(0) }, 'debug' => \$self->{ debug }, 'summarize' => \$self->{ summarize }, - 'yaml' => \$self->{ yaml }, 'limit=i' => \$self->{ limit_lines }, + 'trim-prefix=s' => \$self->{ trim_prefix }, ) || pod2usage(1); - # summary is always generated in YAML mode - $self->{ summarize } ||= $self->{ yaml }; - # Should be exactly one argument left - the filename. if (@args > 1) { print STDERR "Too many arguments: @args\n"; @@ -1671,11 +1647,15 @@ sub set_options_from_args # - truncates line if too long sub normalize_line { - my ($self, $line) = @_; + my ($self, $line, $trim_prefix) = @_; # Note: don't use Text::Trim here, it's surprisingly slow. $line =~ s/\s+\z//; + if (length $trim_prefix) { + $line =~ s/$trim_prefix//; + } + # Truncate lines exceeding $MAX_LINE_LENGTH to $MAX_LINE_LENGTH my $length = length($line); if ($length > $MAX_LINE_LENGTH) { @@ -1735,7 +1715,7 @@ sub read_file_from_url sub read_file { - my ($self) = @_; + my ($self, $trim_prefix) = @_; my $file = $self->{ file }; @@ -1766,7 +1746,7 @@ sub read_file @lines = split( qr{\n}, $uncompressed ); # normalize before returning - @lines = map { $self->normalize_line($_) } @lines; + @lines = map { $self->normalize_line($_, $trim_prefix) } @lines; return @lines; } @@ -1945,37 +1925,6 @@ sub testscript_error_should_retry return; } -# Create a handler for an embedded YAML chunk. -sub yaml_chunk_handler -{ - my ($self, %chunk) = @_; - - $chunk{ begin_re } = $RE{ yaml_begin }; - - $chunk{ begin_sub } = sub { - my ($chunk_ref, $line, $out) = @_; - my $text = "$line\n" . $chunk_ref->{ details }; - my $loaded = eval { YAML::Load( $text ) }; - if ($loaded) { - # for backwards compatibility: message now named 'message', but used - # to be named 'error', support both for a while. - if (my $error = delete $loaded->{ error }) { - $loaded->{ message } = $error; - } - push @{$out->{ yaml_fail }}, $loaded; - - # Retry if it makes sense; note that we only retry on errors, not failures - if ($chunk{ type } eq 'error' && $self->testscript_error_should_retry( $loaded )) { - $out->{ should_retry } = 1; - } - } else { - warn "log seems to contain a corrupt YAML block:\n$text\nFailed to parse: $@"; - } - }; - - return $self->chunk_handler( 'yaml', %chunk ); -} - # Create a handler for an embedded qmake chunk. # Actually, only looks for a "Project ERROR: " line. # Ideally this would not be necessary, but unfortunately the exit code from @@ -2005,6 +1954,26 @@ sub qmake_chunk_handler return $self->chunk_handler( 'qmake', %chunk ); } +# Create a handler for configure test output. +# The sole purpose of this handler is to skip the configure output from the +# log, as it pointlessly triggers the compiler and linker error handlers. +# We don't try to identify actual configuration failures - the output is +# rather heterogenous and subject to change, so it would unreasonable to try +# to keep up. However, the log is rather short when configure actually fails, +# so it's no problem to have to look inside it. +sub configure_chunk_handler +{ + my ($self, %chunk) = @_; + + $chunk{ begin_re } = $RE{ configure_begin }; + + $chunk{ begin_sub } = sub { return 1; }; + + $chunk{ read_sub } = sub { }; + + return $self->chunk_handler( 'configure', %chunk ); +} + # Add a $tool failure identified from $line into $out, a hashref # being constructed during identify_failures. sub add_tool_fail @@ -2122,11 +2091,6 @@ sub identify_failures ); } - # opening a YAML error message? - elsif ($save_failures && $line =~ $RE{ yaml_end }) { - $chunk_handler = $self->yaml_chunk_handler( type => $+{ type } ); - } - # compiler failed? # elsif ($save_failures && $line =~ $RE{ compile_fail }) { @@ -2192,6 +2156,12 @@ sub identify_failures add_tool_fail( $out, $tool, $line ); } + # ignorable configure output? + # + elsif ($line =~ $RE{ configure_end }) { + $chunk_handler = $self->configure_chunk_handler(); + } + # Badly understood glitchy behavior? elsif ($save_failures && $line =~ $RE{ glitch }) { $out->{ should_retry } = 1; @@ -2239,23 +2209,6 @@ sub extract_autotest_fail return; } -sub extract_yaml_fail -{ - my ($self, %args) = @_; - - my $fail = $args{ fail }; - my $lines_ref = $args{ lines_ref }; - - foreach my $error (@{ $fail->{ yaml_fail } || []} ) { - my @lines = split( /\n/, $error->{ message } ); - push @{$lines_ref}, @lines; - # each failure gets one trailing blank line to separate it from others - push @{$lines_ref}, q{}; - } - - return; -} - sub extract { my ($self, %args) = @_; @@ -2298,12 +2251,6 @@ sub extract $summary = $self->extract_summary( $fail ); } - # YAML failures (explicit failure messages from the test script(s)) come first. - $self->extract_yaml_fail( - fail => $fail, - lines_ref => \@detail, - ); - # Output any autotest failures next. $self->extract_autotest_fail( fail => $fail, @@ -2591,11 +2538,8 @@ sub extract_summary my $linked_too_early = $fail->{ linker_attempted_to_link_too_early }; if ($linked_too_early) { my @libs = keys %{ $linked_too_early }; - - Lingua::EN::Inflect::NUM( scalar(@libs) ); - my $project = (@libs > 1) ? 'project(s)' : 'project'; - my $that_lib_was = inflect 'PL(that) PL(library) PL(was)'; + my $that_lib_was = scalar(@libs) > 1 ? 'those libraries were' : 'that library was'; my $lib = WORDLIST( @libs, { conj => q{and/or} } ); $summary .= "\n\nIt seems that some $project tried to link against $lib " @@ -2663,19 +2607,6 @@ sub output { my ($self, $data) = @_; - if ($self->{ yaml }) { - my %yamldata = %{ $data || {} }; - - # in the YAML document, output the lines as a single scalar - # rather than a list - if ($yamldata{ detail }) { - $yamldata{ detail } = join( "\n", @{ $yamldata{ detail } || [] } ); - } - - print YAML::Dump( \%yamldata ); - return; - } - my $summary; my $detail_indent = q{}; |