Testparser: Testfunctions and testrows

Added parsing of testfunctions and testrows from the
testreport xml file archive on testresults server.
Added saving the testfunctions and testrows data
into the database tables.
Some refactoring made to be able to parse and save the
new data (checking of existing database table indexes
before creating them), and to optimize the performance
(build delete queries).
New options infomsg and skiptestrowpass, and printing
of errors, warnings and execution time added to help
the testing.

Change-Id: I1abedd0f31ace6c494a3b4abda4423d68e89b740
Reviewed-by: Tony Sarajärvi <tony.sarajarvi@theqtcompany.com>

1 files changed, 669 insertions, 291 deletions

diff --git a/non-puppet/qtmetrics2/testparser.pl b/non-puppet/qtmetrics2/testparser.pl
index 0c535e3..d3bbb87 100644
--- a/non-puppet/qtmetrics2/testparser.pl
+++ b/non-puppet/qtmetrics2/testparser.pl
@@ -45,7 +45,7 @@ testparser.pl - Qt CI system, Gather metrics from build logs to SQL database
 
 =head1 SYNOPSIS
 
-$ ./testparser.pl -method <single|full> workdir [-delete ] [-verbose] [-sqloutput <file>] [-reload] [-limit <DATE>]
+$ ./testparser.pl -method <single|full> workdir [-delete ] [-verbose] [-infomsg] [-sqloutput <file>] [-reload] [-skiptestrowpass] [-limit <DATE>]
 
 Scan through logs for one or several builds in one go.
 
@@ -72,6 +72,11 @@ See L<METHOD> for more information.
 
 Prints a lot more information of what the script does.
 
+=item -infomsg
+
+Prints the INFO messages in addition to the ERROR and WARNING messages which
+are printed by default.
+
 =item -sqloutput <file>
 
 Define a file into which table injection commands are written to.
@@ -84,6 +89,12 @@ Possible to use when using 'single' as L<METHOD>. Reloads information
 for given build into database by removing the old data matching
 the project and project number currently read.
 
+=item -skiptestrowpass
+
+Testrow_run result "pass" will not be saved into the database.
+This will reduce the testrow_run table size because over 99% of the results
+are "pass". This will also speed up the testparser run a bit.
+
 =item -limit <DATE>
 
 Possible to use when using 'full' as L<METHOD>. Skips folders that
@@ -136,10 +147,21 @@ use Getopt::Long qw( GetOptionsFromArray );
 use DBI();
 use Pod::Usage;
 use POSIX; #to be able to do 'ceil'
+use Archive::Zip qw( :ERROR_CODES :CONSTANTS);
+use XML::LibXML qw( );
 
 my $BUILDSTATEFILE = "state.json.gz";
 my $BUILDLOGFILE = "log.txt.gz";
+my $XMLARCHIVE = "test-logs.zip";
+my $SQLINSERTVALUECOUNT = 2000;     # max count of values collected into single INSERT query
 my $VERBOSE = 0;
+my $INFOMSG = 0;
+
+my $error_count = 0;
+my $warning_count = 0;
+my $info_count = 0;
+
+my $conf_count = 0;
 
 my %cfg_table = ("linux-g++_shadow-build_Ubuntu_11.10_x86" => { 'host_os' => 'linux',
                                                            'host_version' => 'Ubuntu_11.10',
@@ -588,13 +610,15 @@ sub process_arguments
     my %options;
     $options{method} = 'single';
     GetOptionsFromArray( \@args,
-        'method=s'    => \$options{method},
-        'delete'      => \$options{delete},
-        'sqloutput=s' => \$options{sqloutput},
-        'verbose'     => \$options{verbose},
-        'reload'      => \$options{reload},
-        'limit=s'     => \$options{datelimit},
-        'h|help|?'    => sub { pod2usage(1) },
+        'method=s'          => \$options{method},
+        'delete'            => \$options{delete},
+        'sqloutput=s'       => \$options{sqloutput},
+        'verbose'           => \$options{verbose},
+        'infomsg'           => \$options{infomsg},
+        'reload'            => \$options{reload},
+        'skiptestrowpass'   => \$options{skiptestrowpass},
+        'limit=s'           => \$options{datelimit},
+        'h|help|?'          => sub { pod2usage(1) },
     ) || die;
     if ($#args < 0) {
         print "Workpath not defined.\n";
@@ -639,6 +663,7 @@ sub process_arguments
     }
 
     $VERBOSE = 1 if defined $options{verbose};
+    $INFOMSG = 1 if defined $options{infomsg};
 
     return %options;
 }
@@ -650,19 +675,19 @@ sub uncompress_to_scalar
     my $gzoutput;
 
     if (check_exists_and_openable($input)) {
-      use Archive::Extract;
-      my $ae = Archive::Extract->new( archive => $input ) or warn ("Can't create archive object.");
-      if ($ae->is_gz) {
-          #print "Gzip compressed\n";
-          local $/;
-          gunzip $input => \$gzoutput or warn "gunzip filed: $GunzipError\n";
-      } else{
-          print "warning: Inputfile $input is not a .gz file\n";
-      }
-      return $gzoutput;
+        use Archive::Extract;
+        my $ae = Archive::Extract->new( archive => $input ) or warn ("ERROR: Can't create archive object.");
+        if ($ae->is_gz) {
+            #print "Gzip compressed\n";
+            local $/;
+            gunzip $input => \$gzoutput or warn "ERROR: gunzip failed: $GunzipError\n";
+        } else {
+            print "WARNING: Inputfile $input is not a .gz file\n";
+        }
+        return $gzoutput;
     }
     else {
-        print "Scalar being return is null\n";
+        print "WARNING: Scalar being return is null\n";
         return;
     }
 }
@@ -722,7 +747,7 @@ sub read_build_data
     my %data;
     my $backup_time;
 
-    print "Getting data of $inputfolder.\n";
+    print "Reading data of $inputfolder.\n";
 
     # the 'result' data might get redefined later, if one of the configurations has failed
     $data{RESULT} = if_defined($statehash->{build}->{result});
@@ -737,19 +762,22 @@ sub read_build_data
     #loop through all the runs (an array)
     foreach my $runhash (@{$statehash->{build}->{runs}}) {
 
+        print "-----------------------------------------------\n";
         my $result = if_defined($runhash->{result});
         my $number = if_defined($runhash->{number});
 
         my $cfg = if_defined($runhash->{url});
         $cfg =~ s{^.*?cfg=(.*)/\d+/}{$1};
         print "cfg = $cfg\n";
+        $conf_count++;
 
         if (defined $cfg and defined $data{NUMBER}) {
             if ($number ne $data{NUMBER}) {
                 #TODO: $data{url} and $data{BUILD_NUMBER} might be undef
-                print "ALARM! In $data{URL} $cfg\'s build number $number does not match main number $data{BUILD_NUMBER}\n";
+                print "WARNING: In $data{URL} $cfg\'s build number $number does not match main number $data{BUILD_NUMBER}\n";
                 print "       Marking this configuration as \"CANCELLED\"\n";
                 $result = "CANCELLED";
+                $warning_count++;
                 last;
             }
         }
@@ -767,6 +795,7 @@ sub read_build_data
             $data{cfg}{$cfg}{builddata}{RESULT} = "undef";
         }
 
+        # read the log.txt.gz file and get all the data from the log
         if (check_exists_and_openable(catfile($inputfolder,$cfg,$BUILDLOGFILE))) {
             my @content_in_array = split("\n",uncompress_to_scalar(catfile($inputfolder,$cfg,$BUILDLOGFILE)));
             $data{cfg}{$cfg}{logdata} = get_log_data(@content_in_array);
@@ -775,6 +804,96 @@ sub read_build_data
             $backup_time = epoch_s_to_iso(get_modify_time(catfile($inputfolder,$cfg,$BUILDLOGFILE)));
         }
 
+        # uncompress the test-logs.zip file and gather all the xml data from it.
+        my $zip = Archive::Zip->new();
+        if (check_exists_and_openable(catfile($inputfolder,$cfg,$XMLARCHIVE))) {
+            unless ( $zip->read( catfile($inputfolder,$cfg,$XMLARCHIVE) ) == AZ_OK ) {
+                die "Unable to open zip file!";
+            }
+            foreach my $testset_project (keys %{$data{cfg}{$cfg}{testresults}{all_tests}}) {
+                foreach my $testsetname (keys %{$data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}}) {
+                    if ($testsetname eq "") {
+                        print "WARNING: Testset name is empty\n";
+                        $warning_count++;
+                    } else {
+                        ## (these test prints are disabled as they produce huge amount of lines which may make the output unreadable; uncomment when needed in testing)
+                        ## print "We are looking for '$data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{xmllogfile}' inside the zip file '". catfile($inputfolder,$cfg,$XMLARCHIVE)."'\n" if $VERBOSE;
+                        my $xml_file = $data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{xmllogfile};
+                        next if ($xml_file eq "");      # xml file may not be available for all testsets (e.g. for cmake)
+
+                        my $content_in_xml = $zip->contents("test-logs/$data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{xmllogfile}");
+                        my $parser = XML::LibXML->new();
+                        my $xml;
+                        eval {
+                            $xml = $parser->parse_string($content_in_xml);
+                        };
+                        if ($@) {
+                            print "WARNING: Cannot read XML from \"$data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{xmllogfile}\" in testset \"$testsetname\"\n";
+                            $warning_count++;
+                            next;
+                        }
+                        foreach my $tc ($xml->findnodes('/TestCase')){
+                            foreach my $tf ($tc->findnodes('TestFunction')){
+                                my $tfname = $tf->findvalue('./@name');
+                                my $duration = 0;
+                                my $result = "";
+                                foreach my $d ($tf->findnodes('Duration')) {
+                                    $duration = $d->findvalue('./@msecs');
+                                }
+                                ## print "$tfname ($duration ms)\n" if $VERBOSE;
+                                $data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{testfunctions}{$tfname}{duration} = ceil($duration/100);      # save duration in deciseconds (in milliseconds in the xml)
+
+                                # get testfunction or testrow result from Incident node
+                                foreach my $incident ($tf->findnodes('Incident')) {
+                                    $result = $incident->findvalue('./@type');
+                                    next if ($result eq "xfail");                           # Do not store "xfail", instead look for the 2nd incident that says 'pass'
+                                    my $testrowNameI  = $incident->find('DataTag');
+                                    if ($testrowNameI eq "") {                              # no DataTag -> result is for the testfunction
+                                        $data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{testfunctions}{$tfname}{result} = $result;
+                                        ## print "<i> testfunction $tfname: $result\n" if $VERBOSE;
+                                    } else {                                                # DataTag(s) -> results are for testrows
+                                        if (defined $data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{testfunctions}{$tfname}{DataTags}{$testrowNameI}{result}) {
+                                            print "INFO: Duplicate testrow $testset_project - $testsetname - $tfname - \"$testrowNameI\" " .
+                                                "($result - $data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{testfunctions}{$tfname}{DataTags}{$testrowNameI}{result})\n" if $INFOMSG;
+                                            $info_count++;
+                                        }
+                                        $data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{testfunctions}{$tfname}{DataTags}{$testrowNameI}{result} = $result;
+                                        ## print "<i> testrow $testrowNameM: $result\n" if $VERBOSE;
+                                    }
+
+                                }
+                                # get testfunction or testrow result from Message node ('skip' only)
+                                if ($result eq "") {
+                                    foreach my $message ($tf->findnodes('Message')) {
+                                        next if ($message->findvalue('./@type') ne "skip"); # only "skip" result saved (others may be e.g. "qwarn" or "qdebug")
+                                        $result = $message->findvalue('./@type');
+                                        # if the test functions has datatags, we will collect all the results as separate ones
+                                        my $testrowNameM  = $message->find('DataTag');
+                                        if ($testrowNameM eq "") {                          # no DataTag -> result is for the testfunction
+                                            $data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{testfunctions}{$tfname}{result} = $result;
+                                            ## print "<m> testfunction $tfname: $result\n" if $VERBOSE;
+                                        } else {                                            # DataTag(s) -> results are for testrows
+                                            if (defined $data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{testfunctions}{$tfname}{DataTags}{$testrowNameM}{result}) {
+                                                print "INFO: Duplicate testrow $testset_project - $testsetname - $tfname - \"$testrowNameM\" " .
+                                                    "($result - $data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{testfunctions}{$tfname}{DataTags}{$testrowNameM}{result})\n" if $INFOMSG;
+                                                $info_count++;
+                                            }
+                                            $data{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$testsetname}{testfunctions}{$tfname}{DataTags}{$testrowNameM}{result} = $result;
+                                            ## print "<m> testrow $testrowNameM: $result\n" if $VERBOSE;
+                                        }
+                                    }
+                                }
+                                if ($result eq "") {
+                                    print "WARNING: No result found for \"$tfname\" in \"$testsetname\" (in $cfg).\n" if ($result eq "");
+                                    $warning_count++;
+                                }
+                            } # end of foreach $tf
+                        } # end of foreach $tc
+                    } # end of if else
+                } # end of foreach $testsetname
+            } # end of foreach $testset_project
+        } # end of if check_exists_and_openable
+
         my $cfg_timestamp = epoch_ms_to_iso($runhash->{timestamp});
         $data{cfg}{$cfg}{builddata}{TIMESTAMP} = $cfg_timestamp ? $cfg_timestamp : $backup_time;
         $data{cfg}{$cfg}{builddata}{DURATION} = ms_to_hms($runhash->{duration});
@@ -815,7 +934,7 @@ sub exists_in_array
 
 sub get_log_data
 {
-    print "Getting general data from log files.\n";
+    print "Reading general data from log files.\n";
     my @filecontent = @_;
     my %logdata;
     $logdata{project} = getdata(\@filecontent, qr/^Started by upstream project "(.*)" build number \d+$/);
@@ -860,14 +979,14 @@ sub parse_testpath_from_path {
 
 sub get_test_results
 {
-    print "Getting test results.\n";
+    print "Reading test results.\n";
     my @filecontent = @_;
 
     my $RESULTPARTSTR = qr/=== Timing: =================== TEST RUN COMPLETED! ============================/;
     my $RESULTPARTSTR2 = qr/=== Failures: ==================================================================/;
     my $RESULTPARTSTR3 = qr/=== Totals: .*=/;
 
-    my $phase = 0;
+    my $step = 0;
     my $autotest = 0;
     my %testresults;
     my $total_autotests = 0;
@@ -876,70 +995,98 @@ sub get_test_results
     my $testsetname = "";
     my $testsetmodule = "";
     my $testsetpath = "";
+    my $testsetxmllogfilename = "";
     foreach my $line (@filecontent) {
         $line =~ s/[\n|\r]$//g;
-        $phase = 1 if ($line =~ m/^$RESULTPARTSTR$/);
-        $phase = 2 if ($line =~ m/^$RESULTPARTSTR2$/);
+        $step = 1 if ($line =~ m/^$RESULTPARTSTR$/);
+        $step = 2 if ($line =~ m/^$RESULTPARTSTR2$/);
         $autotest = 1 if ($line =~ m/#=#.*?#=#\s\>(.*)$/);
         $autotest = 0 if ($line =~ m/#=#.*?#=#\s\<(.*)\s#=# Elapsed (\d+) second\(s\).$/);
 
         if (1 == $autotest) {
-            if ($line =~ m/^QtQA::App::TestRunner: begin (.*?)\s@\s(.*):\s\[/) {
+            my $testdatabegin = 0;
+            # detect the test begin line, different variations below (you can verify e.g. with http://rubular.com/ )
+            # QtQA::App::TestRunner: begin qml @ /work/build/qtwebchannel/tests/auto/qml: [./qml] [-silent] [-o] [/work/ci/qt/qt5/dev/_artifacts/test-logs/qml-testresults-00.xml,xml] [-o] [-,txt] [-import] [/work/build/qtwebchannel/tests/auto/qml/../../../qml] [-import]
+            # QtQA::App::TestRunner: begin tst_qmlvisual @ C:\work\build\qtquick1\tests\auto\declarative\qmlvisual: [release\tst_qmlvisual.exe] [-silent] [-o] [C:\work\ci\qt\qt5\5.5\_artifacts\test-logs\tst_qmlvisual.exe-testresults-00.xml,xml] [-o] [-,txt]
+            if ($line =~ m/^QtQA::App::TestRunner: begin (.*?)\s@\s(.*):\s\[.*[\/\\](.*),xml/) {
+                $testsetname = $1;
+                $testsetmodule = parse_module_from_path($2);
+                $testsetpath = parse_testpath_from_path($2);
+                $testsetxmllogfilename = $3;
+                $testdatabegin = 1;
+            }
+            # QtQA::App::TestRunner: begin cmake (qtxmlpatterns) @ /work/build/qtxmlpatterns/tests/auto/cmake/build: [ctest] [--output-on-failure]
+            if (!$testdatabegin and $line =~ m/^QtQA::App::TestRunner: begin (.*?)\s@\s(.*):/) {
                 $testsetname = $1;
                 $testsetmodule = parse_module_from_path($2);
                 $testsetpath = parse_testpath_from_path($2);
+                $testsetxmllogfilename = "";
+                $testdatabegin = 1;
+            }
+            if ($testdatabegin) {
+                if (!defined $testsetname or $testsetname eq "") {
+                    warn "ERROR: Testset name empty or not defined";
+                    $error_count++;
+                    next;
+                }
                 # testsets can be skipped if needed (e.g. unsupported format)
                 #next if ($testsetname =~ m/license/);
                 #next if ($testsetname =~ m/tst_headers/);
                 #next if ($testsetname =~ m/tst_bic/);
                 $testdata = 1;
                 # set initial values for a new test set
-                if (!$testresults{all_tests}{$testsetname}) {
-                    $testresults{all_tests}{$testsetname}{module} = $testsetmodule;
-                    $testresults{all_tests}{$testsetname}{path} = $testsetpath;
-                    $testresults{all_tests}{$testsetname}{runs} = 0;
-                    $testresults{all_tests}{$testsetname}{passed} = 0;
-                    $testresults{all_tests}{$testsetname}{failed} = 0;
-                    $testresults{all_tests}{$testsetname}{skipped} = 0;
-                    $testresults{all_tests}{$testsetname}{blacklisted} = 0;
-                    $testresults{all_tests}{$testsetname}{insignificant} = 0;
-                    $testresults{all_tests}{$testsetname}{duration} = 0;
-                    $testresults{all_tests}{$testsetname}{overall} = 1;             # by default we assume that tests will fail ;)
+                if (!$testresults{all_tests}{$testsetmodule}{$testsetname}) {
+                    $testresults{all_tests}{$testsetmodule}{$testsetname}{path} = $testsetpath;
+                    $testresults{all_tests}{$testsetmodule}{$testsetname}{xmllogfile} = $testsetxmllogfilename;
+                    $testresults{all_tests}{$testsetmodule}{$testsetname}{runs} = 0;
+                    $testresults{all_tests}{$testsetmodule}{$testsetname}{passed} = 0;
+                    $testresults{all_tests}{$testsetmodule}{$testsetname}{failed} = 0;
+                    $testresults{all_tests}{$testsetmodule}{$testsetname}{skipped} = 0;
+                    $testresults{all_tests}{$testsetmodule}{$testsetname}{blacklisted} = 0;
+                    $testresults{all_tests}{$testsetmodule}{$testsetname}{insignificant} = 0;
+                    $testresults{all_tests}{$testsetmodule}{$testsetname}{duration} = 0;
+                    $testresults{all_tests}{$testsetmodule}{$testsetname}{overall} = 1;             # by default we assume that tests will fail ;)
                 }
-                $testresults{all_tests}{$testsetname}{runs}++;
+                $testresults{all_tests}{$testsetmodule}{$testsetname}{runs}++;
                 $total_autotests++;
-            } elsif ($line =~ m/^QtQA::App::TestRunner: test failed, running again to see if it is flaky/) {
-                $testdata = 1;
-                $testresults{all_tests}{$testsetname}{runs}++;
+            } elsif ($testdata and $line =~ m/^QtQA::App::TestRunner: test failed, running again to see if it is flaky/) {
+                $testresults{all_tests}{$testsetmodule}{$testsetname}{runs}++;
             } elsif ($testdata and $line =~ m/^Totals: (\d+) passed, (\d+) failed, (\d+) skipped, (\d+) blacklisted/) {
-                $testresults{all_tests}{$testsetname}{passed} = $1;
-                $testresults{all_tests}{$testsetname}{failed} = $2;
-                $testresults{all_tests}{$testsetname}{skipped} = $3;
-                $testresults{all_tests}{$testsetname}{blacklisted} = $4;
+                $testresults{all_tests}{$testsetmodule}{$testsetname}{passed} = $1;
+                $testresults{all_tests}{$testsetmodule}{$testsetname}{failed} = $2;
+                $testresults{all_tests}{$testsetmodule}{$testsetname}{skipped} = $3;
+                $testresults{all_tests}{$testsetmodule}{$testsetname}{blacklisted} = $4;
             } elsif ($testdata and $line =~ m/^\d+\% tests passed, (\d+) test(?:s)? failed out of (\d+)/) {
-                $testresults{all_tests}{$testsetname}{passed} = $2-$1;
-                $testresults{all_tests}{$testsetname}{failed} = $1;
+                $testresults{all_tests}{$testsetmodule}{$testsetname}{passed} = $2-$1;
+                $testresults{all_tests}{$testsetmodule}{$testsetname}{failed} = $1;
             } elsif ($testdata and $line =~ m/^QtQA::App::TestRunner: Process exited due to signal (\d+); dumped core/) {
-                $testresults{all_tests}{$testsetname}{overall} = $1;                # use exit code
+                $testresults{all_tests}{$testsetmodule}{$testsetname}{overall} = $1;                # use exit code
             } elsif ($testdata and $line =~ m/^QtQA::App::TestRunner: end .*: (.*) seconds, exit code (\d+)/) {
-                $testresults{all_tests}{$testsetname}{duration} = ceil($1*10);      # duration in deciseconds
-                $testresults{all_tests}{$testsetname}{overall} = $2;                # passed (0)
+                $testresults{all_tests}{$testsetmodule}{$testsetname}{duration} = ceil($1*10);      # save duration in deciseconds (in seconds in the log)
+                $testresults{all_tests}{$testsetmodule}{$testsetname}{overall} = $2;                # passed (0)
                 $testdata = 0;
             }
         }
 
-        if (2 == $phase) {
+        # checking from the end of the log if any tests are marked as insignificant
+        # TODO: the testset module is not available on this line (so all testsets with the same name will be marked here), insignificance should be checked from the line after the 'end testsetname'
+        if (2 == $step) {
             if ($line =~ m/^\s{2}(.*?)\s*(\[insignificant\])*$/) {
-                print "Found test case '$1'\n" if $VERBOSE;
+                my $ts_name = $1;
                 if (!defined $2) {
-                    push (@{$testresults{failed_tests}}, $1);
+                    push (@{$testresults{failed_tests}}, $ts_name);
+                    print "Found failed test '$ts_name'.\n" if $VERBOSE;
                 } elsif ("[insignificant]" eq $2) {
-                    push (@{$testresults{insignificant_failed_tests}}, $1);
-                    if (exists $testresults{all_tests}{$1}) {
-                        $testresults{all_tests}{$1}{insignificant} = 1;
+                    push (@{$testresults{insignificant_failed_tests}}, $ts_name);
+                    print "Found failed insignificant test '$ts_name'.\n" if $VERBOSE;
+                    foreach my $ts_module (keys %{$testresults{all_tests}}) {
+                        if (exists $testresults{all_tests}{$ts_module}{$ts_name}) {
+                            $testresults{all_tests}{$ts_module}{$ts_name}{insignificant} = 1;
+                        }
                     }
                 } else {
-                    push (@{$testresults{unspecified_tests}}, $1);
+                    push (@{$testresults{unspecified_tests}}, $ts_name);
+                    print "Found unspecified test '$ts_name'.\n" if $VERBOSE;
                 }
             }
             else { last if ($line =~ m/^$RESULTPARTSTR3$/); }
@@ -951,7 +1098,7 @@ sub get_test_results
 
 sub get_phase_times
 {
-    print "Getting times for different phases.\n";
+    print "Reading times for different phases.\n";
     my @filecontent = @_;
 
     my $TIMESTR = qr/\w{3}\s\w{3}\s+\d+\s\d{2}:\d{2}:\d{2}\s\d{4}/;
@@ -988,33 +1135,49 @@ sub get_phase_times
     return (\%phasedata);
 }
 
+sub escape {
+    my $string = shift;
+    $string =~ s/\\/\\\\/g;        # escape '\' character (this one must be first)
+    $string =~ s/\"/\\\"/g;        # escape '"' character
+    return $string;
+}
+
+sub escape_sql_wildcard {
+    my $string = shift;
+    $string =~ s/\\/\\\\\\\\/g;   # escape '\' character (this one must be first)
+    $string =~ s/\"/\\\"/g;       # escape '"' character
+    $string =~ s/\%/\\\%/g;       # escape '%' character (wildcard in SQL)
+    $string =~ s/\_/\\\_/g;       # escape '_' character (wildcard in SQL)
+    return $string;
+}
+
 sub sql_connect
 {
     my $dbh;
-    print "Connecting to MySQL...\n";
+    print "Connecting to database...\n";
     $ENV{HOME} = $ENV{HOMEPATH} if ($^O =~ m/mswin32/i);
 
     # Connect to the database.
-    die "Can't access SQL configuration" if (!check_exists_and_openable ("$ENV{HOME}/.my.cnf"));
+    die "Can't access database configuration" if (!check_exists_and_openable ("$ENV{HOME}/.my.cnf"));
     my $dsn = "DBI:mysql:;mysql_read_default_file=$ENV{HOME}/.my.cnf";
     eval {
         $dbh = DBI->connect($dsn, undef, undef, {'RaiseError' => 1});
     };
     if ($@) {
-        die("Connection to SQL failed because $@");
+        die("Connection to database failed because $@");
     }
     return $dbh;
 }
 
 sub sql_disconnect
 {
-    print "Disconnecting from MySQL...\n";
+    print "Disconnecting from database...\n";
     my $dbh = shift;
     eval {
         $dbh->disconnect();
     };
     if ($@) {
-        die("Disconnection from SQL failed because $@");
+        die("Disconnection from database failed because $@");
     }
 }
 
@@ -1059,6 +1222,8 @@ sub sql_drop_tables
 sub sql_create_tables
 {
     my $dbh = shift;
+    my %options = %{(shift)};
+    my $output = $options{sqloutput};
     print "Creating new tables (if they do not exist yet).\n";
 
     $dbh->{AutoCommit} = 0;  # enable transactions, if possible
@@ -1072,14 +1237,14 @@ sub sql_create_tables
                 logs_current          INT UNSIGNED          NOT NULL,
                 logs_total            INT UNSIGNED          NOT NULL
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         # gives the db_status table initial values, since only one row is used in this table
         if ("0E0" eq $dbh->do ("SELECT * FROM db_status")) {
             $dbh->do (
                 "INSERT IGNORE INTO db_status (refreshed, refresh_in_progress, logs_current, logs_total)
                     VALUES ('2015-05-01 00:00', 0, 0, 0);"
-            );
+            ) if !$output;
         }
 
         $dbh->do (
@@ -1089,7 +1254,7 @@ sub sql_create_tables
                 UNIQUE INDEX unique_branch (name),
                 CONSTRAINT branch_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS compiler (
@@ -1098,7 +1263,7 @@ sub sql_create_tables
                 UNIQUE INDEX unique_compiler (compiler),
                 CONSTRAINT compiler_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS conf (
@@ -1112,7 +1277,7 @@ sub sql_create_tables
                 UNIQUE INDEX unique_conf (name),
                 CONSTRAINT conf_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS conf_run (
@@ -1127,11 +1292,16 @@ sub sql_create_tables
                 duration              TIME                  NOT NULL,
                 CONSTRAINT conf_run_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
-        $dbh->do (
-            "CREATE INDEX by_project_run ON conf_run (project_run_id DESC, result)"
-        );
+        # create index if not exists (Note: MariaDB versions from 2015 would support "CREATE INDEX IF NOT EXISTS")
+        if ("0E0" eq $dbh->do ("SELECT INDEX_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS
+            WHERE TABLE_CATALOG = \"def\" AND TABLE_SCHEMA = DATABASE() AND
+            INDEX_NAME = \"by_project_run\" AND TABLE_NAME = \"conf_run\"")) {
+            $dbh->do (
+                "CREATE INDEX by_project_run ON conf_run (project_run_id DESC, result)"
+            ) if !$output;
+        }
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS phase (
@@ -1140,7 +1310,7 @@ sub sql_create_tables
                 UNIQUE INDEX unique_phase (name),
                 CONSTRAINT phase_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS phase_run (
@@ -1151,7 +1321,7 @@ sub sql_create_tables
                 end                   TIMESTAMP             NOT NULL,
                 CONSTRAINT phase_run_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS platform (
@@ -1162,7 +1332,7 @@ sub sql_create_tables
                 UNIQUE INDEX unique_platform (os,os_version,arch),
                 CONSTRAINT platform_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS project (
@@ -1171,7 +1341,7 @@ sub sql_create_tables
                 UNIQUE INDEX unique_project (name),
                 CONSTRAINT project_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS project_run (
@@ -1186,15 +1356,23 @@ sub sql_create_tables
                 UNIQUE INDEX unique_project_run (project_id,branch_id,state_id,build_key),
                 CONSTRAINT project_run_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
-        $dbh->do (
-            "CREATE INDEX by_timestamp ON project_run (timestamp, state_id, project_id)"
-        );
-
-        $dbh->do (
-            "CREATE INDEX by_state ON project_run (state_id, project_id, timestamp)"
-        );
+        # create index if not exists (Note: MariaDB versions from 2015 would support "CREATE INDEX IF NOT EXISTS")
+        if ("0E0" eq $dbh->do ("SELECT INDEX_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS
+            WHERE TABLE_CATALOG = \"def\" AND TABLE_SCHEMA = DATABASE() AND
+            INDEX_NAME = \"by_timestamp\" AND TABLE_NAME = \"project_run\"")) {
+            $dbh->do (
+                "CREATE INDEX by_timestamp ON project_run (timestamp, state_id, project_id)"
+            ) if !$output;
+        }
+        if ("0E0" eq $dbh->do ("SELECT INDEX_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS
+            WHERE TABLE_CATALOG = \"def\" AND TABLE_SCHEMA = DATABASE() AND
+            INDEX_NAME = \"by_state\" AND TABLE_NAME = \"project_run\"")) {
+            $dbh->do (
+                "CREATE INDEX by_state ON project_run (state_id, project_id, timestamp)"
+            ) if !$output;
+        }
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS state (
@@ -1203,17 +1381,17 @@ sub sql_create_tables
                 UNIQUE INDEX unique_state (name),
                 CONSTRAINT state_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS testfunction (
                 id                    MEDIUMINT UNSIGNED    NOT NULL  AUTO_INCREMENT,
                 testset_id            SMALLINT UNSIGNED     NOT NULL,
-                name                  VARCHAR(50)           NOT NULL,
+                name                  VARCHAR(100)          CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
                 UNIQUE INDEX unique_testfunction (testset_id,name),
                 CONSTRAINT testfunction_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS testfunction_run (
@@ -1224,17 +1402,27 @@ sub sql_create_tables
                 duration              SMALLINT UNSIGNED     NOT NULL,
                 CONSTRAINT testfunction_run_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
+        # create index if not exists (Note: MariaDB versions from 2015 would support "CREATE INDEX IF NOT EXISTS")
+        if ("0E0" eq $dbh->do ("SELECT INDEX_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS
+            WHERE TABLE_CATALOG = \"def\" AND TABLE_SCHEMA = DATABASE() AND
+            INDEX_NAME = \"by_testset_run\" AND TABLE_NAME = \"testfunction_run\"")) {
+            $dbh->do (
+                "CREATE INDEX by_testset_run ON testfunction_run (testset_run_id DESC, result, testfunction_id)"
+            ) if !$output;
+        }
+
+        # must not use UNIQUE for testrow.name as it may contain trailing spaces
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS testrow (
                 id                    MEDIUMINT UNSIGNED    NOT NULL  AUTO_INCREMENT,
                 testfunction_id       MEDIUMINT UNSIGNED    NOT NULL,
-                name                  VARCHAR(100)          NOT NULL,
-                UNIQUE INDEX unique_testdata (testfunction_id,name),
+                name                  VARCHAR(500)          CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+                INDEX testrow (testfunction_id,name),
                 CONSTRAINT testrow_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS testrow_run (
@@ -1243,7 +1431,16 @@ sub sql_create_tables
                 result                ENUM('pass','fail','xpass','xfail','skip','bpass','bfail','bxpass','bxfail','bskip')    NOT NULL,
                 CONSTRAINT testrow_run_pk PRIMARY KEY (testrow_id,testfunction_run_id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
+
+        # create index if not exists (Note: MariaDB versions from 2015 would support "CREATE INDEX IF NOT EXISTS")
+        if ("0E0" eq $dbh->do ("SELECT INDEX_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS
+            WHERE TABLE_CATALOG = \"def\" AND TABLE_SCHEMA = DATABASE() AND
+            INDEX_NAME = \"by_testfunction_run\" AND TABLE_NAME = \"testrow_run\"")) {
+            $dbh->do (
+                "CREATE INDEX by_testfunction_run ON testrow_run (testfunction_run_id DESC, result, testrow_id)"
+            ) if !$output;
+        }
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS testset (
@@ -1253,7 +1450,7 @@ sub sql_create_tables
                 UNIQUE INDEX unique_testset (project_id,name),
                 CONSTRAINT testset_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
+        ) if !$output;
 
         $dbh->do (
             "CREATE TABLE IF NOT EXISTS testset_run (
@@ -1269,19 +1466,30 @@ sub sql_create_tables
                 total_blacklisted     SMALLINT UNSIGNED     NOT NULL,
                 CONSTRAINT testset_run_pk PRIMARY KEY (id)
             ) ENGINE MyISAM"
-        );
-
-        $dbh->do (
-            "CREATE INDEX by_conf_run ON testset_run (conf_run_id DESC, run, result)"
-        );
-
-        $dbh->do (
-            "CREATE INDEX by_testset ON testset_run (testset_id, result)"
-        );
+        ) if !$output;
 
-        $dbh->do (
-            "CREATE INDEX by_run ON testset_run (run, result)"
-        );
+        # create index if not exists (Note: MariaDB versions from 2015 would support "CREATE INDEX IF NOT EXISTS")
+        if ("0E0" eq $dbh->do ("SELECT INDEX_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS
+            WHERE TABLE_CATALOG = \"def\" AND TABLE_SCHEMA = DATABASE() AND
+            INDEX_NAME = \"by_conf_run\" AND TABLE_NAME = \"testset_run\"")) {
+            $dbh->do (
+                "CREATE INDEX by_conf_run ON testset_run (conf_run_id DESC, run, result)"
+            ) if !$output;
+        }
+        if ("0E0" eq $dbh->do ("SELECT INDEX_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS
+            WHERE TABLE_CATALOG = \"def\" AND TABLE_SCHEMA = DATABASE() AND
+            INDEX_NAME = \"by_testset\" AND TABLE_NAME = \"testset_run\"")) {
+            $dbh->do (
+                "CREATE INDEX by_testset ON testset_run (testset_id, result)"
+            ) if !$output;
+        }
+        if ("0E0" eq $dbh->do ("SELECT INDEX_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS
+            WHERE TABLE_CATALOG = \"def\" AND TABLE_SCHEMA = DATABASE() AND
+            INDEX_NAME = \"by_run\" AND TABLE_NAME = \"testset_run\"")) {
+            $dbh->do (
+                "CREATE INDEX by_run ON testset_run (run, result)"
+            ) if !$output;
+        }
 
         $dbh->commit;   # commit the changes if we get this far
     };
@@ -1300,6 +1508,7 @@ sub sql
     my $dbh = shift;
     my %options = %{(shift)};
     my %datahash = %{(shift)};
+    my %db_status = %{(shift)};
     my $output = $options{sqloutput};
 
     # split the full project name into project (anything before first "_"), branch (anything between first and last "_") and state (anything after last "_")
@@ -1312,138 +1521,115 @@ sub sql
     if (defined $options{reload}) {
         eval {
             # if 'reload' is defined in options, remove possible data from sql database before storing new data
-            print "Deleting old data from database.\n";
-
-            my $query =
-                "DELETE FROM phase_run
-                    WHERE conf_run_id IN (
-                        SELECT conf_run.id
-                            FROM conf_run
-                                INNER JOIN project_run ON conf_run.project_run_id = project_run.id
-                                INNER JOIN project ON project_run.project_id = project.id
-                                INNER JOIN branch ON project_run.branch_id = branch.id
-                                INNER JOIN state ON project_run.state_id = state.id
-                            WHERE project.name = \"$projectname\" AND
-                                branch.name = \"$branchname\" AND
-                                state.name = \"$statename\" AND
-                                project_run.build_key = $datahash{BUILD_NUMBER}
-                    )";
-            print "$query\n" if $VERBOSE or $output;
-            $dbh->do ($query) or print "removal of old data in phase_run failed: $!\n" if !$output;
-
-            $query =
-                "DELETE FROM testrow_run
-                    WHERE testfunction_run_id IN (
-                        SELECT testfunction_run.id
-                            FROM testfunction_run
-                                INNER JOIN testset_run ON testfunction_run.testset_run_id = testset_run.id
-                                INNER JOIN conf_run ON testset_run.conf_run_id = conf_run.id
-                                INNER JOIN project_run ON conf_run.project_run_id = project_run.id
-                                INNER JOIN project ON project_run.project_id = project.id
-                                INNER JOIN branch ON project_run.branch_id = branch.id
-                                INNER JOIN state ON project_run.state_id = state.id
-                            WHERE project.name = \"$projectname\" AND
-                                branch.name = \"$branchname\" AND
-                                state.name = \"$statename\" AND
-                                project_run.build_key = $datahash{BUILD_NUMBER}
-                    )";
-            print "$query\n" if $VERBOSE or $output;
-            $dbh->do ($query) or print "removal of old data in testrow_run failed: $!\n" if !$output;
-
-            $query =
-                "DELETE FROM testfunction_run
-                    WHERE testset_run_id IN (
-                        SELECT testset_run.id
-                            FROM testset_run
-                                INNER JOIN conf_run ON testset_run.conf_run_id = conf_run.id
-                                INNER JOIN project_run ON conf_run.project_run_id = project_run.id
-                                INNER JOIN project ON project_run.project_id = project.id
-                                INNER JOIN branch ON project_run.branch_id = branch.id
-                                INNER JOIN state ON project_run.state_id = state.id
-                            WHERE project.name = \"$projectname\" AND
-                                branch.name = \"$branchname\" AND
-                                state.name = \"$statename\" AND
-                                project_run.build_key = $datahash{BUILD_NUMBER}
-                    )";
-            print "$query\n" if $VERBOSE or $output;
-            $dbh->do ($query) or print "removal of old data in testfunction_run failed: $!\n" if !$output;
-
-            $query =
-                "DELETE FROM testset_run
-                    WHERE conf_run_id IN (
-                        SELECT conf_run.id
-                            FROM conf_run
-                                INNER JOIN project_run ON conf_run.project_run_id = project_run.id
-                                INNER JOIN project ON project_run.project_id = project.id
-                                INNER JOIN branch ON project_run.branch_id = branch.id
-                                INNER JOIN state ON project_run.state_id = state.id
-                            WHERE project.name = \"$projectname\" AND
-                            branch.name = \"$branchname\" AND
-                            state.name = \"$statename\" AND
-                            project_run.build_key = $datahash{BUILD_NUMBER}
-                    )";
-            print "$query\n" if $VERBOSE or $output;
-            $dbh->do ($query) or print "removal of old data in testset_run failed: $!\n" if !$output;
-
-            $query =
-                "DELETE FROM conf_run
-                    WHERE project_run_id IN (
-                        SELECT project_run.id
-                            FROM project_run
-                                WHERE project_run.project_id = (SELECT id FROM project WHERE name = \"$projectname\") AND
-                                    project_run.branch_id = (SELECT id FROM branch WHERE name = \"$branchname\") AND
-                                    project_run.state_id = (SELECT id FROM state WHERE name = \"$statename\") AND
-                                    project_run.build_key = $datahash{BUILD_NUMBER}
-                    )";
-            print "$query\n" if $VERBOSE or $output;
-            $dbh->do ($query) or print "removal of old data in conf_run failed: $!\n" if !$output;
-
-            $query =
-                "DELETE FROM project_run
-                    WHERE project_id = (SELECT project.id FROM project WHERE project.name = \"$projectname\") AND
+            print "Deleting old data from database...\n";
+            my $project_run_id;
+            my $select;
+            my $delete;
+
+            # get project_run.id
+            $select =
+                "SELECT id FROM project_run
+                    WHERE
+                        project_id = (SELECT project.id FROM project WHERE project.name = \"$projectname\") AND
                         branch_id = (SELECT branch.id FROM branch WHERE  branch.name = \"$branchname\") AND
                         state_id = (SELECT state.id FROM state WHERE state.name = \"$statename\") AND
                         build_key = $datahash{BUILD_NUMBER}";
-            print "$query\n" if $VERBOSE or $output;
-            $dbh->do ($query) or print "removal of old data in project_run failed: $!\n" if !$output;
+            my $ids = $dbh->selectall_arrayref($select, { Slice => {} });
+            foreach my $row (@$ids) {
+                $project_run_id = $row->{id};
+            }
+
+            # get conf_run.ids
+            $select = "SELECT id FROM conf_run WHERE project_run_id = $project_run_id";
+            $ids = $dbh->selectall_arrayref($select, { Slice => {} });
+            foreach my $row (@$ids) {
+                my $conf_run_id = $row->{id};
+
+                # get testset_run.ids
+                $select = "SELECT id FROM testset_run WHERE conf_run_id = $conf_run_id";
+                my $ids = $dbh->selectall_arrayref($select, { Slice => {} });
+                foreach my $row (@$ids) {
+                    my $testset_run_id = $row->{id};
+
+                    # get testfunction_run.ids
+                    $select = "SELECT id FROM testfunction_run WHERE testset_run_id = $testset_run_id";
+                    my $ids = $dbh->selectall_arrayref($select, { Slice => {} });
+                    foreach my $row (@$ids) {
+                        my $testfunction_run_id = $row->{id};
+
+                        # delete testrow_runs
+                        $delete = "DELETE FROM testrow_run WHERE testfunction_run_id = $testfunction_run_id";
+                        print "$delete\n" if $VERBOSE or $output;
+                        $dbh->do ($delete) or print "removal of old data in testrow_run failed: $!\n" if !$output;
+                    }
+
+                    # delete testfunction_runs
+                    $delete = "DELETE FROM testfunction_run WHERE testset_run_id = $testset_run_id";
+                    print "$delete\n" if $VERBOSE or $output;
+                    $dbh->do ($delete) or print "removal of old data in testfunction_run failed: $!\n" if !$output;
+                }
+
+                # delete testset_runs
+                $delete = "DELETE FROM testset_run WHERE conf_run_id = $conf_run_id";
+                print "$delete\n" if $VERBOSE or $output;
+                $dbh->do ($delete) or print "removal of old data in testset_run failed: $!\n" if !$output;
+
+                # delete phase_runs
+                $delete = "DELETE FROM phase_run WHERE conf_run_id = $conf_run_id";
+                print "$delete\n" if $VERBOSE or $output;
+                $dbh->do ($delete) or print "removal of old data in phase_run failed: $!\n" if !$output;
+            }
+
+            # delete conf_runs
+            $delete = "DELETE FROM conf_run WHERE project_run_id = $project_run_id";
+            print "$delete\n" if $VERBOSE or $output;
+            $dbh->do ($delete) or print "removal of old data in conf_run failed: $!\n" if !$output;
+
+            # delete project_run
+            $delete = "DELETE FROM project_run WHERE id = $project_run_id";
+            print "$delete\n" if $VERBOSE or $output;
+            $dbh->do ($delete) or print "removal of old data in project_run failed: $!\n" if !$output;
 
             $dbh->commit;   # commit the changes if we get this far
         };
         if ($@) {
-            warn "Transaction aborted because $@";
+            warn "ERROR: Transaction aborted because $@";
+            $error_count++;
             eval { $dbh->rollback };
         } else {
             print "Data deleted.\n";
         }
     }
 
-    print "Storing data to database.\n";
+    print "Storing data to database...\n";
 
     eval {
         # Default timestamp and duration to zero if cannot be read from the log
         my $timestamp = $datahash{TIMESTAMP} ? "\"$datahash{TIMESTAMP}\"" : "0";
         my $duration = $datahash{DURATION} ? "\"$datahash{DURATION}\"" : "0";
 
+        ###
         # insert data into project tables
+        #
 
         if ("0E0" eq $dbh->do ("SELECT name FROM project WHERE name = \"$projectname\"")) {
             my $query =
                 "INSERT INTO project (name) VALUES (\"$projectname\")";
-            print OUTPUT "$query\n" if $VERBOSE or $output;
+            print OUTPUT "$query\n\n" if $VERBOSE or $output;
             $dbh->do ($query) or print "insert into project failed: $!\n" if !$output;
         }
 
         if ("0E0" eq $dbh->do ("SELECT name FROM branch WHERE name = \"$branchname\"")) {
             my $query =
                 "INSERT INTO branch (name) VALUES (\"$branchname\")";
-            print OUTPUT "$query\n" if $VERBOSE or $output;
+            print OUTPUT "$query\n\n" if $VERBOSE or $output;
             $dbh->do ($query) or print "insert into branch failed: $!\n" if !$output;
         }
 
         if ("0E0" eq $dbh->do ("SELECT name FROM state WHERE name = \"$statename\"")) {
             my $query =
                 "INSERT INTO state (name) VALUES (\"$statename\")";
-            print OUTPUT "$query\n" if $VERBOSE or $output;
+            print OUTPUT "$query\n\n" if $VERBOSE or $output;
             $dbh->do ($query) or print "insert into state failed: $!\n" if !$output;
         }
 
@@ -1454,14 +1640,23 @@ sub sql
                     WHERE project.name = \"$projectname\" AND
                         branch.name = \"$branchname\" AND
                         state.name = \"$statename\"";
-        print OUTPUT "$query\n" if $VERBOSE or $output;
+        print OUTPUT "$query\n\n" if $VERBOSE or $output;
         $dbh->do ($query) or print "insert into project_run failed: $!\n" if !$output;
 
-        #insert data into configuration tables
+        ###
+        # insert data into configuration tables
+        #
 
+        my $conf_counter = 0;
         foreach my $cfg (keys %{$datahash{cfg}}) {
+            $conf_counter++;
+            print "- $conf_counter/$conf_count: $cfg.\n";
+            $db_status{current} = $conf_counter;
+            $db_status{total} = $conf_count;
+            sql_update_progress($dbh, \%db_status);
             if (!exists $cfg_table{$cfg}) {
-                print OUTPUT "ERROR: Configuration '$cfg' not defined for this parser.\n";
+                print "ERROR: Configuration '$cfg' not defined for this parser.\n";
+                $error_count++;
                 next;
             }
             my $host_os           = $cfg_table{$cfg}{'host_os'};
@@ -1483,28 +1678,28 @@ sub sql
             if ("0E0" eq $dbh->do ("SELECT os, os_version, arch FROM platform WHERE os = \"$host_os\" AND os_version = \"$host_version\" AND arch = \"$host_arch\"")) {
                 my $query =
                     "INSERT INTO platform (os,os_version,arch) VALUES (\"$host_os\",\"$host_version\",\"$host_arch\")";
-                print OUTPUT "$query\n" if $VERBOSE or $output;
+                print OUTPUT "$query\n\n" if $VERBOSE or $output;
                 $dbh->do ($query) or print "insert into platform failed: $!\n" if !$output;
             }
 
             if ("0E0" eq $dbh->do ("SELECT os, os_version, arch FROM platform WHERE os = \"$target_os\" AND os_version = \"$target_version\" AND arch = \"$target_arch\"")) {
                 my $query =
                     "INSERT INTO platform (os,os_version,arch) VALUES (\"$target_os\",\"$target_version\",\"$target_arch\")";
-                print OUTPUT "$query\n" if $VERBOSE or $output;
+                print OUTPUT "$query\n\n" if $VERBOSE or $output;
                 $dbh->do ($query) or print "insert into platform failed: $!\n" if !$output;
             }
 
             if ("0E0" eq $dbh->do ("SELECT compiler FROM compiler WHERE compiler = \"$host_compiler\"")) {
                 my $query =
                     "INSERT INTO compiler (compiler) VALUES (\"$host_compiler\")";
-                print OUTPUT "$query\n" if $VERBOSE or $output;
+                print OUTPUT "$query\n\n" if $VERBOSE or $output;
                 $dbh->do ($query) or print "insert into compiler failed: $!\n" if !$output;
             }
 
             if ("0E0" eq $dbh->do ("SELECT compiler FROM compiler WHERE compiler = \"$target_compiler\"")) {
                 my $query =
                     "INSERT INTO compiler (compiler) VALUES (\"$target_compiler\")";
-                print OUTPUT "$query\n" if $VERBOSE or $output;
+                print OUTPUT "$query\n\n" if $VERBOSE or $output;
                 $dbh->do ($query) or print "insert into compiler failed: $!\n" if !$output;
             }
 
@@ -1530,73 +1725,258 @@ sub sql
                             project_run.branch_id = (SELECT id FROM branch WHERE name = \"$branchname\") AND
                             project_run.state_id = (SELECT id FROM state WHERE name = \"$statename\") AND
                             project_run.build_key =  $datahash{BUILD_NUMBER}";
-            print OUTPUT "$query\n" if $VERBOSE or $output;
+            print OUTPUT "$query\n\n" if $VERBOSE or $output;
             $dbh->do ($query) or print "insert into conf_run failed: $!\n" if !$output;
+            my $conf_run_id = $dbh->last_insert_id("", "", "conf_run", "");
 
-            #insert data into test tables
+            ###
+            # insert data into testset tables
+            #
 
             if (defined $datahash{cfg}{$cfg}{testresults}{all_tests}) {
-                foreach my $test (keys %{$datahash{cfg}{$cfg}{testresults}{all_tests}}) {
-
-                    # insert testset project first
-                    my $testset_project = $datahash{cfg}{$cfg}{testresults}{all_tests}{$test}{module};
-                    print "ERORR: $test doesn't have parent module set (in $cfg).\n" if (!defined $testset_project);
-                    if ("0E0" eq $dbh->do ("SELECT name FROM project WHERE name = \"$testset_project\"")) {
-                        my $query =
-                            "INSERT INTO project (name) VALUES (\"$testset_project\")";
-                        print OUTPUT "$query\n" if $VERBOSE or $output;
-                        $dbh->do ($query) or print "insert into project failed: $!\n" if !$output;
-                    }
-                    # insert testset connected to its parent project
-                    if ("0E0" eq $dbh->do ("SELECT testset.name FROM testset INNER JOIN project ON testset.project_id = project.id
-                        WHERE testset.name = \"$test\" AND project.name = \"$testset_project\"")) {
-                        my $query =
-                            "INSERT INTO testset (project_id, name) SELECT id, \"$test\" FROM project WHERE project.name = \"$testset_project\"";
-                        print OUTPUT "$query\n" if $VERBOSE or $output;
-                        $dbh->do ($query) or print "insert into testset failed: $!\n" if !$output;
-                    }
-
-                    my $testset_result;
-                    if ($datahash{cfg}{$cfg}{testresults}{all_tests}{$test}{overall} == 0) {
-                        $testset_result = $datahash{cfg}{$cfg}{testresults}{all_tests}{$test}{insignificant} ? "\"ipassed\"" : "\"passed\"";
-                    } else {
-                        $testset_result = $datahash{cfg}{$cfg}{testresults}{all_tests}{$test}{insignificant} ? "\"ifailed\"" : "\"failed\"";
-                    }
-                    # testset is connected to its parent project (e.g. QtConnectivity) while testset_run to project where run (e.g. Qt5)
-                    my $query =
-                        "INSERT INTO testset_run (testset_id, conf_run_id, run, result, duration, total_passed, total_failed, total_skipped, total_blacklisted)
-                            SELECT testset.id,
-                                    conf_run.id,
-                                    $datahash{cfg}{$cfg}{testresults}{all_tests}{$test}{runs},
-                                    $testset_result,
-                                    $datahash{cfg}{$cfg}{testresults}{all_tests}{$test}{duration},
-                                    $datahash{cfg}{$cfg}{testresults}{all_tests}{$test}{passed},
-                                    $datahash{cfg}{$cfg}{testresults}{all_tests}{$test}{failed},
-                                    $datahash{cfg}{$cfg}{testresults}{all_tests}{$test}{skipped},
-                                    $datahash{cfg}{$cfg}{testresults}{all_tests}{$test}{blacklisted}
-                                FROM testset, conf_run
-                                WHERE testset.name = \"$test\" AND
-                                testset.project_id = (SELECT id FROM project WHERE project.name = \"$testset_project\") AND
-                                conf_run.id = (
-                                    SELECT conf_run.id
-                                        FROM conf_run
-                                            INNER JOIN conf ON conf_run.conf_id = conf.id
-                                            INNER JOIN project_run ON conf_run.project_run_id = project_run.id
-                                            INNER JOIN project ON project_run.project_id = project.id
-                                            INNER JOIN branch ON project_run.branch_id = branch.id
-                                            INNER JOIN state ON project_run.state_id = state.id
-                                        WHERE conf.name = \"$cfg\" AND
-                                            project_run.build_key = $datahash{BUILD_NUMBER} AND
-                                            project.name = \"$projectname\" AND
-                                            branch.name = \"$branchname\" AND
-                                            state.name = \"$statename\" )";
-                    print "$cfg - $test\n" if $VERBOSE;
-                    print OUTPUT "$query\n" if $VERBOSE or $output;
-                    $dbh->do ($query) or print "insert into testset_run failed: $!\n" if !$output;
-                }
-            }
-
-            #insert data into phase tables
+                foreach my $testset_project (keys %{$datahash{cfg}{$cfg}{testresults}{all_tests}}) {
+                    foreach my $test (keys %{$datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}}) {
+
+                        if ($test eq "") {
+                            print "WARNING: Testset data not inserted into database because testset name is empty (in $cfg)\n";
+                            $warning_count++;
+                        } else {
+
+                            # insert testset project first if not yet in database
+                            if ("0E0" eq $dbh->do ("SELECT name FROM project WHERE name = \"$testset_project\"")) {
+                                my $query =
+                                    "INSERT INTO project (name) VALUES (\"$testset_project\")";
+                                print OUTPUT "$query\n\n" if $VERBOSE or $output;
+                                $dbh->do ($query) or print "insert into project failed: $!\n" if !$output;
+                            }
+
+                            # insert testset connected to its parent project if not yet in database
+                            if ("0E0" eq $dbh->do ("SELECT testset.name FROM testset INNER JOIN project ON testset.project_id = project.id
+                                WHERE testset.name = \"$test\" AND project.name = \"$testset_project\"")) {
+                                my $query =
+                                    "INSERT INTO testset (project_id, name) SELECT id, \"$test\" FROM project WHERE project.name = \"$testset_project\"";
+                                print OUTPUT "$query\n\n" if $VERBOSE or $output;
+                                $dbh->do ($query) or print "insert into testset failed: $!\n" if !$output;
+                            }
+
+                            # get testset project id for later use in INSERT queries
+                            my $select;
+                            my $testset_project_id;
+                            my $ids;
+                            $select = "SELECT id FROM project WHERE project.name = \"$testset_project\"";
+                            $ids = $dbh->selectall_arrayref($select, { Slice => {} });
+                            foreach my $row (@$ids) {
+                                $testset_project_id = $row->{id};
+                            }
+
+                            # insert testset_run; testset is connected to its parent project (e.g. QtConnectivity) while testset_run to project where run (e.g. Qt5)
+                            my $testset_result;
+                            if ($datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{overall} == 0) {
+                                $testset_result = $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{insignificant} ? "\"ipassed\"" : "\"passed\"";
+                            } else {
+                                $testset_result = $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{insignificant} ? "\"ifailed\"" : "\"failed\"";
+                            }
+                            my $query =
+                                "INSERT INTO testset_run (testset_id, conf_run_id, run, result, duration, total_passed, total_failed, total_skipped, total_blacklisted)
+                                    SELECT testset.id,
+                                            conf_run.id,
+                                            $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{runs},
+                                            $testset_result,
+                                            $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{duration},
+                                            $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{passed},
+                                            $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{failed},
+                                            $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{skipped},
+                                            $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{blacklisted}
+                                        FROM testset, conf_run
+                                        WHERE testset.name = \"$test\" AND
+                                            testset.project_id = $testset_project_id AND
+                                            conf_run.id = $conf_run_id";
+                            print "$cfg - $testset_project - $test\n" if $VERBOSE;
+                            print OUTPUT "$query\n\n" if $VERBOSE or $output;
+                            $dbh->do ($query) or print "insert into testset_run failed: $!\n" if !$output;
+                            my $testset_run_id = $dbh->last_insert_id("", "", "testset_run", "");
+
+                            ###
+                            # insert data into testfunction tables
+                            #
+
+                            my $testrow_run_insert_counter = 0;
+                            my $testrow_run_insert_values = "";
+
+                            # insert into testfunction table if not yet in database (testfunction connected to its parent testset)
+                            foreach my $testfunction (keys %{$datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{testfunctions}}) {
+                                if ("0E0" eq $dbh->do ("SELECT testfunction.name FROM testfunction
+                                    INNER JOIN testset ON testfunction.testset_id = testset.id
+                                    INNER JOIN project ON testset.project_id = project.id
+                                    WHERE testfunction.name = \"$testfunction\" AND testset.name = \"$test\" AND project.name = \"$testset_project\"")) {
+                                    my $query =
+                                        "INSERT INTO testfunction (testset_id, name)
+                                            SELECT id, \"$testfunction\" FROM testset
+                                                WHERE testset.name = \"$test\" AND
+                                                    testset.project_id = $testset_project_id";
+                                    print OUTPUT "$query\n\n" if $VERBOSE or $output;
+                                    $dbh->do ($query) or print "insert into testfunction failed: $!\n" if !$output;
+                                }
+                            }
+
+                            # insert into testfunction_run table
+                            foreach my $testfunction (keys %{$datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{testfunctions}}) {
+                                my $testfunction_result = "na";             # assume the testfunction result is not available
+                                if (defined $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{testfunctions}{$testfunction}{result}) {
+                                    $testfunction_result = $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{testfunctions}{$testfunction}{result};
+                                } else {
+                                    # calculate the result from the testrow results, if any (use just the plain pass/fail/skip values)
+                                    my $testrow_result = "na";              # assume the testrow result is not available
+                                    my $result = "pass";                    # calculate the 'worst' result from the testrow results
+                                    foreach my $testrow (keys %{$datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{testfunctions}{$testfunction}{DataTags}}) {
+                                        $testrow_result = $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{testfunctions}{$testfunction}{DataTags}{$testrow}{result};
+                                        # "skip" to overwrite "pass"
+                                        if ($result eq "pass") {
+                                            $result = "skip" if ($testrow_result eq "skip" or $testrow_result eq "bskip");
+                                        }
+                                        # "fail" to overwrite others
+                                        if ($result ne "fail") {
+                                            $result = "fail" if ($testrow_result eq "fail" or $testrow_result eq "bfail" or $testrow_result eq "xpass" or $testrow_result eq "bxpass");
+                                        }
+                                    }
+                                    if ($testrow_result eq "na") {
+                                        $testfunction_result = "na";        # testfunction result is not available because testrows do not have any result
+                                    } else {
+                                        $testfunction_result = $result;     # testfunction result calculated from the testrow results
+                                    }
+                                }
+
+                                # get testfunction.id for INSERT query below
+                                my $testfunction_id;
+                                my $select =
+                                    "SELECT testfunction.id
+                                        FROM testfunction
+                                            INNER JOIN testset ON testfunction.testset_id = testset.id
+                                            INNER JOIN project ON testset.project_id = project.id
+                                        WHERE testfunction.name = \"$testfunction\" AND
+                                            testset.name = \"$test\" AND
+                                            project.name = \"$testset_project\"";
+                                my $ids = $dbh->selectall_arrayref($select, { Slice => {} });
+                                foreach my $row (@$ids) {
+                                    $testfunction_id = $row->{id};
+                                }
+
+                                # insert testfunction_run
+                                my $testfunction_run_insert_values =
+                                    "($testfunction_id, $testset_run_id, \"$testfunction_result\", $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{testfunctions}{$testfunction}{duration})";
+                                my $query = "INSERT INTO testfunction_run (testfunction_id, testset_run_id, result, duration) VALUES $testfunction_run_insert_values";
+                                print OUTPUT "$query\n\n" if $VERBOSE or $output;
+                                $dbh->do ($query) or print "insert into testfunction_run failed: $!\n" if !$output;
+                                my $testfunction_run_id = $dbh->last_insert_id("", "", "testfunction_run", "");
+
+                                ###
+                                # insert into testrow tables
+                                #
+
+                                #insert into testrow table
+                                foreach my $testrow (keys %{$datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{testfunctions}{$testfunction}{DataTags}}) {
+
+                                    # certain characters must be escaped for SQL
+                                    my $testrow_sql_escaped;
+                                    my $testrow_sql_wildcard;
+                                    $testrow_sql_escaped = escape($testrow);
+                                    # testrow.name is checked with WHERE LIKE (instead of '=') so the SQL wildcard characters must be escaped
+                                    $testrow_sql_wildcard = escape_sql_wildcard($testrow);
+
+                                    # insert testrow connected to its parent testfunction if not yet in database (must use 'LIKE' instead of '=' for comparing testrow.name because of trailing spaces)
+                                    if ("0E0" eq $dbh->do ("SELECT testrow.name FROM testrow
+                                        INNER JOIN testfunction ON testrow.testfunction_id = testfunction.id
+                                        INNER JOIN testset ON testfunction.testset_id = testset.id
+                                        INNER JOIN project ON testset.project_id = project.id
+                                        WHERE testrow.name LIKE \"$testrow_sql_wildcard\" AND testfunction.name = \"$testfunction\" AND testset.name = \"$test\" AND project.name = \"$testset_project\"")) {
+                                        my $query =
+                                            "INSERT INTO testrow (testfunction_id, name)
+                                                SELECT id, \"$testrow_sql_escaped\" FROM testfunction
+                                                    WHERE testfunction.name = \"$testfunction\" AND
+                                                    testfunction.testset_id = (
+                                                        SELECT id FROM testset
+                                                            WHERE testset.name = \"$test\" AND
+                                                                testset.project_id = $testset_project_id)";
+                                        print OUTPUT "$query\n\n" if $VERBOSE or $output;
+                                        $dbh->do ($query) or print "insert into testrow failed: $!\n" if !$output;
+                                    }
+                                } # end of foreach $testrow
+
+                                # insert into testrow_run table (done in second loop to optimize INSERT performance by collecting values into a bunch)
+                                foreach my $testrow (keys %{$datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{testfunctions}{$testfunction}{DataTags}}) {
+
+                                    # certain characters must be escaped for SQL
+                                    my $testrow_sql_escaped;
+                                    my $testrow_sql_wildcard;
+                                    $testrow_sql_escaped = escape($testrow);
+                                    # testrow.name is checked with WHERE LIKE (instead of '=') so the SQL wildcard characters must be escaped
+                                    $testrow_sql_wildcard = escape_sql_wildcard($testrow);
+
+                                    my $testrow_result;
+                                    if (defined $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{testfunctions}{$testfunction}{DataTags}{$testrow}{result}) {
+                                        $testrow_result = $datahash{cfg}{$cfg}{testresults}{all_tests}{$testset_project}{$test}{testfunctions}{$testfunction}{DataTags}{$testrow}{result};
+                                    } else {
+                                        $testrow_result = "na";     # testfunction result is missing
+                                    }
+
+                                    ## don't save the "pass" results (performance issue) if related option set (Note: all the testfunction_runs must be saved to get their ids)
+                                    if (!(defined $options{skiptestrowpass} and $testrow_result eq "pass")) {
+
+                                        # get testrow.id for INSERT query below
+                                        # (must use 'LIKE' instead of '=' for comparing testrow.name because of trailing spaces)
+                                        my $testrow_id;
+                                        my $select =
+                                            "SELECT testrow.id
+                                                FROM testrow
+                                                    INNER JOIN testfunction ON testrow.testfunction_id = testfunction.id
+                                                    INNER JOIN testset ON testfunction.testset_id = testset.id
+                                                    INNER JOIN project ON testset.project_id = project.id
+                                                WHERE testrow.name LIKE \"$testrow_sql_wildcard\" AND
+                                                    testfunction.name = \"$testfunction\" AND
+                                                    testset.name = \"$test\" AND
+                                                    project.name = \"$testset_project\"";
+                                        my $ids = $dbh->selectall_arrayref($select, { Slice => {} });
+                                        foreach my $row (@$ids) {
+                                            $testrow_id = $row->{id};
+                                        }
+
+                                        # collect values
+                                        $testrow_run_insert_counter++;
+                                        $testrow_run_insert_values .= "," if $testrow_run_insert_counter > 1;
+                                        $testrow_run_insert_values .= "($testrow_id, $testfunction_run_id, \"$testrow_result\")";
+                                        print "$cfg - $testset_project - $test - $testfunction - $testrow\n" if $VERBOSE;
+
+                                    } # end of if defined skiptestrowpass
+
+                                    # insert a bunch of values for better performance
+                                    if ($testrow_run_insert_counter > $SQLINSERTVALUECOUNT) {
+                                        my $query = "INSERT INTO testrow_run (testrow_id, testfunction_run_id, result) VALUES $testrow_run_insert_values";
+                                        print OUTPUT "$query\n\n" if $VERBOSE or $output;
+                                        $dbh->do ($query) or print "insert into testrow_run failed: $!\n" if !$output;
+                                        $testrow_run_insert_counter = 0;
+                                        $testrow_run_insert_values = "";
+                                    }
+
+                                } # end of foreach $testrow
+
+                            } # end of foreach $testfunction
+
+                            # insert the testrow_runs (those remaining from the inner loop above) by each testset; in a bunch for better performance
+                            if ($testrow_run_insert_values ne "") {
+                                my $query = "INSERT INTO testrow_run (testrow_id, testfunction_run_id, result) VALUES $testrow_run_insert_values";
+                                print OUTPUT "$query\n\n" if $VERBOSE or $output;
+                                $dbh->do ($query) or print "insert into testrow_run failed: $!\n" if !$output;
+                            }
+
+                        } # end of if else
+
+                    } # end of foreach $test
+                } # end of foreach $testset_project
+            } # end of if defined {all_tests}
+
+            ###
+            # insert data into phase tables
+            #
 
             if (defined $datahash{cfg}{$cfg}{phases}) {
                 foreach my $phase (keys(%{$datahash{cfg}{$cfg}{phases}})) {
@@ -1608,7 +1988,7 @@ sub sql
                     if ("0E0" eq $dbh->do ("SELECT name FROM phase WHERE name = \"$phase\"")) {
                         my $query =
                             "INSERT INTO phase (name) VALUES (\"$phase\")";
-                        print OUTPUT "$query\n" if $VERBOSE or $output;
+                        print OUTPUT "$query\n\n" if $VERBOSE or $output;
                         $dbh->do ($query) or print "insert into phase failed: $!\n" if !$output;
                     }
 
@@ -1617,20 +1997,8 @@ sub sql
                             SELECT phase.id, conf_run.id, \"$start\", \"$end\"
                                 FROM phase, conf_run
                                 WHERE phase.name = \"$phase\" AND
-                                    conf_run.id = (
-                                        SELECT conf_run.id
-                                            FROM conf_run
-                                                INNER JOIN conf ON conf_run.conf_id = conf.id
-                                                INNER JOIN project_run ON conf_run.project_run_id = project_run.id
-                                                INNER JOIN project ON project_run.project_id = project.id
-                                                INNER JOIN branch ON project_run.branch_id = branch.id
-                                                INNER JOIN state ON project_run.state_id = state.id
-                                            WHERE conf.name = \"$cfg\" AND
-                                                project_run.build_key = $datahash{BUILD_NUMBER} AND
-                                                project.name = \"$projectname\" AND
-                                                branch.name = \"$branchname\" AND
-                                                state.name = \"$statename\" )";
-                    print OUTPUT "$query\n" if $VERBOSE or $output;
+                                    conf_run.id = $conf_run_id";
+                    print OUTPUT "$query\n\n" if $VERBOSE or $output;
                     $dbh->do ($query) or print "insert into phase_run failed: $!\n" if !$output;
                 }
             }
@@ -1639,7 +2007,8 @@ sub sql
         $dbh->commit;   # commit the changes if we get this far
     };
     if ($@) {
-        warn "Transaction aborted because $@";
+        warn "ERROR: Transaction aborted because $@";
+        $error_count++;
         # now rollback to undo the incomplete changes
         # but do it in an eval{} as it may also fail
         eval { $dbh->rollback };
@@ -1728,6 +2097,8 @@ sub run
 
     my @inputfolders;
 
+    my $start_time = time();
+
     if ($options{method} =~ m/^full$/) {
         @inputfolders = @{(get_all_folders($options{workpath}, $options{datelimit}))};
     }
@@ -1738,7 +2109,7 @@ sub run
     my $dbh = sql_connect();
 
     sql_drop_tables($dbh) if (defined $options{delete});
-    sql_create_tables($dbh);
+    sql_create_tables($dbh, \%options);
 
     my %db_status = (
         date    => DateTime->now(),
@@ -1772,17 +2143,24 @@ sub run
         my %datahash = %{read_build_data($statehash, $inputfolder)};
         $datahash{TIMESTAMP} = $modify_time if ($datahash{TIMESTAMP} eq "");
 
+        print "-----------------------------------------------\n";
         print "Build Summary:\n";
         print "Name: $datahash{FULLDISPLAYNAME}\n";
         print "Build number: $datahash{BUILD_NUMBER}\n";
         print "Result: $datahash{RESULT}\n";
         print "Build date: $datahash{TIMESTAMP}\n";
-        sql($dbh, \%options, \%datahash);
-        print "$inputfolder processed.\n\n";
+        sql($dbh, \%options, \%datahash, \%db_status);
+        print "$inputfolder processed.\n";
     }
     $db_status{rebuild} = 0;
     sql_update_progress($dbh, \%db_status);
     sql_disconnect($dbh);
+    print "-----------------------------------------------\n";
+    print "Parsing errors:   $error_count\n";
+    print "Parsing warnings: $warning_count\n";
+    print "Parsing infos:    $info_count\n";
+    print "Done (duration: ", strftime("\%H:\%M:\%S", gmtime(time() - $start_time)), ")\n\n";
+
     return;
 }
 run( @ARGV ) unless caller;