#!/usr/bin/env perl use 5.010; use strict; use warnings; use utf8; use Readonly; =head1 NAME 01-coveragerunner_testcocoon.t - basic test for coveragerunner_testcocoon.pl =head1 SYNOPSIS perl ./01-coveragerunner_testcocoon.t This test will run the coveragerunner_testcocoon.pl script with a few different types of subprocesses and verify that behavior is as expected. =cut use Encode; use English qw( -no_match_vars ); use FindBin; use Readonly; use Test::More; use Capture::Tiny qw( capture ); use Env::Path; use File::Basename; use File::Copy; use File::Find::Rule; use File::Path qw( mkpath ); use File::Spec::Functions; use File::Temp qw( tempdir ); use lib "$FindBin::Bin/../../lib/perl5"; use QtQA::Test::More qw( is_or_like create_mock_command ); # Directory separator, quoted for regex Readonly my $DS => ($OSNAME eq 'MSWin32') ? q{\\\\} : q{/}; # Tempdir testcocoon for tests template Readonly my $TEMPDIR_TOPLEVEL => File::Spec->tmpdir; Readonly my $TEMPDIR_TEMPLATE => qr{\Q$TEMPDIR_TOPLEVEL\E${DS}testcocoon_runner\..{6}}sm; # Preamble always printed out at the beginning Readonly my $COVERAGERUNNER_PREAMBLE => qr{Gather all library files covered in a global database\sList of all source files in coverage.*}; # Reused paths to csmes for the libraries # Note for this (and others), the module/.. part is optional, # because that may be redundant and File::Find may already clean up that # before returning. Readonly my $LIST_CSMESLIB => qr{${TEMPDIR_TEMPLATE}${DS}(?:module${DS}\.\.${DS})?qtbase${DS}lib${DS}lib\.csmes.* ${TEMPDIR_TEMPLATE}${DS}(?:module${DS}\.\.${DS})?qtbase${DS}lib${DS}lib2\.csmes.*}sm; # First merge command Readonly my $COVERAGERUNNER_CMMERGE_SRC => qr{\+ cmmerge --append --output=${TEMPDIR_TEMPLATE}${DS}module_coverage_src-[0-9]{8}-[0-9]{4}\.csmes ${TEMPDIR_TEMPLATE}${DS}(?:module${DS}\.\.${DS})?qtbase${DS}lib${DS}lib2\.csmes.*}; # Second merge command Readonly my $COVERAGERUNNER_CMMERGE_GLOBAL => qr{\+ cmmerge --append --output=${TEMPDIR_TEMPLATE}${DS}module_coverage_global-[0-9]{8}-[0-9]{4}\.csmes ${TEMPDIR_TEMPLATE}${DS}module_coverage_unittests-[0-9]{8}-[0-9]{4}\.csmes.*}; # Cmreport command Readonly my $COVERAGERUNNER_CMREPORT => qr{\+ cmreport --csmes=${TEMPDIR_TEMPLATE}${DS}module_coverage_global-[0-9]{8}-[0-9]{4}\.csmes --xml=${TEMPDIR_TEMPLATE}${DS}module_coverage_report-[0-9]{8}-[0-9]{4}\.xml --select=.* --source=all --source-sort=name --global=all.*}; # gzip command Readonly my $COVERAGERUNNER_GZIP => qr{\+ gzip ${TEMPDIR_TEMPLATE}${DS}module_coverage_global-[0-9]{8}-[0-9]{4}\.csmes.*}; # xml2html_testcocoon command Readonly my $COVERAGERUNNER_XML2HTML => qr{\+ xml2html_testcocoon --xml ${TEMPDIR_TEMPLATE}${DS}module_coverage_report-[0-9]{8}-[0-9]{4}\.xml --module module --output ${TEMPDIR_TEMPLATE}.*}; # Standard output up to the first merge (gathering the lib and plugins csmes) Readonly my $FIRST_CMMERGE => qr{$COVERAGERUNNER_PREAMBLE $LIST_CSMESLIB $COVERAGERUNNER_CMMERGE_SRC}sm; # Standard output up to the second merge (gathering the lib and plugins csmes) Readonly my $SECOND_CMMERGE => qr{$FIRST_CMMERGE cmmerge success lib.* End of list.* $COVERAGERUNNER_CMMERGE_GLOBAL}sm; # Standard output until cmreport command Readonly my $CMREPORT => qr{$SECOND_CMMERGE cmmerge success tests.* $COVERAGERUNNER_CMREPORT}sm; # Standard output until gzip Readonly my $GZIP => qr{$CMREPORT cmreport success.* $COVERAGERUNNER_GZIP}sm; # Standard output until xml2html_testcocoon Readonly my $XML2HTML => qr{$GZIP gzip success.* $COVERAGERUNNER_XML2HTML}sm; # Successful run Readonly my $TESTSCRIPT_SUCCESS => qr{$XML2HTML xml2html_testcocoon success.*}sm; # Invalid --qtcoverage-test-ouput required argument Readonly my $INVALID_TESTS_OUTPUT => qr{${TEMPDIR_TEMPLATE}${DS}tests_invalid\.csmes does not exist. Either the tests have not been run or coverage was not enabled at build time}; # Run coveragerunner_testcocoon sub test_run { my ($params_ref) = @_; my @args = @{$params_ref->{ args }}; my $expected_stdout = $params_ref->{ expected_stdout }; my $expected_stderr = $params_ref->{ expected_stderr }; my $expected_success = $params_ref->{ expected_success }; my $testname = $params_ref->{ testname } || q{}; my $status; my ($output, $error) = capture { $status = system( 'perl', "$FindBin::Bin/../coveragerunner_testcocoon.pl", @args ); }; if ($expected_success) { is ( $status, 0, "$testname exits zero" ); } else { isnt( $status, 0, "$testname exits non-zero" ); } is_or_like( $output, $expected_stdout, "$testname output looks correct" ); is_or_like( $error, $expected_stderr, "$testname error looks correct" ); return; } sub test_success { my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 ); init_test_env($tempdir); # Put our mock command first in PATH local $ENV{ PATH } = $ENV{ PATH }; Env::Path->PATH->Prepend( $tempdir ); create_mock_command( name => 'cmmerge', directory => $tempdir, sequence => [ { exitcode => 0, stdout => "cmmerge success lib\n"}, { exitcode => 0, stdout => "cmmerge success tests\n"}, ], ); create_mock_command( name => 'cmreport', directory => $tempdir, sequence => [ { exitcode => 0, stdout => "cmreport success\n"}, ], ); create_mock_command( name => 'gzip', directory => $tempdir, sequence => [ { exitcode => 0, stdout => "gzip success\n"}, ], ); local $ENV{ TESTING_COVERAGERUNNER } = 1; create_mock_command( name => 'xml2html_testcocoon', directory => $tempdir, sequence => [ { exitcode => 0, stdout => "xml2html_testcocoon success\n"}, ], ); my $module_gitdir = catfile( $tempdir, 'module'); my $test_csmes = catfile( $tempdir, 'tests.csmes' ); test_run({ args => [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'], expected_stdout => $TESTSCRIPT_SUCCESS, expected_stderr => q{}, expected_success => 1, testname => 'basic test success', }); return; } sub test_first_merge_failure { my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 ); init_test_env($tempdir); # Put our mock command first in PATH local $ENV{ PATH } = $ENV{ PATH }; Env::Path->PATH->Prepend( $tempdir ); create_mock_command( name => 'cmmerge', directory => $tempdir, sequence => [ { exitcode => 1, stderr => "first cmmerge failure\n", stdout => ""}, ], ); my $module_gitdir = catfile( $tempdir, 'module'); my $test_csmes = catfile( $tempdir, 'tests.csmes' ); test_run({ args => [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'], expected_stdout => $FIRST_CMMERGE, expected_stderr => qr{first cmmerge failure\n}sm, expected_success => 0, testname => 'test first merge failed', }); return; } sub test_second_merge_failure { my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 ); init_test_env($tempdir); # Put our mock command first in PATH local $ENV{ PATH } = $ENV{ PATH }; Env::Path->PATH->Prepend( $tempdir ); create_mock_command( name => 'cmmerge', directory => $tempdir, sequence => [ { exitcode => 0, stdout => "cmmerge success lib\n"}, { exitcode => 2, stderr => "second cmmerge failure\n", stdout => ""}, ], ); my $module_gitdir = catfile( $tempdir, 'module'); my $test_csmes = catfile( $tempdir, 'tests.csmes' ); test_run({ args => [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'], expected_stdout => $SECOND_CMMERGE, expected_stderr => qr{second cmmerge failure\n}sm, expected_success => 0, testname => 'test second merge failed', }); return; } sub test_cmreport_failure { my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 ); init_test_env($tempdir); # Put our mock command first in PATH local $ENV{ PATH } = $ENV{ PATH }; Env::Path->PATH->Prepend( $tempdir ); create_mock_command( name => 'cmmerge', directory => $tempdir, sequence => [ { exitcode => 0, stdout => "cmmerge success lib\n"}, { exitcode => 0, stdout => "cmmerge success tests\n"}, ], ); create_mock_command( name => 'cmreport', directory => $tempdir, sequence => [ { exitcode => 3, stderr => "cmreport failure\n", stdout => ""}, ], ); my $module_gitdir = catfile( $tempdir, 'module'); my $test_csmes = catfile( $tempdir, 'tests.csmes' ); test_run({ args => [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'], expected_stdout => $CMREPORT, expected_stderr => qr{cmreport failure\n}sm, expected_success => 0, testname => 'test cmreport failure', }); return; } sub test_gzip_failure { my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 ); init_test_env($tempdir); # Put our mock command first in PATH local $ENV{ PATH } = $ENV{ PATH }; Env::Path->PATH->Prepend( $tempdir ); create_mock_command( name => 'cmmerge', directory => $tempdir, sequence => [ { exitcode => 0, stdout => "cmmerge success lib\n"}, { exitcode => 0, stdout => "cmmerge success tests\n"}, ], ); create_mock_command( name => 'cmreport', directory => $tempdir, sequence => [ { exitcode => 0, stdout => "cmreport success\n"}, ], ); create_mock_command( name => 'gzip', directory => $tempdir, sequence => [ { exitcode => 4, stderr => "gzip failure\n", stdout => ""}, ], ); my $module_gitdir = catfile( $tempdir, 'module'); my $test_csmes = catfile( $tempdir, 'tests.csmes' ); test_run({ args => [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'], expected_stdout => $GZIP, expected_stderr => qr{gzip failure\n}sm, expected_success => 0, testname => 'test gzip failure', }); return; } sub test_xml2html_failure { my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 ); init_test_env($tempdir); # Put our mock command first in PATH local $ENV{ PATH } = $ENV{ PATH }; Env::Path->PATH->Prepend( $tempdir ); create_mock_command( name => 'cmmerge', directory => $tempdir, sequence => [ { exitcode => 0, stdout => "cmmerge success lib\n"}, { exitcode => 0, stdout => "cmmerge success tests\n"}, ], ); create_mock_command( name => 'cmreport', directory => $tempdir, sequence => [ { exitcode => 0, stdout => "cmreport success\n"}, ], ); create_mock_command( name => 'gzip', directory => $tempdir, sequence => [ { exitcode => 0, stdout => "gzip success\n"}, ], ); local $ENV{ TESTING_COVERAGERUNNER } = 1; create_mock_command( name => 'xml2html_testcocoon', directory => $tempdir, sequence => [ { exitcode => 5, stderr => "xml2html_testcocoon failure\n"}, ], ); my $module_gitdir = catfile( $tempdir, 'module'); my $test_csmes = catfile( $tempdir, 'tests.csmes' ); test_run({ args => [ '--qt-gitmodule-dir', $module_gitdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_csmes, 'testname'], expected_stdout => $XML2HTML, expected_stderr => qr{xml2html_testcocoon failure\n}sm, expected_success => 0, testname => 'test xml2html failure', }); return; } sub test_invalid_qtcoverage_tests_output { my $tempdir = tempdir( 'testcocoon_runner.XXXXXX', TMPDIR => 1, CLEANUP => 1 ); init_test_env($tempdir); my $test_invalid_csmes = catfile( $tempdir, 'tests_invalid.csmes' ); test_run({ args => [ '--qt-gitmodule-dir', $tempdir, '--qt-gitmodule', 'module', '--qtcoverage-tests-output', $test_invalid_csmes, 'testname'], expected_stdout => q{}, expected_stderr => $INVALID_TESTS_OUTPUT, expected_success => 0, testname => 'invalid csmes output file', }); return; } sub init_test_env { my ($tempdir) = @_; # Create qtbase path my $qtbase_path = catfile($tempdir, 'qtbase'); if (! -d $qtbase_path && ! mkpath( $qtbase_path )) { die "mkpath $qtbase_path: $!"; } # Create plugins path my $plugins_path = catfile($qtbase_path, 'plugins'); if (! -d $plugins_path && ! mkpath( $plugins_path )) { die "mkpath $plugins_path: $!"; } # Create "module" path my $module_gitdir = catfile( $tempdir, 'module'); if (! -d $module_gitdir && ! mkpath( $module_gitdir )) { die "mkpath $module_gitdir: $!"; } # Create lib path my $lib_path = catfile($qtbase_path, 'lib'); if (! -d $lib_path && ! mkpath( $lib_path )) { die "mkpath $lib_path: $!"; } # Create a csmes in lib folder open(my $lib, ">", "$lib_path/lib.csmes") or die $!; close($lib); open(my $lib2, ">", "$lib_path/lib2.csmes") or die $!; close($lib2); # Create a tests result database csmes open(my $test, ">", "$tempdir/tests.csmes") or die $!; close($test); return; } sub run { test_success; test_first_merge_failure; test_second_merge_failure; test_cmreport_failure; test_gzip_failure; test_invalid_qtcoverage_tests_output; test_xml2html_failure; done_testing; return; } run if (!caller); 1;