From a5e03f59f4bbfe1cf40ec084d137bf4c91923731 Mon Sep 17 00:00:00 2001 From: BogDan Vatra Date: Fri, 24 May 2019 12:34:28 +0300 Subject: Say hello to androidtestrunner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit androidtestrunner is a tool needed to run qt tests on Android. Now you can run tests as simple as you run them on Linux, macOS, Windows. "$ make check" it's all you need to run tests on the default android device. ANDROID_DEVICE_SERIAL env variable can be used to use a specific android serial. Use cases: $ make -j1 check -j1 is needed to make sure we don't run multiple tests in parallel. $ ANDROID_DEVICE_SERIAL="emulator-5554" make check Run the test on "emulator-5554" $ make TESTARGS="-- -xml" check Switch to xml output. All params after -- are passed to test application. $ make TESTARGS="-- -o out.xml,xml -o out.txt,txt -o -,tap -vs" check Create two files out.xml and out.txt in the current folder and print "tap" format to stdout and enable logging of every signal emission. [ChangeLog][Android] Make it easy to run Qt tests on Android. "$ make check" is all it's needed to run a test on an Android device. Change-Id: I1a7f64b62608f7367b5a6aabf5d6c6e7e50242e6 Reviewed-by: Jędrzej Nowacki --- tests/auto/android/runtests_androiddeployqt.pl | 550 ------------------------- 1 file changed, 550 deletions(-) delete mode 100755 tests/auto/android/runtests_androiddeployqt.pl (limited to 'tests/auto') diff --git a/tests/auto/android/runtests_androiddeployqt.pl b/tests/auto/android/runtests_androiddeployqt.pl deleted file mode 100755 index 24b44cf9b2..0000000000 --- a/tests/auto/android/runtests_androiddeployqt.pl +++ /dev/null @@ -1,550 +0,0 @@ -#!/usr/bin/perl -w -############################################################################# -## -## Copyright (C) 2012-2013 BogDan Vatra -## Copyright (C) 2016 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the test suite of the Qt Toolkit. -## -## $QT_BEGIN_LICENSE:GPL-EXCEPT$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3 as published by the Free Software -## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -use Cwd; -use Cwd 'abs_path'; -use File::Basename; -use File::Temp 'tempdir'; -use File::Path 'remove_tree'; -use Getopt::Long; -use Pod::Usage; -use XML::Simple; -use Term::ANSIColor; - -### default options -my @stack = cwd; -my $device_serial=""; # "-s device_serial"; -my $deployqt_device_serial=""; # "-device device_serial"; -my $log_out="xml"; -my $max_runtime = 5; -my $className="org.qtproject.qt5.android.bindings.QtActivity"; -my $jobs = 4; -my $testsubset = ""; -my $man = 0; -my $help = 0; -my $make_clean = 0; -my $stop_on_fail = 0; -my $time_out=400; -my $android_toolchain_version = "4.8"; -my $host_arch = "linux-x86"; -my $android_sdk_dir = "$ENV{'ANDROID_SDK_ROOT'}"; -my $android_ndk_dir = "$ENV{'ANDROID_NDK_ROOT'}"; -my $android_to_connect = "$ENV{'ANDROID_DEVICE'}"; -my $ant_tool = `which ant`; -my $silent = 0; -chomp $ant_tool; -my $strip_tool=""; -my $readelf_tool=""; -# for ci usage -my @failures = ''; -my $total_tests = 0; -my $total_failed = 0; -my $failed_insignificants = 0; -my $ci_use = 0; -my $start = time(); -my $uninstall = 0; - -GetOptions('h|help' => \$help - , man => \$man - , 's|serial=s' => \$device_serial - , 't|test=s' => \$testsubset - , 'c|clean' => \$make_clean - , 'j|jobs=i' => \$jobs - , 'logtype=s' => \$log_out - , 'runtime=i' => \$max_runtime - , 'sdk=s' => \$android_sdk_dir - , 'ndk=s' => \$android_ndk_dir - , 'toolchain=s' => \$android_toolchain_version - , 'host=s' => \$host_arch - , 'ant=s' => \$ant_tool - , 'strip=s' => \$strip_tool - , 'readelf=s' => \$readelf_tool - , 'testcase=s' => \$testcase - , 'f|fail' => sub { $stop_on_fail = 1 } - , 'silent' => sub { $silent = 1 } - , 'ci' => sub { $ci_use = 1 } - , 'uninstall' => sub { $uninstall = 1 } - ) or pod2usage(2); -pod2usage(1) if $help; -pod2usage(-verbose => 2) if $man; - -if ($ci_use){ - use QMake::Project; -} -my $adb_tool="$android_sdk_dir/platform-tools/adb"; - -# For CI. Nodes are connecting to test devices over IP, which is stored to env variable -if ($android_to_connect ne ""){ - print " Found device to be connected from env: $android_to_connect \n"; - system("$adb_tool disconnect $android_to_connect"); - system("$adb_tool connect $android_to_connect"); - sleep(2);# let it connect - system("$adb_tool -s $android_to_connect reboot &");# adb bug, it blocks forever - sleep(15); # wait for the device to come up again - system("$adb_tool disconnect $android_to_connect");# cleans up the left adb reboot process - system("$adb_tool connect $android_to_connect"); - $device_serial =$android_to_connect; -} - -system("$adb_tool devices") == 0 or die "No device found, please plug/start at least one device/emulator\n"; # make sure we have at least on device attached - -$deployqt_device_serial = "--device $device_serial" if ($device_serial); -$device_serial = "-s $device_serial" if ($device_serial); -$testsubset="/$testsubset" if ($testsubset); - -$strip_tool="$android_ndk_dir/toolchains/arm-linux-androideabi-$android_toolchain_version/prebuilt/$host_arch/bin/arm-linux-androideabi-strip" unless($strip_tool); -$readelf_tool="$android_ndk_dir/toolchains/arm-linux-androideabi-$android_toolchain_version/prebuilt/$host_arch/bin/arm-linux-androideabi-readelf" unless($readelf_tool); -$readelf_tool="$readelf_tool -d -w "; - -sub dir -{ -# print "@stack\n"; -} - -sub pushd ($) -{ - unless ( chdir $_[0] ) - { - warn "Error: $!\n"; - return; - } - unshift @stack, cwd; - dir; -} - -sub popd () -{ - @stack > 1 and shift @stack; - chdir $stack[0]; - dir; -} - -############################## -# Read possible insignificance -# from pro file -############################## -sub check_if_insignificant -{ - return 0 if ( !$ci_use ); - my $case = shift; - my $insignificant = 0; - my $prj = QMake::Project->new( 'Makefile' ); - $insignificant = $prj->test( 'insignificant_test' ); - return $insignificant; -} - -############################## -# Print output from given -# $testresult.txt file -############################## -sub print_output -{ - my $res_file = shift; - my $case = shift; - my $insignificant = shift; - my $print_all = 0; - $total_tests++; - if (-e $res_file) { - open my $file, $res_file or die "Could not open $res_file: $!"; - while (my $line = <$file>) { - if ($line =~ m/^FAIL/) { - print "$line"; - # Pretend to be like the "real" testrunner and print out - # all steps - $print_all = 1; - } - } - close $file; - if ($print_all) { - # In case we are printing all, the test failed - system("cat $res_file"); - if ($insignificant) { - print " Testrunner: $case failed, but it is marked with insignificant_test\n"; - push (@failures ,(basename($case)." [insignificant]")); - $failed_insignificants++; - } else { - $total_failed++; - push (@failures ,(basename($case))); - } - } else { - my $cmd = "sed -n 'x;\$p' ${res_file}"; - my $summary = qx(${cmd}); - if ($summary =~ m/^Totals/) { - print "$summary"; - } else { - print "Error: The log is incomplete. Looks like you have to increase the timeout."; - # The test log seems inclomplete, considering the test as failed. - if ($insignificant) { - print " Testrunner: $case failed, but it is marked with insignificant_test\n"; - push (@failures ,(basename($case)." [insignificant]")); - $failed_insignificants++; - } else { - $total_failed++; - push (@failures ,(basename($case))); - } - } - } - } else { - if ($insignificant) { - print " Failed to execute $case, but it is marked with insignificant_test\n"; - push (@failures ,(basename($case)." [insignificant]")); - $failed_insignificants++; - } else { - print "Failed to execute $case \n"; - $total_failed++; - push (@failures ,(basename($case))); - } - } -} - -############################## -# Print summary of test run -############################## - -sub print_summary -{ - my $total = time()-$start; - my $h = 0; - my $m = 0; - my $s = 0; - my $exit = 0; - print "=== Timing: =================== TEST RUN COMPLETED! ============================\n"; - if ($total > 60*60) { - $h = int($total/60/60); - $s = int($total - $h*60*60); - - $m = int($s/60); - $s = 0; - print "Total: $h hours $m minutes\n"; - } elsif ($total > 60) { - $m = int($total/60); - $s = int($total - $m*60); - print "Total: $m minutes $s seconds\n"; - } else { - $s = int($total); - print "Total: $s seconds\n"; - } - - print "=== Failures: =================================================================="; - foreach my $failed (@failures) { - print $failed."\n"; - $exit = 1; - } - print "=== Totals: ".$total_tests." tests, ".($total_tests-$total_failed). - " passes, ".$failed_insignificants. - " insignificant fails ======================\n"; - return $exit; -} - - -sub waitForProcess -{ - my $process=shift; - my $action=shift; - my $timeout=shift; - my $sleepPeriod=shift; - $sleepPeriod=1 if !defined($sleepPeriod); - print "Waiting for $process ".$timeout*$sleepPeriod." seconds to" if (!$silent); - print $action?" start...\n":" die...\n" if (!$silent); - while ($timeout--) - { - my $output = `$adb_tool $device_serial shell ps 2>&1`; # get current processes - #FIXME check why $output is not matching m/.*S $process\n/ or m/.*S $process$/ (eol) - my $res=($output =~ m/.*S $process/)?1:0; # check the procress - if ($action == $res) - { - print "... succeed\n" if (!$silent); - return 1; - } - sleep($sleepPeriod); - print "timeount in ".$timeout*$sleepPeriod." seconds\n" if (!$silent); - } - print "... failed\n" if (!$silent); - return 0; -} - -my $src_dir_qt=abs_path(dirname($0)."/../../.."); -my $quadruplor_dir="$src_dir_qt/tests/auto/android"; -my $qmake_path="$src_dir_qt/bin/qmake"; -my $androiddeployqt_path="$src_dir_qt/bin/androiddeployqt"; -my $tests_dir="$src_dir_qt/tests$testsubset"; -my $temp_dir=tempdir(CLEANUP => 1); -my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); -my $output_dir=$stack[0]."/".(1900+$year)."-$mon-$mday-$hour:$min"; -mkdir($output_dir); -unlink("latest"); -system(" ln -s $output_dir latest"); -my $sdk_api=0; -my $output = `$adb_tool $device_serial shell getprop`; # get device properties -if ($output =~ m/.*\[ro.build.version.sdk\]: \[(\d+)\]/) -{ - $sdk_api=int($1); - $sdk_api=5 if ($sdk_api>5 && $sdk_api<8); - $sdk_api=9 if ($sdk_api>9); -} - -sub checkXMLOutput -{ - print color 'bold red'; - my $fileName = shift; - my $XMLOutput = eval { XMLin($fileName, ForceArray => 1) }; - if (!defined($XMLOutput)) { - print "Can't parse the $fileName file, probably the test crased.\n"; - print color 'reset'; - die "Stopping\n" if $stop_on_fail; - return; - } - my $testName = $XMLOutput->{name}; - my $fail = 0; - while (my($node_key, $node_valule) = each (%{$XMLOutput})) { - next if $node_key ne "TestFunction"; - while (my($function_key, $function_valule) = each (%{$node_valule})) { - while (my($test_key, $test_valule) = each (%{$function_valule})) { - next if $test_key ne "Incident"; - for my $incident (@{$test_valule}) { - if (($incident->{type} ne "pass") && ($incident->{type} ne "xfail")) { - print "test $testName::$function_key failed $incident->{file}:$incident->{line}\n"; - $fail = 1; - } - } - } - } - } - print color 'reset'; - die "Stopping\n" if $stop_on_fail and $fail; -} - -sub startTest -{ - my $testName = shift; - my $packageName = "org.qtproject.example.tst_$testName"; - my $intentName = "$packageName/org.qtproject.qt5.android.bindings.QtActivity"; - my $output_file = shift; - my $insignificance = shift; - my $get_xml= 0; - my $get_txt= 0; - my $testLib =""; - if ($log_out eq "xml") { - $testLib="-o /data/data/$packageName/output.xml,xml"; - $get_xml = 1; - } elsif ($log_out eq "txt") { - $testLib="-o /data/data/$packageName/output.txt,txt"; - $get_txt = 1; - } else { - $testLib="-o /data/data/$packageName/output.xml,xml -o /data/data/$packageName/output.txt,txt"; - $get_xml = 1; - $get_txt = 1; - } - - my $cmd="${adb_tool} ${device_serial} shell am start -e applicationArguments \"${testLib}\" -n ${intentName}"; - my $res = qx(${cmd}); - print $res if (!$silent); - #wait to start (if it has not started and quit already) - waitForProcess($packageName,1,10); - - #wait to stop - unless(waitForProcess($packageName,0,$time_out,5)) - { - #killProcess($packageName); - print "Someone should kill $packageName\n"; - system("$adb_tool $device_serial uninstall $packageName") if ($uninstall); - return 1; - } - - # Wait for three seconds to allow process to write all data - sleep(3); - - system("$adb_tool $device_serial pull /data/data/$packageName/output.xml $output_dir/$output_file.xml") if ($get_xml); - - system("$adb_tool $device_serial pull /data/data/$packageName/output.txt $output_dir/$output_file.txt") if ($get_txt); - if ($get_txt){ - print "Test results for $packageName:\n"; - my $insig = - print_output("$output_dir/$output_file.txt", $packageName, $insignificance); - } - system("$adb_tool $device_serial uninstall $packageName") if ($uninstall); - - checkXMLOutput("$output_dir/$output_file.xml") if ($get_xml); - return 1; -} - -########### build qt tests and benchmarks ########### -pushd($tests_dir); -print "Building $tests_dir \n"; -system("make distclean") if ($make_clean); -system("$qmake_path -r") == 0 or die "Can't run qmake\n"; #exec qmake -system("make -j$jobs") == 0 or warn "Can't build all tests\n"; #exec make - -my $testsFiles = ""; -if ($testcase) { - $testsFiles=`find . -name libtst_$testcase.so`; # only tests -} else { - $testsFiles=`find . -name libtst_*.so`; # only tests -} - -foreach (split("\n",$testsFiles)) -{ - chomp; #remove white spaces - pushd(abs_path(dirname($_))); # cd to application dir - my $insig = check_if_insignificant(); - my $cmd="make INSTALL_ROOT=${temp_dir} install"; - my $res = qx(${cmd}); - print $res if (!$silent); - my $application=basename(cwd); - if ($silent) { - $cmd="$androiddeployqt_path --install ${deployqt_device_serial} --output ${temp_dir} --deployment debug --verbose --input android-libtst_${application}.so-deployment-settings.json >/dev/null 2>&1"; - } else { - $cmd="$androiddeployqt_path --install ${deployqt_device_serial} --output ${temp_dir} --deployment debug --verbose --input android-libtst_${application}.so-deployment-settings.json"; - } - $res = qx(${cmd}); - print $res if (!$silent); - my $output_name=dirname($_); - $output_name =~ s/\.//; # remove first "." character - $output_name =~ s/\///; # remove first "/" character - $output_name =~ s/\//_/g; # replace all "/" with "_" - $output_name=$application unless($output_name); - $time_out=$max_runtime*60/5; # 5 minutes time out for a normal test - - $applicationLibrary = `find $temp_dir -name libtst_bench_$application.so`; - - if ($applicationLibrary) - { - $time_out=5*60/5; # 10 minutes for a benchmark - $application = "bench_$application"; - } - else - { - $applicationLibrary = `find $temp_dir -name libtst_$application.so`; - } - - if (!$applicationLibrary) - { - print "Can't find application binary libtst_$application.so in $temp_dir!\n"; - } - else - { - startTest($application, "$output_name", $insig) or warn "Can't run $application ...\n"; - } - - popd(); - remove_tree( $temp_dir, {keep_root => 1} ); -} -print_summary() if ($ci_use); -popd(); - -__END__ - -=head1 NAME - -Script to run all qt tests/benchmarks to an android device/emulator - -=head1 SYNOPSIS - -runtests.pl [options] - -=head1 OPTIONS - -=over 8 -=item B<-f --fail> - -Stop the script when test fails. Default 0 - -=item B<-s --serial = serial> - -Device serial number. May be empty if only one device is attached. - -=item B<-t --test = test_subset> - -Tests subset (e.g. benchmarks, auto, auto/qbuffer, etc.). - -=item B<-c --clean> - -Clean tests before building them. - -=item B<-j --jobs = number> - -Make jobs when building tests. - -=item B<--sdk = sdk_path> - -Android SDK path. - -=item B<--ndk = ndk_path> - -Android NDK path. - -=item B<--ant = ant_tool_path> - -Ant tool path. - -=item B<--strip = strip_tool_path> - -Android strip tool path, used to deploy qt libs. - -=item B<--readelf = readelf_tool_path> - -Android readelf tool path, used to check if a test application uses qt OpenGL. - -=item B<--logtype = xml|txt|both> - -The format of log file, default is xml. - -=item B<--runtime = minutes> - -The timeout period before stopping individual tests from running. - -=item B<-silent> - -Suppress output of system commands. - -=item B<-ci> - -Enables checking if test is insignificant or not. Also prints test -summary after all tests has been executed. - -=item B<-uninstall> - -Uninstalls the test after has been executed. - -=item B<-h --help> - -Print a brief help message and exits. - -=item B<--man> - -Prints the manual page and exits. - -=back - -=head1 DESCRIPTION - -B will run all qt tests/benchmarks to an android device/emulator. - -=cut -- cgit v1.2.3