diff options
| -rw-r--r-- | mkspecs/features/qmltestcase.prf | 10 | ||||
| -rw-r--r-- | mkspecs/features/testcase.prf | 14 | ||||
| -rw-r--r-- | src/src.pro | 10 | ||||
| -rw-r--r-- | src/tools/androidtestrunner/androidtestrunner.pro | 13 | ||||
| -rw-r--r-- | src/tools/androidtestrunner/main.cpp | 464 | ||||
| -rwxr-xr-x | tests/auto/android/runtests_androiddeployqt.pl | 550 |
6 files changed, 505 insertions, 556 deletions
diff --git a/mkspecs/features/qmltestcase.prf b/mkspecs/features/qmltestcase.prf index b4b1224781..ae4ebef513 100644 --- a/mkspecs/features/qmltestcase.prf +++ b/mkspecs/features/qmltestcase.prf @@ -1,8 +1,14 @@ !isEmpty(SOURCES) { QT += qml qmltest load(testcase) - contains(TEMPLATE, vc.*): DEFINES += QUICK_TEST_SOURCE_DIR=\"$$_PRO_FILE_PWD_\" - else: DEFINES += QUICK_TEST_SOURCE_DIR=$$shell_quote(\"$$_PRO_FILE_PWD_\") + !android { + contains(TEMPLATE, vc.*): DEFINES += QUICK_TEST_SOURCE_DIR=\"$$_PRO_FILE_PWD_\" + else: DEFINES += QUICK_TEST_SOURCE_DIR=$$shell_quote(\"$$_PRO_FILE_PWD_\") + } else { + !isEmpty(RESOURCES): warning("The RESOURCES qmake variable is empty, the test will probably fail to run") + DEFINES += QUICK_TEST_SOURCE_DIR=\":/\" + } + } else { # Allow a project to run tests without a CPP stub TEMPLATE = aux diff --git a/mkspecs/features/testcase.prf b/mkspecs/features/testcase.prf index b8102c26b5..bc1ee22701 100644 --- a/mkspecs/features/testcase.prf +++ b/mkspecs/features/testcase.prf @@ -52,14 +52,24 @@ debug_and_release:debug_and_release_target { } # Allow for a custom test runner script -$${type}.commands += $(TESTRUNNER) + +android: isEmpty($(TESTRUNNER)) { + qtPrepareTool(ANDROIDTESTRUNNER, androidtestrunner) + qtPrepareTool(ANDROIDDEPLOYQT, androiddeployqt) + isEmpty(ANDROID_DEPLOYMENT_SETTINGS_FILE): ANDROID_DEPLOYMENT_SETTINGS_FILE = $$OUT_PWD/android-$$TARGET-deployment-settings.json + contains(QMAKE_HOST.os, Windows): extension = .exe + $${type}.commands = $$ANDROIDTESTRUNNER --androiddeployqt \"$$ANDROIDDEPLOYQT --input $$ANDROID_DEPLOYMENT_SETTINGS_FILE\" + $${type}.commands += --path \"$$OUT_PWD/android-build\" + $${type}.commands += --adb \"$$shell_path($${ANDROID_SDK_ROOT}$${QMAKE_DIR_SEP}platform-tools$${QMAKE_DIR_SEP}adb$${extension})\" + $${type}.commands += --make \"$(MAKE) -f $(MAKEFILE)\" +} else: $${type}.commands += $(TESTRUNNER) unix { isEmpty(TEST_TARGET_DIR): TEST_TARGET_DIR = . app_bundle: \ $${type}.commands += $${TEST_TARGET_DIR}/$(QMAKE_TARGET).app/Contents/MacOS/$(QMAKE_TARGET) - else: \ + else: !android: \ $${type}.commands += $${TEST_TARGET_DIR}/$(QMAKE_TARGET) } else { # Windows diff --git a/src/src.pro b/src/src.pro index 1c76a2e46f..a39b718e10 100644 --- a/src/src.pro +++ b/src/src.pro @@ -57,6 +57,10 @@ src_tools_androiddeployqt.subdir = tools/androiddeployqt src_tools_androiddeployqt.target = sub-androiddeployqt src_tools_androiddeployqt.depends = src_corelib +src_tools_androidtestrunner.subdir = tools/androidtestrunner +src_tools_androidtestrunner.target = sub-androidtestrunner +src_tools_androidtestrunner.depends = src_corelib + src_tools_qvkgen.subdir = tools/qvkgen src_tools_qvkgen.target = sub-qvkgen force_bootstrap: src_tools_qvkgen.depends = src_tools_bootstrap @@ -189,8 +193,10 @@ qtConfig(dbus) { } android { - SUBDIRS += src_tools_androiddeployqt - TOOLS += src_tools_androiddeployqt + SUBDIRS += src_tools_androiddeployqt \ + src_tools_androidtestrunner + TOOLS += src_tools_androiddeployqt \ + src_tools_androidtestrunner } qtConfig(concurrent): SUBDIRS += src_concurrent diff --git a/src/tools/androidtestrunner/androidtestrunner.pro b/src/tools/androidtestrunner/androidtestrunner.pro new file mode 100644 index 0000000000..641d3e0003 --- /dev/null +++ b/src/tools/androidtestrunner/androidtestrunner.pro @@ -0,0 +1,13 @@ +option(host_build) +CONFIG += console + +SOURCES += \ + main.cpp + +# Required for declarations of popen/pclose on Windows +windows: QMAKE_CXXFLAGS += -U__STRICT_ANSI__ + +DEFINES += QT_NO_CAST_FROM_ASCII QT_NO_CAST_TO_ASCII +DEFINES += QT_NO_FOREACH + +load(qt_app) diff --git a/src/tools/androidtestrunner/main.cpp b/src/tools/androidtestrunner/main.cpp new file mode 100644 index 0000000000..c7c27c97df --- /dev/null +++ b/src/tools/androidtestrunner/main.cpp @@ -0,0 +1,464 @@ +/**************************************************************************** +** +** Copyright (C) 2019 BogDan Vatra <bogdan@kde.org> +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the tools applications of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include <QCoreApplication> +#include <QDir> +#include <QHash> +#include <QRegExp> +#include <QXmlStreamReader> + +#include <algorithm> +#include <chrono> +#include <functional> +#include <thread> + +#ifdef Q_CC_MSVC +#define popen _popen +#define QT_POPEN_READ "rb" +#define pclose _pclose +#else +#define QT_POPEN_READ "r" +#endif + +struct Options +{ + bool helpRequested = false; + bool verbose = false; + std::chrono::seconds timeout{300}; // 5minutes + QString androidDeployQtCommand; + QString buildPath; + QString adbCommand{QStringLiteral("adb")}; + QString makeCommand; + QString package; + QString activity; + QStringList testArgsList; + QHash<QString, QString> outFiles; + QString testArgs; + QHash<QString, std::function<bool(const QByteArray &)>> checkFiles = { + {QStringLiteral("txt"), [](const QByteArray &data) -> bool { + return data.indexOf("\nFAIL! : ") < 0; + }}, + {QStringLiteral("csv"), [](const QByteArray &/*data*/) -> bool { + // It seems csv is broken + return true; + }}, + {QStringLiteral("xml"), [](const QByteArray &data) -> bool { + QXmlStreamReader reader{data}; + while (!reader.atEnd()) { + reader.readNext(); + if (reader.isStartElement() && reader.name() == QStringLiteral("Incident") && + reader.attributes().value(QStringLiteral("type")).toString() == QStringLiteral("fail")) { + return false; + } + } + return true; + }}, + {QStringLiteral("lightxml"), [](const QByteArray &data) -> bool { + return data.indexOf("\n<Incident type=\"fail\" ") < 0; + }}, + {QStringLiteral("xunitxml"), [](const QByteArray &data) -> bool { + QXmlStreamReader reader{data}; + while (!reader.atEnd()) { + reader.readNext(); + if (reader.isStartElement() && reader.name() == QStringLiteral("testcase") && + reader.attributes().value(QStringLiteral("result")).toString() == QStringLiteral("fail")) { + return false; + } + } + return true; + }}, + {QStringLiteral("teamcity"), [](const QByteArray &data) -> bool { + return data.indexOf("' message='Failure! |[Loc: ") < 0; + }}, + {QStringLiteral("tap"), [](const QByteArray &data) -> bool { + return data.indexOf("\nnot ok ") < 0; + }}, + }; +}; + +static Options g_options; + +static bool execCommand(const QString &command, QByteArray *output = nullptr, bool verbose = false) +{ +#if defined(Q_OS_WIN32) + QString processedCommand = QLatin1Char('\"') + command + QLatin1Char('\"'); +#else + const QString& processedCommand = command; +#endif + + if (verbose) + fprintf(stdout, "Execute %s\n", processedCommand.toUtf8().constData()); + FILE *process = popen(processedCommand.toUtf8().constData(), QT_POPEN_READ); + + if (!process) { + fprintf(stderr, "Cannot execute command %s", qPrintable(processedCommand)); + return false; + } + char buffer[512]; + while (fgets(buffer, sizeof(buffer), process)) { + if (output) + output->append(buffer); + if (verbose) + fprintf(stdout, "%s", buffer); + } + return pclose(process) == 0; +} + +// Copy-pasted from qmake/library/ioutil.cpp +inline static bool hasSpecialChars(const QString &arg, const uchar (&iqm)[16]) +{ + for (int x = arg.length() - 1; x >= 0; --x) { + ushort c = arg.unicode()[x].unicode(); + if ((c < sizeof(iqm) * 8) && (iqm[c / 8] & (1 << (c & 7)))) + return true; + } + return false; +} + +static QString shellQuoteUnix(const QString &arg) +{ + // Chars that should be quoted (TM). This includes: + static const uchar iqm[] = { + 0xff, 0xff, 0xff, 0xff, 0xdf, 0x07, 0x00, 0xd8, + 0x00, 0x00, 0x00, 0x38, 0x01, 0x00, 0x00, 0x78 + }; // 0-32 \'"$`<>|;&(){}*?#!~[] + + if (!arg.length()) + return QStringLiteral("\"\""); + + QString ret(arg); + if (hasSpecialChars(ret, iqm)) { + ret.replace(QLatin1Char('\''), QStringLiteral("'\\''")); + ret.prepend(QLatin1Char('\'')); + ret.append(QLatin1Char('\'')); + } + return ret; +} + +static QString shellQuoteWin(const QString &arg) +{ + // Chars that should be quoted (TM). This includes: + // - control chars & space + // - the shell meta chars "&()<>^| + // - the potential separators ,;= + static const uchar iqm[] = { + 0xff, 0xff, 0xff, 0xff, 0x45, 0x13, 0x00, 0x78, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x10 + }; + + if (!arg.length()) + return QStringLiteral("\"\""); + + QString ret(arg); + if (hasSpecialChars(ret, iqm)) { + // Quotes are escaped and their preceding backslashes are doubled. + // It's impossible to escape anything inside a quoted string on cmd + // level, so the outer quoting must be "suspended". + ret.replace(QRegExp(QStringLiteral("(\\\\*)\"")), QStringLiteral("\"\\1\\1\\^\"\"")); + // The argument must not end with a \ since this would be interpreted + // as escaping the quote -- rather put the \ behind the quote: e.g. + // rather use "foo"\ than "foo\" + int i = ret.length(); + while (i > 0 && ret.at(i - 1) == QLatin1Char('\\')) + --i; + ret.insert(i, QLatin1Char('"')); + ret.prepend(QLatin1Char('"')); + } + return ret; +} + +static QString shellQuote(const QString &arg) +{ + if (QDir::separator() == QLatin1Char('\\')) + return shellQuoteWin(arg); + else + return shellQuoteUnix(arg); +} + +static bool parseOptions() +{ + QStringList arguments = QCoreApplication::arguments(); + int i = 1; + for (; i < arguments.size(); ++i) { + const QString &argument = arguments.at(i); + if (argument.compare(QStringLiteral("--androiddeployqt"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.androidDeployQtCommand = arguments.at(++i).trimmed(); + } else if (argument.compare(QStringLiteral("--adb"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.adbCommand = arguments.at(++i); + } else if (argument.compare(QStringLiteral("--path"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.buildPath = arguments.at(++i); + } else if (argument.compare(QStringLiteral("--make"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.makeCommand = arguments.at(++i); + } else if (argument.compare(QStringLiteral("--activity"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.activity = arguments.at(++i); + } else if (argument.compare(QStringLiteral("--timeout"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.timeout = std::chrono::seconds{arguments.at(++i).toInt()}; + } else if (argument.compare(QStringLiteral("--help"), Qt::CaseInsensitive) == 0) { + g_options.helpRequested = true; + } else if (argument.compare(QStringLiteral("--verbose"), Qt::CaseInsensitive) == 0) { + g_options.verbose = true; + } else if (argument.compare(QStringLiteral("--"), Qt::CaseInsensitive) == 0) { + ++i; + break; + } else { + g_options.testArgsList << arguments.at(i); + } + } + for (;i < arguments.size(); ++i) + g_options.testArgsList << arguments.at(i); + + if (g_options.helpRequested || g_options.androidDeployQtCommand.isEmpty() || g_options.buildPath.isEmpty()) + return false; + + QString serial = qEnvironmentVariable("ANDROID_DEVICE_SERIAL"); + if (!serial.isEmpty()) + g_options.adbCommand += QStringLiteral(" -s %1").arg(serial); + return true; +} + +static void printHelp() +{// "012345678901234567890123456789012345678901234567890123456789012345678901" + fprintf(stderr, "Syntax: %s <options> -- [TESTARGS] \n" + "\n" + " Creates an Android package in a temp directory <destination> and\n" + " runs it on the default emulator/device or on the one specified by\n" + " \"ANDROID_DEVICE_SERIAL\" environment variable.\n\n" + " Mandatory arguments:\n" + " --androiddeployqt <androiddeployqt cmd>: The androiddeployqt:\n" + " path including its additional arguments.\n" + " --path <path>: The path where androiddeployqt will build the .apk.\n" + " Optional arguments:\n" + " --adb <adb cmd>: The Android ADB command. If missing the one from\n" + " $PATH will be used.\n" + " --activity <acitvity>: The Activity to run. If missing the first\n" + " activity from AndroidManifest.qml file will be used.\n" + " --timout <seconds>: Timeout to run the test.\n" + " Default is 5 minutes.\n" + " --make <make cmd>: make command, needed to install the qt library.\n" + " If make is missing make sure the --path is set.\n" + " -- arguments that will be passed to the test application.\n" + " --verbose: Prints out information during processing.\n" + " --help: Displays this information.\n\n", + qPrintable(QCoreApplication::arguments().at(0)) + ); +} + +static QString packageNameFromAndroidManifest(const QString &androidManifestPath) +{ + QFile androidManifestXml(androidManifestPath); + if (androidManifestXml.open(QIODevice::ReadOnly)) { + QXmlStreamReader reader(&androidManifestXml); + while (!reader.atEnd()) { + reader.readNext(); + if (reader.isStartElement() && reader.name() == QStringLiteral("manifest")) + return reader.attributes().value(QStringLiteral("package")).toString(); + } + } + return {}; +} + +static QString activityFromAndroidManifest(const QString &androidManifestPath) +{ + QFile androidManifestXml(androidManifestPath); + if (androidManifestXml.open(QIODevice::ReadOnly)) { + QXmlStreamReader reader(&androidManifestXml); + while (!reader.atEnd()) { + reader.readNext(); + if (reader.isStartElement() && reader.name() == QStringLiteral("activity")) + return reader.attributes().value(QStringLiteral("android:name")).toString(); + } + } + return {}; +} + +static void setOutputFile(QString file, QString format) +{ + if (file.isEmpty()) + file = QStringLiteral("-"); + if (format.isEmpty()) + format = QStringLiteral("txt"); + + g_options.outFiles[format] = file; +} + +static bool parseTestArgs() +{ + QRegExp newLoggingFormat{QStringLiteral("(.*),(txt|csv|xunitxml|xml|lightxml|teamcity|tap)")}; + QRegExp oldFormats{QStringLiteral("-(txt|csv|xunitxml|xml|lightxml|teamcity|tap)")}; + + QString file; + QString logType; + QString unhandledArgs; + for (int i = 0; i < g_options.testArgsList.size(); ++i) { + const QString &arg = g_options.testArgsList[i].trimmed(); + if (arg == QStringLiteral("-o")) { + if (i >= g_options.testArgsList.size() - 1) + return false; // missing file argument + + const auto &filePath = g_options.testArgsList[++i]; + if (!newLoggingFormat.exactMatch(filePath)) { + file = filePath; + } else { + const auto capturedTexts = newLoggingFormat.capturedTexts(); + setOutputFile(capturedTexts.at(1), capturedTexts.at(2)); + } + } else if (oldFormats.exactMatch(arg)) { + logType = oldFormats.capturedTexts().at(1); + } else { + unhandledArgs += QStringLiteral(" %1").arg(arg); + } + } + if (g_options.outFiles.isEmpty() || !file.isEmpty() || !logType.isEmpty()) + setOutputFile(file, logType); + + for (const auto &format : g_options.outFiles.keys()) + g_options.testArgs += QStringLiteral(" -o output.%1,%1").arg(format); + + g_options.testArgs += unhandledArgs; + g_options.testArgs = QStringLiteral("shell am start -e applicationArguments \"%1\" -n %2/%3").arg(shellQuote(g_options.testArgs.trimmed()), + g_options.package, + g_options.activity); + return true; +} + +static bool isRunning() { + QByteArray output; + if (!execCommand(QStringLiteral("%1 shell 'ps | grep \" %2\"'").arg(g_options.adbCommand, + shellQuoteUnix(g_options.package)), &output)) { + + return false; + } + return output.indexOf(" " + g_options.package.toUtf8()) > -1; +} + +static bool waitToFinish() +{ + using clock = std::chrono::system_clock; + auto start = clock::now(); + // wait to start + while (!isRunning()) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if ((clock::now() - start) > std::chrono::seconds{10}) + return false; + } + + // Wait to finish + while (isRunning()) { + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + if ((clock::now() - start) > g_options.timeout) + return false; + } + return true; +} + + +static bool pullFiles() +{ + bool ret = true; + for (auto it = g_options.outFiles.constBegin(); it != g_options.outFiles.end(); ++it) { + QByteArray output; + if (!execCommand(QStringLiteral("%1 shell run-as %2 cat files/output.%3") + .arg(g_options.adbCommand, g_options.package, it.key()), &output)) { + return false; + } + auto checkerIt = g_options.checkFiles.find(it.key()); + ret &= checkerIt != g_options.checkFiles.end() && checkerIt.value()(output); + if (it.value() == QStringLiteral("-")){ + fprintf(stdout, "%s", output.constData()); + fflush(stdout); + } else { + QFile out{it.value()}; + if (!out.open(QIODevice::WriteOnly)) + return false; + out.write(output); + } + } + return ret; +} + +int main(int argc, char *argv[]) +{ + QCoreApplication a(argc, argv); + if (!parseOptions()) { + printHelp(); + return 1; + } + + if (!g_options.makeCommand.isEmpty()) { + // we need to run make INSTALL_ROOT=path install to install the application file(s) first + if (!execCommand(QStringLiteral("%1 INSTALL_ROOT=%2 install") + .arg(g_options.makeCommand, g_options.buildPath), nullptr, g_options.verbose)) { + return 1; + } + } + // Run androiddeployqt + static auto verbose = g_options.verbose ? QStringLiteral("--verbose") : QStringLiteral(); + if (!execCommand(QStringLiteral("%1 %3 --gradle --reinstall --output %2").arg(g_options.androidDeployQtCommand, + g_options.buildPath, + verbose), nullptr, g_options.verbose)) { + return 1; + } + + QString manifest = g_options.buildPath + QStringLiteral("/AndroidManifest.xml"); + g_options.package = packageNameFromAndroidManifest(manifest); + if (g_options.activity.isEmpty()) + g_options.activity = activityFromAndroidManifest(manifest); + + // parseTestArgs depends on g_options.package + if (!parseTestArgs()) + return 1; + + // start the tests + bool res = execCommand(QStringLiteral("%1 %2").arg(g_options.adbCommand, g_options.testArgs), + nullptr, g_options.verbose) && waitToFinish(); + if (res) + res &= pullFiles(); + res &= execCommand(QStringLiteral("%1 uninstall %2").arg(g_options.adbCommand, g_options.package), + nullptr, g_options.verbose); + fflush(stdout); + return res ? 0 : 1; +} diff --git a/tests/auto/android/runtests_androiddeployqt.pl b/tests/auto/android/runtests_androiddeployqt.pl deleted file mode 100755 index 24b44cf9b2..0000000000 --- a/tests/auto/android/runtests_androiddeployqt.pl +++ /dev/null @@ -1,550 +0,0 @@ -#!/usr/bin/perl -w -############################################################################# -## -## Copyright (C) 2012-2013 BogDan Vatra <bogdan@kde.org> -## Copyright (C) 2016 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the test suite of the Qt Toolkit. -## -## $QT_BEGIN_LICENSE:GPL-EXCEPT$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3 as published by the Free Software -## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -use Cwd; -use Cwd 'abs_path'; -use File::Basename; -use File::Temp 'tempdir'; -use File::Path 'remove_tree'; -use Getopt::Long; -use Pod::Usage; -use XML::Simple; -use Term::ANSIColor; - -### default options -my @stack = cwd; -my $device_serial=""; # "-s device_serial"; -my $deployqt_device_serial=""; # "-device device_serial"; -my $log_out="xml"; -my $max_runtime = 5; -my $className="org.qtproject.qt5.android.bindings.QtActivity"; -my $jobs = 4; -my $testsubset = ""; -my $man = 0; -my $help = 0; -my $make_clean = 0; -my $stop_on_fail = 0; -my $time_out=400; -my $android_toolchain_version = "4.8"; -my $host_arch = "linux-x86"; -my $android_sdk_dir = "$ENV{'ANDROID_SDK_ROOT'}"; -my $android_ndk_dir = "$ENV{'ANDROID_NDK_ROOT'}"; -my $android_to_connect = "$ENV{'ANDROID_DEVICE'}"; -my $ant_tool = `which ant`; -my $silent = 0; -chomp $ant_tool; -my $strip_tool=""; -my $readelf_tool=""; -# for ci usage -my @failures = ''; -my $total_tests = 0; -my $total_failed = 0; -my $failed_insignificants = 0; -my $ci_use = 0; -my $start = time(); -my $uninstall = 0; - -GetOptions('h|help' => \$help - , man => \$man - , 's|serial=s' => \$device_serial - , 't|test=s' => \$testsubset - , 'c|clean' => \$make_clean - , 'j|jobs=i' => \$jobs - , 'logtype=s' => \$log_out - , 'runtime=i' => \$max_runtime - , 'sdk=s' => \$android_sdk_dir - , 'ndk=s' => \$android_ndk_dir - , 'toolchain=s' => \$android_toolchain_version - , 'host=s' => \$host_arch - , 'ant=s' => \$ant_tool - , 'strip=s' => \$strip_tool - , 'readelf=s' => \$readelf_tool - , 'testcase=s' => \$testcase - , 'f|fail' => sub { $stop_on_fail = 1 } - , 'silent' => sub { $silent = 1 } - , 'ci' => sub { $ci_use = 1 } - , 'uninstall' => sub { $uninstall = 1 } - ) or pod2usage(2); -pod2usage(1) if $help; -pod2usage(-verbose => 2) if $man; - -if ($ci_use){ - use QMake::Project; -} -my $adb_tool="$android_sdk_dir/platform-tools/adb"; - -# For CI. Nodes are connecting to test devices over IP, which is stored to env variable -if ($android_to_connect ne ""){ - print " Found device to be connected from env: $android_to_connect \n"; - system("$adb_tool disconnect $android_to_connect"); - system("$adb_tool connect $android_to_connect"); - sleep(2);# let it connect - system("$adb_tool -s $android_to_connect reboot &");# adb bug, it blocks forever - sleep(15); # wait for the device to come up again - system("$adb_tool disconnect $android_to_connect");# cleans up the left adb reboot process - system("$adb_tool connect $android_to_connect"); - $device_serial =$android_to_connect; -} - -system("$adb_tool devices") == 0 or die "No device found, please plug/start at least one device/emulator\n"; # make sure we have at least on device attached - -$deployqt_device_serial = "--device $device_serial" if ($device_serial); -$device_serial = "-s $device_serial" if ($device_serial); -$testsubset="/$testsubset" if ($testsubset); - -$strip_tool="$android_ndk_dir/toolchains/arm-linux-androideabi-$android_toolchain_version/prebuilt/$host_arch/bin/arm-linux-androideabi-strip" unless($strip_tool); -$readelf_tool="$android_ndk_dir/toolchains/arm-linux-androideabi-$android_toolchain_version/prebuilt/$host_arch/bin/arm-linux-androideabi-readelf" unless($readelf_tool); -$readelf_tool="$readelf_tool -d -w "; - -sub dir -{ -# print "@stack\n"; -} - -sub pushd ($) -{ - unless ( chdir $_[0] ) - { - warn "Error: $!\n"; - return; - } - unshift @stack, cwd; - dir; -} - -sub popd () -{ - @stack > 1 and shift @stack; - chdir $stack[0]; - dir; -} - -############################## -# Read possible insignificance -# from pro file -############################## -sub check_if_insignificant -{ - return 0 if ( !$ci_use ); - my $case = shift; - my $insignificant = 0; - my $prj = QMake::Project->new( 'Makefile' ); - $insignificant = $prj->test( 'insignificant_test' ); - return $insignificant; -} - -############################## -# Print output from given -# $testresult.txt file -############################## -sub print_output -{ - my $res_file = shift; - my $case = shift; - my $insignificant = shift; - my $print_all = 0; - $total_tests++; - if (-e $res_file) { - open my $file, $res_file or die "Could not open $res_file: $!"; - while (my $line = <$file>) { - if ($line =~ m/^FAIL/) { - print "$line"; - # Pretend to be like the "real" testrunner and print out - # all steps - $print_all = 1; - } - } - close $file; - if ($print_all) { - # In case we are printing all, the test failed - system("cat $res_file"); - if ($insignificant) { - print " Testrunner: $case failed, but it is marked with insignificant_test\n"; - push (@failures ,(basename($case)." [insignificant]")); - $failed_insignificants++; - } else { - $total_failed++; - push (@failures ,(basename($case))); - } - } else { - my $cmd = "sed -n 'x;\$p' ${res_file}"; - my $summary = qx(${cmd}); - if ($summary =~ m/^Totals/) { - print "$summary"; - } else { - print "Error: The log is incomplete. Looks like you have to increase the timeout."; - # The test log seems inclomplete, considering the test as failed. - if ($insignificant) { - print " Testrunner: $case failed, but it is marked with insignificant_test\n"; - push (@failures ,(basename($case)." [insignificant]")); - $failed_insignificants++; - } else { - $total_failed++; - push (@failures ,(basename($case))); - } - } - } - } else { - if ($insignificant) { - print " Failed to execute $case, but it is marked with insignificant_test\n"; - push (@failures ,(basename($case)." [insignificant]")); - $failed_insignificants++; - } else { - print "Failed to execute $case \n"; - $total_failed++; - push (@failures ,(basename($case))); - } - } -} - -############################## -# Print summary of test run -############################## - -sub print_summary -{ - my $total = time()-$start; - my $h = 0; - my $m = 0; - my $s = 0; - my $exit = 0; - print "=== Timing: =================== TEST RUN COMPLETED! ============================\n"; - if ($total > 60*60) { - $h = int($total/60/60); - $s = int($total - $h*60*60); - - $m = int($s/60); - $s = 0; - print "Total: $h hours $m minutes\n"; - } elsif ($total > 60) { - $m = int($total/60); - $s = int($total - $m*60); - print "Total: $m minutes $s seconds\n"; - } else { - $s = int($total); - print "Total: $s seconds\n"; - } - - print "=== Failures: =================================================================="; - foreach my $failed (@failures) { - print $failed."\n"; - $exit = 1; - } - print "=== Totals: ".$total_tests." tests, ".($total_tests-$total_failed). - " passes, ".$failed_insignificants. - " insignificant fails ======================\n"; - return $exit; -} - - -sub waitForProcess -{ - my $process=shift; - my $action=shift; - my $timeout=shift; - my $sleepPeriod=shift; - $sleepPeriod=1 if !defined($sleepPeriod); - print "Waiting for $process ".$timeout*$sleepPeriod." seconds to" if (!$silent); - print $action?" start...\n":" die...\n" if (!$silent); - while ($timeout--) - { - my $output = `$adb_tool $device_serial shell ps 2>&1`; # get current processes - #FIXME check why $output is not matching m/.*S $process\n/ or m/.*S $process$/ (eol) - my $res=($output =~ m/.*S $process/)?1:0; # check the procress - if ($action == $res) - { - print "... succeed\n" if (!$silent); - return 1; - } - sleep($sleepPeriod); - print "timeount in ".$timeout*$sleepPeriod." seconds\n" if (!$silent); - } - print "... failed\n" if (!$silent); - return 0; -} - -my $src_dir_qt=abs_path(dirname($0)."/../../.."); -my $quadruplor_dir="$src_dir_qt/tests/auto/android"; -my $qmake_path="$src_dir_qt/bin/qmake"; -my $androiddeployqt_path="$src_dir_qt/bin/androiddeployqt"; -my $tests_dir="$src_dir_qt/tests$testsubset"; -my $temp_dir=tempdir(CLEANUP => 1); -my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); -my $output_dir=$stack[0]."/".(1900+$year)."-$mon-$mday-$hour:$min"; -mkdir($output_dir); -unlink("latest"); -system(" ln -s $output_dir latest"); -my $sdk_api=0; -my $output = `$adb_tool $device_serial shell getprop`; # get device properties -if ($output =~ m/.*\[ro.build.version.sdk\]: \[(\d+)\]/) -{ - $sdk_api=int($1); - $sdk_api=5 if ($sdk_api>5 && $sdk_api<8); - $sdk_api=9 if ($sdk_api>9); -} - -sub checkXMLOutput -{ - print color 'bold red'; - my $fileName = shift; - my $XMLOutput = eval { XMLin($fileName, ForceArray => 1) }; - if (!defined($XMLOutput)) { - print "Can't parse the $fileName file, probably the test crased.\n"; - print color 'reset'; - die "Stopping\n" if $stop_on_fail; - return; - } - my $testName = $XMLOutput->{name}; - my $fail = 0; - while (my($node_key, $node_valule) = each (%{$XMLOutput})) { - next if $node_key ne "TestFunction"; - while (my($function_key, $function_valule) = each (%{$node_valule})) { - while (my($test_key, $test_valule) = each (%{$function_valule})) { - next if $test_key ne "Incident"; - for my $incident (@{$test_valule}) { - if (($incident->{type} ne "pass") && ($incident->{type} ne "xfail")) { - print "test $testName::$function_key failed $incident->{file}:$incident->{line}\n"; - $fail = 1; - } - } - } - } - } - print color 'reset'; - die "Stopping\n" if $stop_on_fail and $fail; -} - -sub startTest -{ - my $testName = shift; - my $packageName = "org.qtproject.example.tst_$testName"; - my $intentName = "$packageName/org.qtproject.qt5.android.bindings.QtActivity"; - my $output_file = shift; - my $insignificance = shift; - my $get_xml= 0; - my $get_txt= 0; - my $testLib =""; - if ($log_out eq "xml") { - $testLib="-o /data/data/$packageName/output.xml,xml"; - $get_xml = 1; - } elsif ($log_out eq "txt") { - $testLib="-o /data/data/$packageName/output.txt,txt"; - $get_txt = 1; - } else { - $testLib="-o /data/data/$packageName/output.xml,xml -o /data/data/$packageName/output.txt,txt"; - $get_xml = 1; - $get_txt = 1; - } - - my $cmd="${adb_tool} ${device_serial} shell am start -e applicationArguments \"${testLib}\" -n ${intentName}"; - my $res = qx(${cmd}); - print $res if (!$silent); - #wait to start (if it has not started and quit already) - waitForProcess($packageName,1,10); - - #wait to stop - unless(waitForProcess($packageName,0,$time_out,5)) - { - #killProcess($packageName); - print "Someone should kill $packageName\n"; - system("$adb_tool $device_serial uninstall $packageName") if ($uninstall); - return 1; - } - - # Wait for three seconds to allow process to write all data - sleep(3); - - system("$adb_tool $device_serial pull /data/data/$packageName/output.xml $output_dir/$output_file.xml") if ($get_xml); - - system("$adb_tool $device_serial pull /data/data/$packageName/output.txt $output_dir/$output_file.txt") if ($get_txt); - if ($get_txt){ - print "Test results for $packageName:\n"; - my $insig = - print_output("$output_dir/$output_file.txt", $packageName, $insignificance); - } - system("$adb_tool $device_serial uninstall $packageName") if ($uninstall); - - checkXMLOutput("$output_dir/$output_file.xml") if ($get_xml); - return 1; -} - -########### build qt tests and benchmarks ########### -pushd($tests_dir); -print "Building $tests_dir \n"; -system("make distclean") if ($make_clean); -system("$qmake_path -r") == 0 or die "Can't run qmake\n"; #exec qmake -system("make -j$jobs") == 0 or warn "Can't build all tests\n"; #exec make - -my $testsFiles = ""; -if ($testcase) { - $testsFiles=`find . -name libtst_$testcase.so`; # only tests -} else { - $testsFiles=`find . -name libtst_*.so`; # only tests -} - -foreach (split("\n",$testsFiles)) -{ - chomp; #remove white spaces - pushd(abs_path(dirname($_))); # cd to application dir - my $insig = check_if_insignificant(); - my $cmd="make INSTALL_ROOT=${temp_dir} install"; - my $res = qx(${cmd}); - print $res if (!$silent); - my $application=basename(cwd); - if ($silent) { - $cmd="$androiddeployqt_path --install ${deployqt_device_serial} --output ${temp_dir} --deployment debug --verbose --input android-libtst_${application}.so-deployment-settings.json >/dev/null 2>&1"; - } else { - $cmd="$androiddeployqt_path --install ${deployqt_device_serial} --output ${temp_dir} --deployment debug --verbose --input android-libtst_${application}.so-deployment-settings.json"; - } - $res = qx(${cmd}); - print $res if (!$silent); - my $output_name=dirname($_); - $output_name =~ s/\.//; # remove first "." character - $output_name =~ s/\///; # remove first "/" character - $output_name =~ s/\//_/g; # replace all "/" with "_" - $output_name=$application unless($output_name); - $time_out=$max_runtime*60/5; # 5 minutes time out for a normal test - - $applicationLibrary = `find $temp_dir -name libtst_bench_$application.so`; - - if ($applicationLibrary) - { - $time_out=5*60/5; # 10 minutes for a benchmark - $application = "bench_$application"; - } - else - { - $applicationLibrary = `find $temp_dir -name libtst_$application.so`; - } - - if (!$applicationLibrary) - { - print "Can't find application binary libtst_$application.so in $temp_dir!\n"; - } - else - { - startTest($application, "$output_name", $insig) or warn "Can't run $application ...\n"; - } - - popd(); - remove_tree( $temp_dir, {keep_root => 1} ); -} -print_summary() if ($ci_use); -popd(); - -__END__ - -=head1 NAME - -Script to run all qt tests/benchmarks to an android device/emulator - -=head1 SYNOPSIS - -runtests.pl [options] - -=head1 OPTIONS - -=over 8 -=item B<-f --fail> - -Stop the script when test fails. Default 0 - -=item B<-s --serial = serial> - -Device serial number. May be empty if only one device is attached. - -=item B<-t --test = test_subset> - -Tests subset (e.g. benchmarks, auto, auto/qbuffer, etc.). - -=item B<-c --clean> - -Clean tests before building them. - -=item B<-j --jobs = number> - -Make jobs when building tests. - -=item B<--sdk = sdk_path> - -Android SDK path. - -=item B<--ndk = ndk_path> - -Android NDK path. - -=item B<--ant = ant_tool_path> - -Ant tool path. - -=item B<--strip = strip_tool_path> - -Android strip tool path, used to deploy qt libs. - -=item B<--readelf = readelf_tool_path> - -Android readelf tool path, used to check if a test application uses qt OpenGL. - -=item B<--logtype = xml|txt|both> - -The format of log file, default is xml. - -=item B<--runtime = minutes> - -The timeout period before stopping individual tests from running. - -=item B<-silent> - -Suppress output of system commands. - -=item B<-ci> - -Enables checking if test is insignificant or not. Also prints test -summary after all tests has been executed. - -=item B<-uninstall> - -Uninstalls the test after has been executed. - -=item B<-h --help> - -Print a brief help message and exits. - -=item B<--man> - -Prints the manual page and exits. - -=back - -=head1 DESCRIPTION - -B<This program> will run all qt tests/benchmarks to an android device/emulator. - -=cut |