diff options
author | BogDan Vatra <bogdan@kde.org> | 2019-05-24 12:34:28 +0300 |
---|---|---|
committer | BogDan Vatra <bogdan@kde.org> | 2019-06-06 15:25:07 +0300 |
commit | a5e03f59f4bbfe1cf40ec084d137bf4c91923731 (patch) | |
tree | 645da67d216ce30afe7a89622ac45f9640c2cf8e /src/tools | |
parent | 784c94c7272684439abfa99273f5688d31ab2959 (diff) |
Say hello to androidtestrunner
androidtestrunner is a tool needed to run qt tests on Android.
Now you can run tests as simple as you run them on Linux, macOS,
Windows.
"$ make check" it's all you need to run tests on the default android
device.
ANDROID_DEVICE_SERIAL env variable can be used to use a specific android
serial.
Use cases:
$ make -j1 check
-j1 is needed to make sure we don't run multiple tests in parallel.
$ ANDROID_DEVICE_SERIAL="emulator-5554" make check
Run the test on "emulator-5554"
$ make TESTARGS="-- -xml" check
Switch to xml output. All params after -- are passed to test
application.
$ make TESTARGS="-- -o out.xml,xml -o out.txt,txt -o -,tap -vs" check
Create two files out.xml and out.txt in the current folder and print
"tap" format to stdout and enable logging of every signal emission.
[ChangeLog][Android] Make it easy to run Qt tests on Android.
"$ make check" is all it's needed to run a test on an Android device.
Change-Id: I1a7f64b62608f7367b5a6aabf5d6c6e7e50242e6
Reviewed-by: Jędrzej Nowacki <jedrzej.nowacki@qt.io>
Diffstat (limited to 'src/tools')
-rw-r--r-- | src/tools/androidtestrunner/androidtestrunner.pro | 13 | ||||
-rw-r--r-- | src/tools/androidtestrunner/main.cpp | 464 |
2 files changed, 477 insertions, 0 deletions
diff --git a/src/tools/androidtestrunner/androidtestrunner.pro b/src/tools/androidtestrunner/androidtestrunner.pro new file mode 100644 index 0000000000..641d3e0003 --- /dev/null +++ b/src/tools/androidtestrunner/androidtestrunner.pro @@ -0,0 +1,13 @@ +option(host_build) +CONFIG += console + +SOURCES += \ + main.cpp + +# Required for declarations of popen/pclose on Windows +windows: QMAKE_CXXFLAGS += -U__STRICT_ANSI__ + +DEFINES += QT_NO_CAST_FROM_ASCII QT_NO_CAST_TO_ASCII +DEFINES += QT_NO_FOREACH + +load(qt_app) diff --git a/src/tools/androidtestrunner/main.cpp b/src/tools/androidtestrunner/main.cpp new file mode 100644 index 0000000000..c7c27c97df --- /dev/null +++ b/src/tools/androidtestrunner/main.cpp @@ -0,0 +1,464 @@ +/**************************************************************************** +** +** Copyright (C) 2019 BogDan Vatra <bogdan@kde.org> +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the tools applications of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include <QCoreApplication> +#include <QDir> +#include <QHash> +#include <QRegExp> +#include <QXmlStreamReader> + +#include <algorithm> +#include <chrono> +#include <functional> +#include <thread> + +#ifdef Q_CC_MSVC +#define popen _popen +#define QT_POPEN_READ "rb" +#define pclose _pclose +#else +#define QT_POPEN_READ "r" +#endif + +struct Options +{ + bool helpRequested = false; + bool verbose = false; + std::chrono::seconds timeout{300}; // 5minutes + QString androidDeployQtCommand; + QString buildPath; + QString adbCommand{QStringLiteral("adb")}; + QString makeCommand; + QString package; + QString activity; + QStringList testArgsList; + QHash<QString, QString> outFiles; + QString testArgs; + QHash<QString, std::function<bool(const QByteArray &)>> checkFiles = { + {QStringLiteral("txt"), [](const QByteArray &data) -> bool { + return data.indexOf("\nFAIL! : ") < 0; + }}, + {QStringLiteral("csv"), [](const QByteArray &/*data*/) -> bool { + // It seems csv is broken + return true; + }}, + {QStringLiteral("xml"), [](const QByteArray &data) -> bool { + QXmlStreamReader reader{data}; + while (!reader.atEnd()) { + reader.readNext(); + if (reader.isStartElement() && reader.name() == QStringLiteral("Incident") && + reader.attributes().value(QStringLiteral("type")).toString() == QStringLiteral("fail")) { + return false; + } + } + return true; + }}, + {QStringLiteral("lightxml"), [](const QByteArray &data) -> bool { + return data.indexOf("\n<Incident type=\"fail\" ") < 0; + }}, + {QStringLiteral("xunitxml"), [](const QByteArray &data) -> bool { + QXmlStreamReader reader{data}; + while (!reader.atEnd()) { + reader.readNext(); + if (reader.isStartElement() && reader.name() == QStringLiteral("testcase") && + reader.attributes().value(QStringLiteral("result")).toString() == QStringLiteral("fail")) { + return false; + } + } + return true; + }}, + {QStringLiteral("teamcity"), [](const QByteArray &data) -> bool { + return data.indexOf("' message='Failure! |[Loc: ") < 0; + }}, + {QStringLiteral("tap"), [](const QByteArray &data) -> bool { + return data.indexOf("\nnot ok ") < 0; + }}, + }; +}; + +static Options g_options; + +static bool execCommand(const QString &command, QByteArray *output = nullptr, bool verbose = false) +{ +#if defined(Q_OS_WIN32) + QString processedCommand = QLatin1Char('\"') + command + QLatin1Char('\"'); +#else + const QString& processedCommand = command; +#endif + + if (verbose) + fprintf(stdout, "Execute %s\n", processedCommand.toUtf8().constData()); + FILE *process = popen(processedCommand.toUtf8().constData(), QT_POPEN_READ); + + if (!process) { + fprintf(stderr, "Cannot execute command %s", qPrintable(processedCommand)); + return false; + } + char buffer[512]; + while (fgets(buffer, sizeof(buffer), process)) { + if (output) + output->append(buffer); + if (verbose) + fprintf(stdout, "%s", buffer); + } + return pclose(process) == 0; +} + +// Copy-pasted from qmake/library/ioutil.cpp +inline static bool hasSpecialChars(const QString &arg, const uchar (&iqm)[16]) +{ + for (int x = arg.length() - 1; x >= 0; --x) { + ushort c = arg.unicode()[x].unicode(); + if ((c < sizeof(iqm) * 8) && (iqm[c / 8] & (1 << (c & 7)))) + return true; + } + return false; +} + +static QString shellQuoteUnix(const QString &arg) +{ + // Chars that should be quoted (TM). This includes: + static const uchar iqm[] = { + 0xff, 0xff, 0xff, 0xff, 0xdf, 0x07, 0x00, 0xd8, + 0x00, 0x00, 0x00, 0x38, 0x01, 0x00, 0x00, 0x78 + }; // 0-32 \'"$`<>|;&(){}*?#!~[] + + if (!arg.length()) + return QStringLiteral("\"\""); + + QString ret(arg); + if (hasSpecialChars(ret, iqm)) { + ret.replace(QLatin1Char('\''), QStringLiteral("'\\''")); + ret.prepend(QLatin1Char('\'')); + ret.append(QLatin1Char('\'')); + } + return ret; +} + +static QString shellQuoteWin(const QString &arg) +{ + // Chars that should be quoted (TM). This includes: + // - control chars & space + // - the shell meta chars "&()<>^| + // - the potential separators ,;= + static const uchar iqm[] = { + 0xff, 0xff, 0xff, 0xff, 0x45, 0x13, 0x00, 0x78, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x10 + }; + + if (!arg.length()) + return QStringLiteral("\"\""); + + QString ret(arg); + if (hasSpecialChars(ret, iqm)) { + // Quotes are escaped and their preceding backslashes are doubled. + // It's impossible to escape anything inside a quoted string on cmd + // level, so the outer quoting must be "suspended". + ret.replace(QRegExp(QStringLiteral("(\\\\*)\"")), QStringLiteral("\"\\1\\1\\^\"\"")); + // The argument must not end with a \ since this would be interpreted + // as escaping the quote -- rather put the \ behind the quote: e.g. + // rather use "foo"\ than "foo\" + int i = ret.length(); + while (i > 0 && ret.at(i - 1) == QLatin1Char('\\')) + --i; + ret.insert(i, QLatin1Char('"')); + ret.prepend(QLatin1Char('"')); + } + return ret; +} + +static QString shellQuote(const QString &arg) +{ + if (QDir::separator() == QLatin1Char('\\')) + return shellQuoteWin(arg); + else + return shellQuoteUnix(arg); +} + +static bool parseOptions() +{ + QStringList arguments = QCoreApplication::arguments(); + int i = 1; + for (; i < arguments.size(); ++i) { + const QString &argument = arguments.at(i); + if (argument.compare(QStringLiteral("--androiddeployqt"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.androidDeployQtCommand = arguments.at(++i).trimmed(); + } else if (argument.compare(QStringLiteral("--adb"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.adbCommand = arguments.at(++i); + } else if (argument.compare(QStringLiteral("--path"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.buildPath = arguments.at(++i); + } else if (argument.compare(QStringLiteral("--make"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.makeCommand = arguments.at(++i); + } else if (argument.compare(QStringLiteral("--activity"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.activity = arguments.at(++i); + } else if (argument.compare(QStringLiteral("--timeout"), Qt::CaseInsensitive) == 0) { + if (i + 1 == arguments.size()) + g_options.helpRequested = true; + else + g_options.timeout = std::chrono::seconds{arguments.at(++i).toInt()}; + } else if (argument.compare(QStringLiteral("--help"), Qt::CaseInsensitive) == 0) { + g_options.helpRequested = true; + } else if (argument.compare(QStringLiteral("--verbose"), Qt::CaseInsensitive) == 0) { + g_options.verbose = true; + } else if (argument.compare(QStringLiteral("--"), Qt::CaseInsensitive) == 0) { + ++i; + break; + } else { + g_options.testArgsList << arguments.at(i); + } + } + for (;i < arguments.size(); ++i) + g_options.testArgsList << arguments.at(i); + + if (g_options.helpRequested || g_options.androidDeployQtCommand.isEmpty() || g_options.buildPath.isEmpty()) + return false; + + QString serial = qEnvironmentVariable("ANDROID_DEVICE_SERIAL"); + if (!serial.isEmpty()) + g_options.adbCommand += QStringLiteral(" -s %1").arg(serial); + return true; +} + +static void printHelp() +{// "012345678901234567890123456789012345678901234567890123456789012345678901" + fprintf(stderr, "Syntax: %s <options> -- [TESTARGS] \n" + "\n" + " Creates an Android package in a temp directory <destination> and\n" + " runs it on the default emulator/device or on the one specified by\n" + " \"ANDROID_DEVICE_SERIAL\" environment variable.\n\n" + " Mandatory arguments:\n" + " --androiddeployqt <androiddeployqt cmd>: The androiddeployqt:\n" + " path including its additional arguments.\n" + " --path <path>: The path where androiddeployqt will build the .apk.\n" + " Optional arguments:\n" + " --adb <adb cmd>: The Android ADB command. If missing the one from\n" + " $PATH will be used.\n" + " --activity <acitvity>: The Activity to run. If missing the first\n" + " activity from AndroidManifest.qml file will be used.\n" + " --timout <seconds>: Timeout to run the test.\n" + " Default is 5 minutes.\n" + " --make <make cmd>: make command, needed to install the qt library.\n" + " If make is missing make sure the --path is set.\n" + " -- arguments that will be passed to the test application.\n" + " --verbose: Prints out information during processing.\n" + " --help: Displays this information.\n\n", + qPrintable(QCoreApplication::arguments().at(0)) + ); +} + +static QString packageNameFromAndroidManifest(const QString &androidManifestPath) +{ + QFile androidManifestXml(androidManifestPath); + if (androidManifestXml.open(QIODevice::ReadOnly)) { + QXmlStreamReader reader(&androidManifestXml); + while (!reader.atEnd()) { + reader.readNext(); + if (reader.isStartElement() && reader.name() == QStringLiteral("manifest")) + return reader.attributes().value(QStringLiteral("package")).toString(); + } + } + return {}; +} + +static QString activityFromAndroidManifest(const QString &androidManifestPath) +{ + QFile androidManifestXml(androidManifestPath); + if (androidManifestXml.open(QIODevice::ReadOnly)) { + QXmlStreamReader reader(&androidManifestXml); + while (!reader.atEnd()) { + reader.readNext(); + if (reader.isStartElement() && reader.name() == QStringLiteral("activity")) + return reader.attributes().value(QStringLiteral("android:name")).toString(); + } + } + return {}; +} + +static void setOutputFile(QString file, QString format) +{ + if (file.isEmpty()) + file = QStringLiteral("-"); + if (format.isEmpty()) + format = QStringLiteral("txt"); + + g_options.outFiles[format] = file; +} + +static bool parseTestArgs() +{ + QRegExp newLoggingFormat{QStringLiteral("(.*),(txt|csv|xunitxml|xml|lightxml|teamcity|tap)")}; + QRegExp oldFormats{QStringLiteral("-(txt|csv|xunitxml|xml|lightxml|teamcity|tap)")}; + + QString file; + QString logType; + QString unhandledArgs; + for (int i = 0; i < g_options.testArgsList.size(); ++i) { + const QString &arg = g_options.testArgsList[i].trimmed(); + if (arg == QStringLiteral("-o")) { + if (i >= g_options.testArgsList.size() - 1) + return false; // missing file argument + + const auto &filePath = g_options.testArgsList[++i]; + if (!newLoggingFormat.exactMatch(filePath)) { + file = filePath; + } else { + const auto capturedTexts = newLoggingFormat.capturedTexts(); + setOutputFile(capturedTexts.at(1), capturedTexts.at(2)); + } + } else if (oldFormats.exactMatch(arg)) { + logType = oldFormats.capturedTexts().at(1); + } else { + unhandledArgs += QStringLiteral(" %1").arg(arg); + } + } + if (g_options.outFiles.isEmpty() || !file.isEmpty() || !logType.isEmpty()) + setOutputFile(file, logType); + + for (const auto &format : g_options.outFiles.keys()) + g_options.testArgs += QStringLiteral(" -o output.%1,%1").arg(format); + + g_options.testArgs += unhandledArgs; + g_options.testArgs = QStringLiteral("shell am start -e applicationArguments \"%1\" -n %2/%3").arg(shellQuote(g_options.testArgs.trimmed()), + g_options.package, + g_options.activity); + return true; +} + +static bool isRunning() { + QByteArray output; + if (!execCommand(QStringLiteral("%1 shell 'ps | grep \" %2\"'").arg(g_options.adbCommand, + shellQuoteUnix(g_options.package)), &output)) { + + return false; + } + return output.indexOf(" " + g_options.package.toUtf8()) > -1; +} + +static bool waitToFinish() +{ + using clock = std::chrono::system_clock; + auto start = clock::now(); + // wait to start + while (!isRunning()) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if ((clock::now() - start) > std::chrono::seconds{10}) + return false; + } + + // Wait to finish + while (isRunning()) { + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + if ((clock::now() - start) > g_options.timeout) + return false; + } + return true; +} + + +static bool pullFiles() +{ + bool ret = true; + for (auto it = g_options.outFiles.constBegin(); it != g_options.outFiles.end(); ++it) { + QByteArray output; + if (!execCommand(QStringLiteral("%1 shell run-as %2 cat files/output.%3") + .arg(g_options.adbCommand, g_options.package, it.key()), &output)) { + return false; + } + auto checkerIt = g_options.checkFiles.find(it.key()); + ret &= checkerIt != g_options.checkFiles.end() && checkerIt.value()(output); + if (it.value() == QStringLiteral("-")){ + fprintf(stdout, "%s", output.constData()); + fflush(stdout); + } else { + QFile out{it.value()}; + if (!out.open(QIODevice::WriteOnly)) + return false; + out.write(output); + } + } + return ret; +} + +int main(int argc, char *argv[]) +{ + QCoreApplication a(argc, argv); + if (!parseOptions()) { + printHelp(); + return 1; + } + + if (!g_options.makeCommand.isEmpty()) { + // we need to run make INSTALL_ROOT=path install to install the application file(s) first + if (!execCommand(QStringLiteral("%1 INSTALL_ROOT=%2 install") + .arg(g_options.makeCommand, g_options.buildPath), nullptr, g_options.verbose)) { + return 1; + } + } + // Run androiddeployqt + static auto verbose = g_options.verbose ? QStringLiteral("--verbose") : QStringLiteral(); + if (!execCommand(QStringLiteral("%1 %3 --gradle --reinstall --output %2").arg(g_options.androidDeployQtCommand, + g_options.buildPath, + verbose), nullptr, g_options.verbose)) { + return 1; + } + + QString manifest = g_options.buildPath + QStringLiteral("/AndroidManifest.xml"); + g_options.package = packageNameFromAndroidManifest(manifest); + if (g_options.activity.isEmpty()) + g_options.activity = activityFromAndroidManifest(manifest); + + // parseTestArgs depends on g_options.package + if (!parseTestArgs()) + return 1; + + // start the tests + bool res = execCommand(QStringLiteral("%1 %2").arg(g_options.adbCommand, g_options.testArgs), + nullptr, g_options.verbose) && waitToFinish(); + if (res) + res &= pullFiles(); + res &= execCommand(QStringLiteral("%1 uninstall %2").arg(g_options.adbCommand, g_options.package), + nullptr, g_options.verbose); + fflush(stdout); + return res ? 0 : 1; +} |