diff options
Diffstat (limited to 'tests/auto/testlib/selftests/tst_selftests.cpp')
| -rw-r--r-- | tests/auto/testlib/selftests/tst_selftests.cpp | 232 |
1 files changed, 145 insertions, 87 deletions
diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index c5ae337629..04185e95cd 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -1,35 +1,10 @@ -/**************************************************************************** -** -** Copyright (C) 2020 The Qt Company Ltd. -** Copyright (C) 2016 Intel Corporation. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the test suite of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:GPL-EXCEPT$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 3 as published by the Free Software -** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// Copyright (C) 2016 Intel Corporation. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only #include <QtCore/QCoreApplication> -#if QT_CONFIG(process) +QT_REQUIRE_CONFIG(process); #if QT_CONFIG(temporaryfile) # define USE_DIFF @@ -50,6 +25,8 @@ #include <QtTest/private/qemulationdetector_p.h> +using namespace Qt::StringLiterals; + struct BenchmarkResult { qint64 total; @@ -220,6 +197,8 @@ bool compareOutput(const QString &logger, const QString &subdir, continue; if (actualLineBA.endsWith(" : failure location")) continue; + if (actualLineBA.endsWith(" : message location")) + continue; if (actualLineBA.startsWith("Config: Using QtTest library") // Text build string || actualLineBA.startsWith(" <QtBuild") // XML, Light XML build string @@ -311,8 +290,7 @@ bool compareLine(const QString &logger, const QString &subdir, return compareBenchmarkResult(actualResult, expectedResult, errorMessage); } - if (actualLine.startsWith(QLatin1String(" <Duration msecs=")) - || actualLine.startsWith(QLatin1String("<Duration msecs="))) { + if (actualLine.contains(QLatin1String("<Duration msecs="))) { static QRegularExpression durationRegExp("<Duration msecs=\"[\\d\\.]+\"/>"); QRegularExpressionMatch match = durationRegExp.match(actualLine); if (match.hasMatch()) @@ -454,8 +432,8 @@ BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error) // format: // "function","[globaltag:]tag","metric",value_per_iteration,total,iterations QStringList split = line.split(','); - if (split.count() != 6) { - if (error) *error = QString("Wrong number of columns (%1)").arg(split.count()); + if (split.size() != 6) { + if (error) *error = QString("Wrong number of columns (%1)").arg(split.size()); return out; } @@ -481,11 +459,13 @@ BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error) // This code avoids using a QRegExp because QRegExp might be broken. // Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1) - QString sFirstNumber; - while (!remaining.isEmpty() && !remaining.at(0).isSpace()) { - sFirstNumber += remaining.at(0); - remaining.remove(0,1); - } + const auto begin = remaining.cbegin(); + auto it = std::find_if(begin, remaining.cend(), [](const auto ch) { + return ch.isSpace(); + }); + QString sFirstNumber{std::distance(begin, it), Qt::Uninitialized}; + std::move(begin, it, sFirstNumber.begin()); + remaining.erase(begin, it); remaining = remaining.trimmed(); // 4,000 -> 4000 @@ -658,8 +638,10 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const return true; #endif - if (test == "deleteLater" || test == "deleteLater_noApp" || test == "mouse") - return true; // Missing expectation files + if (!qEnvironmentVariableIsEmpty("WAYLAND_DISPLAY")) { + qDebug() << "TestLogger::shouldIgnoreTest() ignore" << test << "on wayland/xwayland!"; + return true; + } // These tests are affected by timing and whether the CPU tick counter // is monotonically increasing. They won't work on some machines so @@ -678,7 +660,7 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const return true; #endif -#if defined(QT_NO_EXCEPTIONS) || defined(Q_CC_INTEL) || defined(Q_OS_WIN) +#if defined(QT_NO_EXCEPTIONS) || defined(Q_OS_WIN) // Disable this test on Windows or for Intel compiler, as the run-times // will popup dialogs with warnings that uncaught exceptions were thrown if (test == "exceptionthrow") @@ -692,17 +674,18 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const #endif if (test == "benchlibcallgrind") { -#if !(defined(__GNUC__) && defined(__i386) && defined(Q_OS_LINUX)) - // Skip on platforms where callgrind is not available - return true; -#else +#if defined(__GNUC__) && (defined(__i386) || defined(__x86_64)) && defined(Q_OS_LINUX) // Check that it's actually available QProcess checkProcess; - QStringList args; - args << "--version"; + QStringList args{u"--version"_s}; checkProcess.start("valgrind", args); - if (!checkProcess.waitForFinished(-1)) + if (!checkProcess.waitForFinished(-1)) { WARN("Valgrind broken or not available. Not running benchlibcallgrind test!"); + return true; + } +#else + // Skip on platforms where callgrind is not available + return true; #endif } @@ -716,11 +699,15 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const || test == "benchliboptions" || test == "printdatatags" || test == "printdatatagswithglobaltags" - || test == "silent") + || test == "silent" + || test == "silent_fatal") return true; - // `crashes' will not output valid XML on platforms without a crash handler - if (test == "crashes") + // These tests produce variable output (callgrind because of #if-ery, + // crashes by virtue of platform differences in where the output cuts + // off), so only test them for one format, to avoid the need for several + // _n variants for each format. Also, crashes can produce invalid XML. + if (test == "crashes" || test == "benchlibcallgrind") return true; // this test prints out some floats in the testlog and the formatting is @@ -728,10 +715,11 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const if (test == "float") return true; - // these tests are quite slow, and running them for all the loggers significantly - // increases the overall test time. They do not really relate to logging, so it - // should be safe to run them just for the stdout loggers. - if (test == "benchlibcallgrind" || test == "sleep") + // This test is quite slow, and running it for all the loggers + // significantly increases the overall test time. It does not really + // relate to logging, so it should be safe to run it just for the stdout + // loggers. + if (test == "sleep") return true; } @@ -739,12 +727,10 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const || logger == QTestLog::LightXML || logger == QTestLog::JUnitXML)) return true; - if (logger == QTestLog::CSV && !test.startsWith("benchlib")) + // Skip benchmark for TeamCity logger, skip everything else for CSV: + if (logger == (test.startsWith("benchlib") ? QTestLog::TeamCity : QTestLog::CSV)) return true; - if (logger == QTestLog::TeamCity && test.startsWith("benchlib")) - return true; // Skip benchmark for TeamCity logger - if (logger != QTestLog::JUnitXML && test == "junit") return true; @@ -779,29 +765,30 @@ void checkErrorOutput(const QString &test, const QByteArray &errorOutput) || test == "benchlibcallgrind") return; -#ifdef Q_CC_MINGW - if (test == "blacklisted" // calls qFatal() - || test == "silent") // calls qFatal() -#endif - return; - #ifdef Q_OS_WIN if (test == "crashes") return; // Complains about uncaught exception #endif -#ifdef Q_OS_LINUX - // QEMU outputs to stderr about uncaught signals - if (QTestPrivate::isRunningArmOnX86() && - (test == "assert" - || test == "blacklisted" - || test == "crashes" - || test == "faildatatype" - || test == "failfetchtype" - || test == "silent" - )) +#ifdef Q_OS_UNIX + if (test == "assert" + || test == "crashes" + || test == "failfetchtype" + || test == "faildatatype") + return; // Outputs "Received signal 6 (SIGABRT)" +#endif + + if (test == "silent_fatal") { +#if defined(__SANITIZE_ADDRESS__) || __has_feature(address_sanitizer) + // Under ASan, this test is not silent + return; +#elif defined(Q_CC_MINGW) + // Originally QTBUG-29014 (I can't reproduce this -Thiago) return; #endif + if (QTestPrivate::isRunningArmOnX86()) + return; // QEMU outputs to stderr about uncaught signals + } INFO(errorOutput.toStdString()); REQUIRE(errorOutput.isEmpty()); @@ -941,8 +928,11 @@ static QProcessEnvironment testEnvironment() if (environment.isEmpty()) { const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment(); const bool preserveLibPath = qEnvironmentVariableIsSet("QT_PRESERVE_TESTLIB_PATH"); - foreach (const QString &key, systemEnvironment.keys()) { + const auto envKeys = systemEnvironment.keys(); + for (const QString &key : envKeys) { const bool useVariable = key == "PATH" || key == "QT_QPA_PLATFORM" + || key == "QTEST_THROW_ON_FAIL"_L1 || key == "QTEST_THROW_ON_SKIP"_L1 + || key == "ASAN_OPTIONS" #if defined(Q_OS_QNX) || key == "GRAPHICS_ROOT" || key == "TZ" #elif defined(Q_OS_UNIX) @@ -987,8 +977,7 @@ TestProcessResult runTestProcess(const QString &test, const QStringList &argumen const bool expectedCrash = test == "assert" || test == "exceptionthrow" || test == "fetchbogus" || test == "crashedterminate" || test == "faildatatype" || test == "failfetchtype" - || test == "crashes" || test == "silent" - || test == "blacklisted" || test == "watchdog"; + || test == "crashes" || test == "silent_fatal" || test == "watchdog"; if (expectedCrash) { environment.insert("QTEST_DISABLE_CORE_DUMP", "1"); @@ -1025,10 +1014,12 @@ TestProcessResult runTestProcess(const QString &test, const QStringList &argumen return { process.exitCode(), standardOutput, standardError }; } +enum class Throw { OnFail = 1 }; + /* Runs a single test and verifies the output against the expected results. */ -void runTest(const QString &test, const TestLoggers &requestedLoggers) +void runTest(const QString &test, const TestLoggers &requestedLoggers, Throw throwing = {}) { TestLoggers loggers; for (auto logger : requestedLoggers) { @@ -1042,6 +1033,10 @@ void runTest(const QString &test, const TestLoggers &requestedLoggers) QStringList arguments; for (auto logger : loggers) arguments += logger.arguments(test); + if (throwing == Throw::OnFail) // don't distinguish between throwonfail/throwonskip + arguments += {"-throwonfail", "-throwonskip"}; + else + arguments += {"-nothrowonfail", "-nothrowonskip"}; CAPTURE(test); CAPTURE(arguments); @@ -1068,9 +1063,9 @@ void runTest(const QString &test, const TestLoggers &requestedLoggers) /* Runs a single test and verifies the output against the expected result. */ -void runTest(const QString &test, const TestLogger &logger) +void runTest(const QString &test, const TestLogger &logger, Throw t = {}) { - runTest(test, TestLoggers{logger}); + runTest(test, TestLoggers{logger}, t); } // ----------------------- Catch helpers ----------------------- @@ -1205,27 +1200,91 @@ TEST_CASE("All loggers can be enabled at the same time") SCENARIO("Test output of the loggers is as expected") { static QStringList tests = QString(QT_STRINGIFY(SUBPROGRAMS)).split(' '); + if (QString override = qEnvironmentVariable("TST_SELFTEST_SUBPROGRAMS"); !override.isEmpty()) + tests = override.split(' ', Qt::SkipEmptyParts); auto logger = GENERATE(filter(isGenericCommandLineLogger, enums<QTestLog::LogMode>())); GIVEN("The " << logger << " logger") { for (QString test : tests) { AND_GIVEN("The " << test << " subtest") { - runTest(test, TestLogger(logger, StdoutOutput)); + WHEN("Throwing on failure or skip") { + runTest(test, TestLogger(logger, StdoutOutput), Throw::OnFail); + } + WHEN("Returning on failure or skip") { + runTest(test, TestLogger(logger, StdoutOutput)); + } } } } } -#endif // QT_CONFIG(process) +struct TestCase { + int expectedExitCode; + const char *cmdline; +}; + +SCENARIO("Exit code is as expected") +{ + // Listing of test command lines and expected exit codes + // NOTE: Use at least 2 spaces to separate arguments because some contain a space themselves. + const struct TestCase testCases[] = { + // 'pass' is a test with no data tags at all + { 0, "pass testNumber1" }, + { 1, "pass unknownFunction" }, + { 1, "pass testNumber1:blah" }, + { 1, "pass testNumber1:blah:blue" }, + // 'counting' is a test that has only local data tags + { 0, "counting testPassPass" }, + { 0, "counting testPassPass:row 1" }, + { 1, "counting testPassPass:blah" }, + { 1, "counting testPassPass:blah:row 1" }, + { 1, "counting testPassPass:blah:blue" }, + // 'globaldata' is a test with global and local data tags + { 0, "globaldata testGlobal" }, + { 0, "globaldata testGlobal:global=true" }, + { 0, "globaldata testGlobal:local=true" }, + { 0, "globaldata testGlobal:global=true:local=true" }, + { 0, "globaldata testGlobal -repeat 2" }, + { 1, "globaldata testGlobal:local=true:global=true" }, + { 1, "globaldata testGlobal:global=true:blah" }, + { 1, "globaldata testGlobal:blah:local=true" }, + { 1, "globaldata testGlobal:blah:global=true" }, + { 1, "globaldata testGlobal:blah" }, + { 1, "globaldata testGlobal:blah:blue" }, + // Passing multiple testcase:data on the command line + { 0, "globaldata testGlobal:global=true skipSingle:global=true:local=true" }, + { 1, "globaldata testGlobal:blah skipSingle:global=true:local=true" }, + { 1, "globaldata testGlobal:global=true skipSingle:blah" }, + { 2, "globaldata testGlobal:blah skipSingle:blue" }, + // Passing -repeat argument + { 1, "pass testNumber1 -repeat" }, + { 0, "pass testNumber1 -repeat 1" }, + { 0, "pass testNumber1 -repeat 1 -o out.xml,xml" }, + { 0, "pass testNumber1 -repeat 2" }, + { 0, "pass testNumber1 -repeat 2 -o -,txt" }, + { 0, "pass testNumber1 -repeat 2 -o -,txt -o log.txt,txt" }, + { 1, "pass testNumber1 -repeat 2 -o log.xml,xml" }, + { 1, "pass testNumber1 -repeat 2 -o -,txt -o -,xml" }, + }; + + size_t n_testCases = sizeof(testCases) / sizeof(*testCases); + for (size_t i = 0; i < n_testCases; i++) { + GIVEN("The command line: " << testCases[i].cmdline) { + const QStringList cmdSplit = QString(testCases[i].cmdline) + .split(QRegularExpression(" +")); // at least 2 spaces + const QString test = cmdSplit[0]; + const QStringList args = cmdSplit.sliced(1); + auto runResult = runTestProcess(test, args); + REQUIRE(runResult.exitCode == testCases[i].expectedExitCode); + } + } +} // ----------------------- Entrypoint ----------------------- int main(int argc, char **argv) { -#if !QT_CONFIG(process) - return 0; -#else std::vector<const char*> args(argv, argv + argc); static auto kRebaseArgument = "--rebase"; @@ -1273,6 +1332,5 @@ int main(int argc, char **argv) } return result; -#endif } |