diff options
Diffstat (limited to 'tests/auto/testlib/selftests/tst_selftests.cpp')
| -rw-r--r-- | tests/auto/testlib/selftests/tst_selftests.cpp | 150 |
1 files changed, 118 insertions, 32 deletions
diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index 5f67d0e25b..04185e95cd 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -1,6 +1,6 @@ // Copyright (C) 2021 The Qt Company Ltd. // Copyright (C) 2016 Intel Corporation. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only #include <QtCore/QCoreApplication> @@ -432,8 +432,8 @@ BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error) // format: // "function","[globaltag:]tag","metric",value_per_iteration,total,iterations QStringList split = line.split(','); - if (split.count() != 6) { - if (error) *error = QString("Wrong number of columns (%1)").arg(split.count()); + if (split.size() != 6) { + if (error) *error = QString("Wrong number of columns (%1)").arg(split.size()); return out; } @@ -459,11 +459,13 @@ BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error) // This code avoids using a QRegExp because QRegExp might be broken. // Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1) - QString sFirstNumber; - while (!remaining.isEmpty() && !remaining.at(0).isSpace()) { - sFirstNumber += remaining.at(0); - remaining.remove(0,1); - } + const auto begin = remaining.cbegin(); + auto it = std::find_if(begin, remaining.cend(), [](const auto ch) { + return ch.isSpace(); + }); + QString sFirstNumber{std::distance(begin, it), Qt::Uninitialized}; + std::move(begin, it, sFirstNumber.begin()); + remaining.erase(begin, it); remaining = remaining.trimmed(); // 4,000 -> 4000 @@ -636,6 +638,11 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const return true; #endif + if (!qEnvironmentVariableIsEmpty("WAYLAND_DISPLAY")) { + qDebug() << "TestLogger::shouldIgnoreTest() ignore" << test << "on wayland/xwayland!"; + return true; + } + // These tests are affected by timing and whether the CPU tick counter // is monotonically increasing. They won't work on some machines so // leave them off by default. Feel free to enable them for your own @@ -692,7 +699,8 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const || test == "benchliboptions" || test == "printdatatags" || test == "printdatatagswithglobaltags" - || test == "silent") + || test == "silent" + || test == "silent_fatal") return true; // These tests produce variable output (callgrind because of #if-ery, @@ -757,29 +765,30 @@ void checkErrorOutput(const QString &test, const QByteArray &errorOutput) || test == "benchlibcallgrind") return; -#ifdef Q_CC_MINGW - if (test == "blacklisted" // calls qFatal() - || test == "silent") // calls qFatal() -#endif - return; - #ifdef Q_OS_WIN if (test == "crashes") return; // Complains about uncaught exception #endif -#ifdef Q_OS_LINUX - // QEMU outputs to stderr about uncaught signals - if (QTestPrivate::isRunningArmOnX86() && - (test == "assert" - || test == "blacklisted" - || test == "crashes" - || test == "faildatatype" - || test == "failfetchtype" - || test == "silent" - )) +#ifdef Q_OS_UNIX + if (test == "assert" + || test == "crashes" + || test == "failfetchtype" + || test == "faildatatype") + return; // Outputs "Received signal 6 (SIGABRT)" +#endif + + if (test == "silent_fatal") { +#if defined(__SANITIZE_ADDRESS__) || __has_feature(address_sanitizer) + // Under ASan, this test is not silent + return; +#elif defined(Q_CC_MINGW) + // Originally QTBUG-29014 (I can't reproduce this -Thiago) return; #endif + if (QTestPrivate::isRunningArmOnX86()) + return; // QEMU outputs to stderr about uncaught signals + } INFO(errorOutput.toStdString()); REQUIRE(errorOutput.isEmpty()); @@ -919,8 +928,11 @@ static QProcessEnvironment testEnvironment() if (environment.isEmpty()) { const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment(); const bool preserveLibPath = qEnvironmentVariableIsSet("QT_PRESERVE_TESTLIB_PATH"); - foreach (const QString &key, systemEnvironment.keys()) { + const auto envKeys = systemEnvironment.keys(); + for (const QString &key : envKeys) { const bool useVariable = key == "PATH" || key == "QT_QPA_PLATFORM" + || key == "QTEST_THROW_ON_FAIL"_L1 || key == "QTEST_THROW_ON_SKIP"_L1 + || key == "ASAN_OPTIONS" #if defined(Q_OS_QNX) || key == "GRAPHICS_ROOT" || key == "TZ" #elif defined(Q_OS_UNIX) @@ -965,8 +977,7 @@ TestProcessResult runTestProcess(const QString &test, const QStringList &argumen const bool expectedCrash = test == "assert" || test == "exceptionthrow" || test == "fetchbogus" || test == "crashedterminate" || test == "faildatatype" || test == "failfetchtype" - || test == "crashes" || test == "silent" - || test == "blacklisted" || test == "watchdog"; + || test == "crashes" || test == "silent_fatal" || test == "watchdog"; if (expectedCrash) { environment.insert("QTEST_DISABLE_CORE_DUMP", "1"); @@ -1003,10 +1014,12 @@ TestProcessResult runTestProcess(const QString &test, const QStringList &argumen return { process.exitCode(), standardOutput, standardError }; } +enum class Throw { OnFail = 1 }; + /* Runs a single test and verifies the output against the expected results. */ -void runTest(const QString &test, const TestLoggers &requestedLoggers) +void runTest(const QString &test, const TestLoggers &requestedLoggers, Throw throwing = {}) { TestLoggers loggers; for (auto logger : requestedLoggers) { @@ -1020,6 +1033,10 @@ void runTest(const QString &test, const TestLoggers &requestedLoggers) QStringList arguments; for (auto logger : loggers) arguments += logger.arguments(test); + if (throwing == Throw::OnFail) // don't distinguish between throwonfail/throwonskip + arguments += {"-throwonfail", "-throwonskip"}; + else + arguments += {"-nothrowonfail", "-nothrowonskip"}; CAPTURE(test); CAPTURE(arguments); @@ -1046,9 +1063,9 @@ void runTest(const QString &test, const TestLoggers &requestedLoggers) /* Runs a single test and verifies the output against the expected result. */ -void runTest(const QString &test, const TestLogger &logger) +void runTest(const QString &test, const TestLogger &logger, Throw t = {}) { - runTest(test, TestLoggers{logger}); + runTest(test, TestLoggers{logger}, t); } // ----------------------- Catch helpers ----------------------- @@ -1183,18 +1200,87 @@ TEST_CASE("All loggers can be enabled at the same time") SCENARIO("Test output of the loggers is as expected") { static QStringList tests = QString(QT_STRINGIFY(SUBPROGRAMS)).split(' '); + if (QString override = qEnvironmentVariable("TST_SELFTEST_SUBPROGRAMS"); !override.isEmpty()) + tests = override.split(' ', Qt::SkipEmptyParts); auto logger = GENERATE(filter(isGenericCommandLineLogger, enums<QTestLog::LogMode>())); GIVEN("The " << logger << " logger") { for (QString test : tests) { AND_GIVEN("The " << test << " subtest") { - runTest(test, TestLogger(logger, StdoutOutput)); + WHEN("Throwing on failure or skip") { + runTest(test, TestLogger(logger, StdoutOutput), Throw::OnFail); + } + WHEN("Returning on failure or skip") { + runTest(test, TestLogger(logger, StdoutOutput)); + } } } } } +struct TestCase { + int expectedExitCode; + const char *cmdline; +}; + +SCENARIO("Exit code is as expected") +{ + // Listing of test command lines and expected exit codes + // NOTE: Use at least 2 spaces to separate arguments because some contain a space themselves. + const struct TestCase testCases[] = { + // 'pass' is a test with no data tags at all + { 0, "pass testNumber1" }, + { 1, "pass unknownFunction" }, + { 1, "pass testNumber1:blah" }, + { 1, "pass testNumber1:blah:blue" }, + // 'counting' is a test that has only local data tags + { 0, "counting testPassPass" }, + { 0, "counting testPassPass:row 1" }, + { 1, "counting testPassPass:blah" }, + { 1, "counting testPassPass:blah:row 1" }, + { 1, "counting testPassPass:blah:blue" }, + // 'globaldata' is a test with global and local data tags + { 0, "globaldata testGlobal" }, + { 0, "globaldata testGlobal:global=true" }, + { 0, "globaldata testGlobal:local=true" }, + { 0, "globaldata testGlobal:global=true:local=true" }, + { 0, "globaldata testGlobal -repeat 2" }, + { 1, "globaldata testGlobal:local=true:global=true" }, + { 1, "globaldata testGlobal:global=true:blah" }, + { 1, "globaldata testGlobal:blah:local=true" }, + { 1, "globaldata testGlobal:blah:global=true" }, + { 1, "globaldata testGlobal:blah" }, + { 1, "globaldata testGlobal:blah:blue" }, + // Passing multiple testcase:data on the command line + { 0, "globaldata testGlobal:global=true skipSingle:global=true:local=true" }, + { 1, "globaldata testGlobal:blah skipSingle:global=true:local=true" }, + { 1, "globaldata testGlobal:global=true skipSingle:blah" }, + { 2, "globaldata testGlobal:blah skipSingle:blue" }, + // Passing -repeat argument + { 1, "pass testNumber1 -repeat" }, + { 0, "pass testNumber1 -repeat 1" }, + { 0, "pass testNumber1 -repeat 1 -o out.xml,xml" }, + { 0, "pass testNumber1 -repeat 2" }, + { 0, "pass testNumber1 -repeat 2 -o -,txt" }, + { 0, "pass testNumber1 -repeat 2 -o -,txt -o log.txt,txt" }, + { 1, "pass testNumber1 -repeat 2 -o log.xml,xml" }, + { 1, "pass testNumber1 -repeat 2 -o -,txt -o -,xml" }, + }; + + size_t n_testCases = sizeof(testCases) / sizeof(*testCases); + for (size_t i = 0; i < n_testCases; i++) { + GIVEN("The command line: " << testCases[i].cmdline) { + const QStringList cmdSplit = QString(testCases[i].cmdline) + .split(QRegularExpression(" +")); // at least 2 spaces + const QString test = cmdSplit[0]; + const QStringList args = cmdSplit.sliced(1); + auto runResult = runTestProcess(test, args); + REQUIRE(runResult.exitCode == testCases[i].expectedExitCode); + } + } +} + // ----------------------- Entrypoint ----------------------- int main(int argc, char **argv) |