diff options
Diffstat (limited to 'tests/auto/testlib/selftests/tst_selftests.cpp')
-rw-r--r-- | tests/auto/testlib/selftests/tst_selftests.cpp | 89 |
1 files changed, 59 insertions, 30 deletions
diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp index f6c97bc345..04185e95cd 100644 --- a/tests/auto/testlib/selftests/tst_selftests.cpp +++ b/tests/auto/testlib/selftests/tst_selftests.cpp @@ -1,6 +1,6 @@ // Copyright (C) 2021 The Qt Company Ltd. // Copyright (C) 2016 Intel Corporation. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only #include <QtCore/QCoreApplication> @@ -459,11 +459,13 @@ BenchmarkResult BenchmarkResult::parse(QString const& line, QString* error) // This code avoids using a QRegExp because QRegExp might be broken. // Sample format: 4,000 msec per iteration (total: 4,000, iterations: 1) - QString sFirstNumber; - while (!remaining.isEmpty() && !remaining.at(0).isSpace()) { - sFirstNumber += remaining.at(0); - remaining.remove(0,1); - } + const auto begin = remaining.cbegin(); + auto it = std::find_if(begin, remaining.cend(), [](const auto ch) { + return ch.isSpace(); + }); + QString sFirstNumber{std::distance(begin, it), Qt::Uninitialized}; + std::move(begin, it, sFirstNumber.begin()); + remaining.erase(begin, it); remaining = remaining.trimmed(); // 4,000 -> 4000 @@ -697,7 +699,8 @@ bool TestLogger::shouldIgnoreTest(const QString &test) const || test == "benchliboptions" || test == "printdatatags" || test == "printdatatagswithglobaltags" - || test == "silent") + || test == "silent" + || test == "silent_fatal") return true; // These tests produce variable output (callgrind because of #if-ery, @@ -762,29 +765,30 @@ void checkErrorOutput(const QString &test, const QByteArray &errorOutput) || test == "benchlibcallgrind") return; -#ifdef Q_CC_MINGW - if (test == "blacklisted" // calls qFatal() - || test == "silent") // calls qFatal() -#endif - return; - #ifdef Q_OS_WIN if (test == "crashes") return; // Complains about uncaught exception #endif -#ifdef Q_OS_LINUX - // QEMU outputs to stderr about uncaught signals - if (QTestPrivate::isRunningArmOnX86() && - (test == "assert" - || test == "blacklisted" - || test == "crashes" - || test == "faildatatype" - || test == "failfetchtype" - || test == "silent" - )) +#ifdef Q_OS_UNIX + if (test == "assert" + || test == "crashes" + || test == "failfetchtype" + || test == "faildatatype") + return; // Outputs "Received signal 6 (SIGABRT)" +#endif + + if (test == "silent_fatal") { +#if defined(__SANITIZE_ADDRESS__) || __has_feature(address_sanitizer) + // Under ASan, this test is not silent + return; +#elif defined(Q_CC_MINGW) + // Originally QTBUG-29014 (I can't reproduce this -Thiago) return; #endif + if (QTestPrivate::isRunningArmOnX86()) + return; // QEMU outputs to stderr about uncaught signals + } INFO(errorOutput.toStdString()); REQUIRE(errorOutput.isEmpty()); @@ -924,8 +928,11 @@ static QProcessEnvironment testEnvironment() if (environment.isEmpty()) { const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment(); const bool preserveLibPath = qEnvironmentVariableIsSet("QT_PRESERVE_TESTLIB_PATH"); - foreach (const QString &key, systemEnvironment.keys()) { + const auto envKeys = systemEnvironment.keys(); + for (const QString &key : envKeys) { const bool useVariable = key == "PATH" || key == "QT_QPA_PLATFORM" + || key == "QTEST_THROW_ON_FAIL"_L1 || key == "QTEST_THROW_ON_SKIP"_L1 + || key == "ASAN_OPTIONS" #if defined(Q_OS_QNX) || key == "GRAPHICS_ROOT" || key == "TZ" #elif defined(Q_OS_UNIX) @@ -970,8 +977,7 @@ TestProcessResult runTestProcess(const QString &test, const QStringList &argumen const bool expectedCrash = test == "assert" || test == "exceptionthrow" || test == "fetchbogus" || test == "crashedterminate" || test == "faildatatype" || test == "failfetchtype" - || test == "crashes" || test == "silent" - || test == "blacklisted" || test == "watchdog"; + || test == "crashes" || test == "silent_fatal" || test == "watchdog"; if (expectedCrash) { environment.insert("QTEST_DISABLE_CORE_DUMP", "1"); @@ -1008,10 +1014,12 @@ TestProcessResult runTestProcess(const QString &test, const QStringList &argumen return { process.exitCode(), standardOutput, standardError }; } +enum class Throw { OnFail = 1 }; + /* Runs a single test and verifies the output against the expected results. */ -void runTest(const QString &test, const TestLoggers &requestedLoggers) +void runTest(const QString &test, const TestLoggers &requestedLoggers, Throw throwing = {}) { TestLoggers loggers; for (auto logger : requestedLoggers) { @@ -1025,6 +1033,10 @@ void runTest(const QString &test, const TestLoggers &requestedLoggers) QStringList arguments; for (auto logger : loggers) arguments += logger.arguments(test); + if (throwing == Throw::OnFail) // don't distinguish between throwonfail/throwonskip + arguments += {"-throwonfail", "-throwonskip"}; + else + arguments += {"-nothrowonfail", "-nothrowonskip"}; CAPTURE(test); CAPTURE(arguments); @@ -1051,9 +1063,9 @@ void runTest(const QString &test, const TestLoggers &requestedLoggers) /* Runs a single test and verifies the output against the expected result. */ -void runTest(const QString &test, const TestLogger &logger) +void runTest(const QString &test, const TestLogger &logger, Throw t = {}) { - runTest(test, TestLoggers{logger}); + runTest(test, TestLoggers{logger}, t); } // ----------------------- Catch helpers ----------------------- @@ -1188,13 +1200,20 @@ TEST_CASE("All loggers can be enabled at the same time") SCENARIO("Test output of the loggers is as expected") { static QStringList tests = QString(QT_STRINGIFY(SUBPROGRAMS)).split(' '); + if (QString override = qEnvironmentVariable("TST_SELFTEST_SUBPROGRAMS"); !override.isEmpty()) + tests = override.split(' ', Qt::SkipEmptyParts); auto logger = GENERATE(filter(isGenericCommandLineLogger, enums<QTestLog::LogMode>())); GIVEN("The " << logger << " logger") { for (QString test : tests) { AND_GIVEN("The " << test << " subtest") { - runTest(test, TestLogger(logger, StdoutOutput)); + WHEN("Throwing on failure or skip") { + runTest(test, TestLogger(logger, StdoutOutput), Throw::OnFail); + } + WHEN("Returning on failure or skip") { + runTest(test, TestLogger(logger, StdoutOutput)); + } } } } @@ -1226,6 +1245,7 @@ SCENARIO("Exit code is as expected") { 0, "globaldata testGlobal:global=true" }, { 0, "globaldata testGlobal:local=true" }, { 0, "globaldata testGlobal:global=true:local=true" }, + { 0, "globaldata testGlobal -repeat 2" }, { 1, "globaldata testGlobal:local=true:global=true" }, { 1, "globaldata testGlobal:global=true:blah" }, { 1, "globaldata testGlobal:blah:local=true" }, @@ -1237,6 +1257,15 @@ SCENARIO("Exit code is as expected") { 1, "globaldata testGlobal:blah skipSingle:global=true:local=true" }, { 1, "globaldata testGlobal:global=true skipSingle:blah" }, { 2, "globaldata testGlobal:blah skipSingle:blue" }, + // Passing -repeat argument + { 1, "pass testNumber1 -repeat" }, + { 0, "pass testNumber1 -repeat 1" }, + { 0, "pass testNumber1 -repeat 1 -o out.xml,xml" }, + { 0, "pass testNumber1 -repeat 2" }, + { 0, "pass testNumber1 -repeat 2 -o -,txt" }, + { 0, "pass testNumber1 -repeat 2 -o -,txt -o log.txt,txt" }, + { 1, "pass testNumber1 -repeat 2 -o log.xml,xml" }, + { 1, "pass testNumber1 -repeat 2 -o -,txt -o -,xml" }, }; size_t n_testCases = sizeof(testCases) / sizeof(*testCases); |