diff options
Diffstat (limited to 'src/testlib/qtestcase.cpp')
| -rw-r--r-- | src/testlib/qtestcase.cpp | 1458 |
1 files changed, 578 insertions, 880 deletions
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index 89530f823f..42795fade7 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -3,13 +3,14 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <QtTest/qtestcase.h> +#include <QtTest/private/qtestcase_p.h> #include <QtTest/qtestassert.h> #include <QtCore/qbytearray.h> #include <QtCore/qcoreapplication.h> #include <QtCore/qdebug.h> #include <QtCore/qdir.h> -#include <QtCore/qdiriterator.h> +#include <QtCore/qdirlisting.h> #include <QtCore/qfile.h> #include <QtCore/qfileinfo.h> #include <QtCore/qfloat16.h> @@ -33,8 +34,12 @@ #include <QtTest/private/qtestresult_p.h> #include <QtTest/private/qsignaldumper_p.h> #include <QtTest/private/qbenchmark_p.h> +#if QT_CONFIG(batch_test_support) +#include <QtTest/private/qtestregistry_p.h> +#endif // QT_CONFIG(batch_test_support) #include <QtTest/private/cycle_p.h> #include <QtTest/private/qtestblacklist_p.h> +#include <QtTest/private/qtestcrashhandler_p.h> #if defined(HAVE_XCTEST) #include <QtTest/private/qxctestlogger_p.h> #endif @@ -60,6 +65,7 @@ #include <memory> #include <mutex> #include <numeric> +#include <optional> #include <stdarg.h> #include <stdio.h> @@ -111,6 +117,10 @@ #include <CoreFoundation/CFPreferences.h> #endif +#if defined(Q_OS_WASM) +#include <emscripten.h> +#endif + #include <vector> QT_BEGIN_NAMESPACE @@ -120,335 +130,229 @@ using namespace Qt::StringLiterals; using QtMiscUtils::toHexUpper; using QtMiscUtils::fromHex; -namespace { -enum DebuggerProgram { None, Gdb, Lldb }; - -#if defined(Q_OS_UNIX) && (!defined(Q_OS_WASM) || QT_CONFIG(thread)) -static struct iovec IoVec(struct iovec vec) +static bool installCoverageTool(const char * appname, const char * testname) { - return vec; +#if defined(__COVERAGESCANNER__) && !QT_CONFIG(testlib_selfcover) + if (!qEnvironmentVariableIsEmpty("QT_TESTCOCOON_ACTIVE")) + return false; + // Set environment variable QT_TESTCOCOON_ACTIVE to prevent an eventual subtest from + // being considered as a stand-alone test regarding the coverage analysis. + qputenv("QT_TESTCOCOON_ACTIVE", "1"); + + // Install Coverage Tool + __coveragescanner_install(appname); + __coveragescanner_testname(testname); + __coveragescanner_clear(); + return true; +#else + Q_UNUSED(appname); + Q_UNUSED(testname); + return false; +#endif } -static struct iovec IoVec(const char *str) + +static bool isValidSlot(const QMetaMethod &sl) { - struct iovec r = {}; - r.iov_base = const_cast<char *>(str); - r.iov_len = strlen(str); - return r; + if (sl.access() != QMetaMethod::Private || sl.parameterCount() != 0 + || sl.returnType() != QMetaType::Void || sl.methodType() != QMetaMethod::Slot) + return false; + const QByteArray name = sl.name(); + return !(name.isEmpty() || name.endsWith("_data") + || name == "initTestCase" || name == "cleanupTestCase" + || name == "init" || name == "cleanup"); } -template <typename... Args> static ssize_t writeToStderr(Args &&... args) +namespace QTestPrivate { - struct iovec vec[] = { IoVec(std::forward<Args>(args))... }; - return ::writev(STDERR_FILENO, vec, std::size(vec)); + Q_TESTLIB_EXPORT Qt::MouseButtons qtestMouseButtons = Qt::NoButton; } -// async-signal-safe conversion from int to string -struct AsyncSafeIntBuffer +namespace { + +class TestFailedException : public std::exception // clazy:exclude=copyable-polymorphic { - // digits10 + 1 for all possible digits - // +1 for the sign - // +1 for the terminating null - static constexpr int Digits10 = std::numeric_limits<int>::digits10 + 3; - std::array<char, Digits10> array; - constexpr AsyncSafeIntBuffer() : array{} {} // initializes array - AsyncSafeIntBuffer(Qt::Initialization) {} // leaves array uninitialized +public: + TestFailedException() = default; + ~TestFailedException() override = default; + + const char *what() const noexcept override { return "QtTest: test failed"; } }; -static struct iovec asyncSafeToString(int n, AsyncSafeIntBuffer &&result = Qt::Uninitialized) +class TestSkippedException : public std::exception // clazy:exclude=copyable-polymorphic { - char *ptr = result.array.data(); - if (false) { -#ifdef __cpp_lib_to_chars - } else if (auto r = std::to_chars(ptr, ptr + result.array.size(), n, 10); r.ec == std::errc{}) { - ptr = r.ptr; -#endif - } else { - // handle the sign - if (n < 0) { - *ptr++ = '-'; - n = -n; - } - - // find the highest power of the base that is less than this number - static constexpr int StartingDivider = ([]() { - int divider = 1; - for (int i = 0; i < std::numeric_limits<int>::digits10; ++i) - divider *= 10; - return divider; - }()); - int divider = StartingDivider; - while (divider && n < divider) - divider /= 10; - - // now convert to string - while (divider > 1) { - int quot = n / divider; - n = n % divider; - divider /= 10; - *ptr++ = quot + '0'; - } - *ptr++ = n + '0'; - } +public: + TestSkippedException() = default; + ~TestSkippedException() override = default; -#ifndef QT_NO_DEBUG - // this isn't necessary, it just helps in the debugger - *ptr = '\0'; -#endif - struct iovec r; - r.iov_base = result.array.data(); - r.iov_len = ptr - result.array.data(); - return r; + const char *what() const noexcept override { return "QtTest: test was skipped"; } }; -#elif defined(Q_OS_WIN) -// Windows doesn't need to be async-safe -template <typename... Args> static void writeToStderr(Args &&... args) -{ - (std::cerr << ... << args); -} -static std::string asyncSafeToString(int n) -{ - return std::to_string(n); -} -#endif // defined(Q_OS_UNIX) } // unnamed namespace -static bool alreadyDebugging() +namespace QTest { -#if defined(Q_OS_LINUX) - int fd = open("/proc/self/status", O_RDONLY); - if (fd == -1) - return false; - char buffer[2048]; - ssize_t size = read(fd, buffer, sizeof(buffer) - 1); - if (size == -1) { - close(fd); - return false; - } - buffer[size] = 0; - const char tracerPidToken[] = "\nTracerPid:"; - char *tracerPid = strstr(buffer, tracerPidToken); - if (!tracerPid) { - close(fd); - return false; - } - tracerPid += sizeof(tracerPidToken); - long int pid = strtol(tracerPid, &tracerPid, 10); - close(fd); - return pid != 0; -#elif defined(Q_OS_WIN) - return IsDebuggerPresent(); -#elif defined(Q_OS_MACOS) - // Check if there is an exception handler for the process: - mach_msg_type_number_t portCount = 0; - exception_mask_t masks[EXC_TYPES_COUNT]; - mach_port_t ports[EXC_TYPES_COUNT]; - exception_behavior_t behaviors[EXC_TYPES_COUNT]; - thread_state_flavor_t flavors[EXC_TYPES_COUNT]; - exception_mask_t mask = EXC_MASK_ALL & ~(EXC_MASK_RESOURCE | EXC_MASK_GUARD); - kern_return_t result = task_get_exception_ports(mach_task_self(), mask, masks, &portCount, - ports, behaviors, flavors); - if (result == KERN_SUCCESS) { - for (mach_msg_type_number_t portIndex = 0; portIndex < portCount; ++portIndex) { - if (MACH_PORT_VALID(ports[portIndex])) { - return true; - } - } - } - return false; -#else - // TODO - return false; -#endif -} -static bool hasSystemCrashReporter() +void Internal::throwOnFail() { throw TestFailedException(); } +void Internal::throwOnSkip() { throw TestSkippedException(); } + +Q_CONSTINIT static QBasicAtomicInt g_throwOnFail = Q_BASIC_ATOMIC_INITIALIZER(0); +Q_CONSTINIT static QBasicAtomicInt g_throwOnSkip = Q_BASIC_ATOMIC_INITIALIZER(0); + +void Internal::maybeThrowOnFail() { -#if defined(Q_OS_MACOS) - return QTestPrivate::macCrashReporterWillShowDialog(); -#else - return false; -#endif + if (g_throwOnFail.loadRelaxed() > 0) + Internal::throwOnFail(); } -static void maybeDisableCoreDump() +void Internal::maybeThrowOnSkip() { -#ifdef RLIMIT_CORE - bool ok = false; - const int disableCoreDump = qEnvironmentVariableIntValue("QTEST_DISABLE_CORE_DUMP", &ok); - if (ok && disableCoreDump) { - struct rlimit limit; - limit.rlim_cur = 0; - limit.rlim_max = 0; - if (setrlimit(RLIMIT_CORE, &limit) != 0) - qWarning("Failed to disable core dumps: %d", errno); - } -#endif + if (g_throwOnSkip.loadRelaxed() > 0) + Internal::throwOnSkip(); } -static DebuggerProgram debugger = None; -static void prepareStackTrace() -{ +/*! + \since 6.8 + \macro QTEST_THROW_ON_FAIL + \relates <QTest> - bool ok = false; - const int disableStackDump = qEnvironmentVariableIntValue("QTEST_DISABLE_STACK_DUMP", &ok); - if (ok && disableStackDump) - return; + When defined, QCOMPARE()/QVERIFY() etc always throw on failure. + QTest::throwOnFail() then no longer has any effect. +*/ - if (hasSystemCrashReporter()) - return; +/*! + \since 6.8 + \macro QTEST_THROW_ON_SKIP + \relates <QTest> -#if defined(Q_OS_MACOS) - #define CSR_ALLOW_UNRESTRICTED_FS (1 << 1) - std::optional<uint32_t> sipConfiguration = qt_mac_sipConfiguration(); - if (!sipConfiguration || !(*sipConfiguration & CSR_ALLOW_UNRESTRICTED_FS)) - return; // LLDB will fail to provide a valid stack trace -#endif + When defined, QSKIP() always throws. QTest::throwOnSkip() then no longer + has any effect. +*/ -#ifdef Q_OS_UNIX - // like QStandardPaths::findExecutable(), but simpler - auto hasExecutable = [](const char *execname) { - std::string candidate; - std::string path; - if (const char *p = getenv("PATH"); p && *p) - path = p; - else - path = _PATH_DEFPATH; - for (const char *p = std::strtok(&path[0], ":'"); p; p = std::strtok(nullptr, ":")) { - candidate = p; - candidate += '/'; - candidate += execname; - if (QT_ACCESS(candidate.data(), X_OK) == 0) - return true; - } - return false; - }; +/*! + \since 6.8 + \class QTest::ThrowOnFailEnabler + \inmodule QtTestLib - static constexpr DebuggerProgram debuggerSearchOrder[] = { -# if defined(Q_OS_QNX) || (defined(Q_OS_LINUX) && !defined(Q_OS_ANDROID)) - Gdb, Lldb -# else - Lldb, Gdb -# endif - }; - for (DebuggerProgram candidate : debuggerSearchOrder) { - switch (candidate) { - case None: - Q_UNREACHABLE(); - break; - case Gdb: - if (hasExecutable("gdb")) { - debugger = Gdb; - return; - } - break; - case Lldb: - if (hasExecutable("lldb")) { - debugger = Lldb; - return; - } - break; - } - } -#endif // Q_OS_UNIX -} + RAII class around setThrowOnFail(). +*/ +/*! + \fn QTest::ThrowOnFailEnabler::ThrowOnFailEnabler() -#if !defined(Q_OS_WASM) || QT_CONFIG(thread) -static void printTestRunTime() -{ - const int msecsFunctionTime = qRound(QTestLog::msecsFunctionTime()); - const int msecsTotalTime = qRound(QTestLog::msecsTotalTime()); - const char *const name = QTest::currentTestFunction(); - writeToStderr("\n ", name ? name : "[Non-test]", - " function time: ", asyncSafeToString(msecsFunctionTime), - "ms, total time: ", asyncSafeToString(msecsTotalTime), "ms\n"); -} + Constructor. Calls \c{setThrowOnFail(true)}. +*/ +/*! + \fn QTest::ThrowOnFailEnabler::~ThrowOnFailEnabler() -static void generateStackTrace() -{ - if (debugger == None || alreadyDebugging()) - return; + Destructor. Calls \c{setThrowOnFail(false)}. +*/ -# if defined(Q_OS_UNIX) && !defined(Q_OS_WASM) && !defined(Q_OS_INTEGRITY) - writeToStderr("\n=== Stack trace ===\n"); +/*! + \since 6.8 + \class QTest::ThrowOnFailDisabler + \inmodule QtTestLib - // execlp() requires null-termination, so call the default constructor - AsyncSafeIntBuffer pidbuffer; - asyncSafeToString(getpid(), std::move(pidbuffer)); + RAII class around setThrowOnFail(). +*/ +/*! + \fn QTest::ThrowOnFailDisabler::ThrowOnFailDisabler() - // Note: POSIX.1-2001 still has fork() in the list of async-safe functions, - // but in a future edition, it might be removed. It would be safer to wake - // up a babysitter thread to launch the debugger. - pid_t pid = fork(); - if (pid == 0) { - // child process - (void) dup2(STDERR_FILENO, STDOUT_FILENO); // redirect stdout to stderr + Constructor. Calls \c{setThrowOnFail(false)}. +*/ +/*! + \fn QTest::ThrowOnFailDisabler::~ThrowOnFailDisabler() - switch (debugger) { - case None: - Q_UNREACHABLE(); - break; - case Gdb: - execlp("gdb", "gdb", "--nx", "--batch", "-ex", "thread apply all bt", - "--pid", pidbuffer.array.data(), nullptr); - break; - case Lldb: - execlp("lldb", "lldb", "--no-lldbinit", "--batch", "-o", "bt all", - "--attach-pid", pidbuffer.array.data(), nullptr); - break; - } - _exit(1); - } else if (pid < 0) { - writeToStderr("Failed to start debugger.\n"); - } else { - int ret; - EINTR_LOOP(ret, waitpid(pid, nullptr, 0)); - } + Destructor. Calls \c{setThrowOnFail(true)}. +*/ - writeToStderr("=== End of stack trace ===\n"); -# endif // Q_OS_UNIX && !Q_OS_WASM -} -#endif // !defined(Q_OS_WASM) || QT_CONFIG(thread) +/*! + \since 6.8 + \class QTest::ThrowOnSkipEnabler + \inmodule QtTestLib -static bool installCoverageTool(const char * appname, const char * testname) -{ -#if defined(__COVERAGESCANNER__) && !QT_CONFIG(testlib_selfcover) - if (!qEnvironmentVariableIsEmpty("QT_TESTCOCOON_ACTIVE")) - return false; - // Set environment variable QT_TESTCOCOON_ACTIVE to prevent an eventual subtest from - // being considered as a stand-alone test regarding the coverage analysis. - qputenv("QT_TESTCOCOON_ACTIVE", "1"); + RAII class around setThrowOnSkip(). +*/ +/*! + \fn QTest::ThrowOnSkipEnabler::ThrowOnSkipEnabler() - // Install Coverage Tool - __coveragescanner_install(appname); - __coveragescanner_testname(testname); - __coveragescanner_clear(); - return true; -#else - Q_UNUSED(appname); - Q_UNUSED(testname); - return false; -#endif -} + Constructor. Calls \c{setThrowOnSkip(true)}. +*/ +/*! + \fn QTest::ThrowOnSkipEnabler::~ThrowOnSkipEnabler() -static bool isValidSlot(const QMetaMethod &sl) -{ - if (sl.access() != QMetaMethod::Private || sl.parameterCount() != 0 - || sl.returnType() != QMetaType::Void || sl.methodType() != QMetaMethod::Slot) - return false; - const QByteArray name = sl.name(); - return !(name.isEmpty() || name.endsWith("_data") - || name == "initTestCase" || name == "cleanupTestCase" - || name == "init" || name == "cleanup"); -} + Destructor. Calls \c{setThrowOnSkip(false)}. +*/ -namespace QTestPrivate +/*! + \since 6.8 + \class QTest::ThrowOnSkipDisabler + \inmodule QtTestLib + + RAII class around setThrowOnSkip(). +*/ +/*! + \fn QTest::ThrowOnSkipDisabler::ThrowOnSkipDisabler() + + Constructor. Calls \c{setThrowOnSkip(false)}. +*/ +/*! + \fn QTest::ThrowOnSkipDisabler::~ThrowOnSkipDisabler() + + Destructor. Calls \c{setThrowOnSkip(true)}. +*/ + +/*! + \since 6.8 + + Enables (\a enable = \c true) or disables (\ enable = \c false) throwing on + QCOMPARE()/QVERIFY() failures (as opposed to just returning from the + immediately-surrounding function context). + + The feature is reference-counted: If you call this function \e{N} times + with \c{true}, you need to call it \e{N} times with \c{false} to get back + to where you started. + + The default is \c{false}, unless the \l{Qt Test Environment Variables} + {QTEST_THROW_ON_FAIL environment variable} is set. + + This call has no effect when the \l{QTEST_THROW_ON_FAIL} C++ macro is + defined. + + \note You must compile your tests with exceptions enabled to use this + feature. + + \sa setThrowOnSkip(), ThrowOnFailEnabler, ThrowOnFailDisabler, QTEST_THROW_ON_FAIL +*/ +void setThrowOnFail(bool enable) noexcept { - Q_TESTLIB_EXPORT Qt::MouseButtons qtestMouseButtons = Qt::NoButton; + g_throwOnFail.fetchAndAddRelaxed(enable ? 1 : -1); } -namespace QTest +/*! + \since 6.8 + + Enables (\a enable = \c true) or disables (\ enable = \c false) throwing on + QSKIP() (as opposed to just returning from the immediately-surrounding + function context). + + The feature is reference-counted: If you call this function \e{N} times + with \c{true}, you need to call it \e{N} times with \c{false} to get back + to where you started. + + The default is \c{false}, unless the \l{Qt Test Environment Variables} + {QTEST_THROW_ON_SKIP environment variable} is set. + + This call has no effect when the \l{QTEST_THROW_ON_SKIP} C++ macro is + defined. + + \note You must compile your tests with exceptions enabled to use this + feature. + + \sa setThrowOnFail(), ThrowOnSkipEnabler, ThrowOnSkipDisabler, QTEST_THROW_ON_SKIP +*/ +void setThrowOnSkip(bool enable) noexcept { + g_throwOnSkip.fetchAndAddRelaxed(enable ? 1 : -1); +} QString Internal::formatTryTimeoutDebugMessage(q_no_char8_t::QUtf8StringView expr, int timeout, int actual) { @@ -464,6 +368,7 @@ class WatchDog; static QObject *currentTestObject = nullptr; static QString mainSourcePath; +static bool inTestFunction = false; #if defined(Q_OS_MACOS) static IOPMAssertionID macPowerSavingDisabled = 0; @@ -482,7 +387,7 @@ public: static QMetaMethod findMethod(const QObject *obj, const char *signature); private: - bool invokeTest(int index, QLatin1StringView tag, WatchDog *watchDog) const; + bool invokeTest(int index, QLatin1StringView tag, std::optional<WatchDog> &watchDog) const; void invokeTestOnData(int index) const; QMetaMethod m_initTestCaseMethod; // might not exist, check isValid(). @@ -527,19 +432,32 @@ static int eventDelay = -1; #if QT_CONFIG(thread) static int timeout = -1; #endif -static bool noCrashHandler = false; +static int repetitions = 1; +static bool repeatForever = false; +static bool skipBlacklisted = false; -/*! \internal - Invoke a method of the object without generating warning if the method does not exist -*/ -static void invokeMethod(QObject *obj, const char *methodName) +namespace Internal { +bool noCrashHandler = false; +} + +static bool invokeTestMethodIfValid(QMetaMethod m, QObject *obj = QTest::currentTestObject) +{ + if (!m.isValid()) + return false; + bool ok = true; + try { ok = m.invoke(obj, Qt ::DirectConnection); } + catch (const TestFailedException &) {} // ignore (used for control flow) + catch (const TestSkippedException &) {} // ditto + // every other exception is someone else's problem + return ok; +} + +static void invokeTestMethodIfExists(const char *methodName, QObject *obj = QTest::currentTestObject) { const QMetaObject *metaObject = obj->metaObject(); int funcIndex = metaObject->indexOfMethod(methodName); - if (funcIndex >= 0) { - QMetaMethod method = metaObject->method(funcIndex); - method.invoke(obj, Qt::DirectConnection); - } + // doesn't generate a warning if it doesn't exist: + invokeTestMethodIfValid(metaObject->method(funcIndex), obj); } int defaultEventDelay() @@ -623,7 +541,7 @@ static void qPrintDataTags(FILE *stream) // Get global data tags: QTestTable::globalTestTable(); - invokeMethod(QTest::currentTestObject, "initTestCase_data()"); + invokeTestMethodIfExists("initTestCase_data()"); const QTestTable *gTable = QTestTable::globalTestTable(); const QMetaObject *currTestMetaObj = QTest::currentTestObject->metaObject(); @@ -642,7 +560,7 @@ static void qPrintDataTags(FILE *stream) QByteArray member; member.resize(qstrlen(slot) + qstrlen("_data()") + 1); qsnprintf(member.data(), member.size(), "%s_data()", slot); - invokeMethod(QTest::currentTestObject, member.constData()); + invokeTestMethodIfExists(member.constData()); const int dataCount = table.dataCount(); localTags.reserve(dataCount); for (int j = 0; j < dataCount; ++j) @@ -650,7 +568,7 @@ static void qPrintDataTags(FILE *stream) // Print all tag combinations: if (gTable->dataCount() == 0) { - if (localTags.count() == 0) { + if (localTags.size() == 0) { // No tags at all, so just print the test function: fprintf(stream, "%s %s\n", currTestMetaObj->className(), slot); } else { @@ -662,7 +580,7 @@ static void qPrintDataTags(FILE *stream) } } else { for (int j = 0; j < gTable->dataCount(); ++j) { - if (localTags.count() == 0) { + if (localTags.size() == 0) { // Only global tags, so print the current one: fprintf( stream, "%s %s __global__ %s\n", @@ -699,10 +617,18 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool int logFormat = -1; // Not set const char *logFilename = nullptr; + repetitions = 1; + repeatForever = false; + QTest::testFunctions.clear(); QTest::testTags.clear(); -#if defined(Q_OS_MAC) && defined(HAVE_XCTEST) + if (qEnvironmentVariableIsSet("QTEST_THROW_ON_FAIL")) + QTest::setThrowOnFail(true); + if (qEnvironmentVariableIsSet("QTEST_THROW_ON_SKIP")) + QTest::setThrowOnSkip(true); + +#if defined(Q_OS_DARWIN) && defined(HAVE_XCTEST) if (QXcodeTestLogger::canLogTestProgress()) logFormat = QTestLog::XCTest; #endif @@ -753,6 +679,15 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool " -maxwarnings n : Sets the maximum amount of messages to output.\n" " 0 means unlimited, default: 2000\n" " -nocrashhandler : Disables the crash handler. Useful for debugging crashes.\n" + " -repeat n : Run the testsuite n times or until the test fails.\n" + " Useful for finding flaky tests. If negative, the tests are\n" + " repeated forever. This is intended as a developer tool, and\n" + " is only supported with the plain text logger.\n" + " -skipblacklisted : Skip blacklisted tests. Useful for measuring test coverage.\n" + " -[no]throwonfail : Enables/disables throwing on QCOMPARE()/QVERIFY()/etc.\n" + " Default: off, unless QTEST_THROW_ON_FAIL is set." + " -[no]throwonskip : Enables/disables throwing on QSKIP().\n" + " Default: off, unless QTEST_THROW_ON_SKIP is set." "\n" " Benchmarking options:\n" #if QT_CONFIG(valgrind) @@ -902,8 +837,26 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool } else { QTestLog::setMaxWarnings(qToInt(argv[++i])); } + } else if (strcmp(argv[i], "-repeat") == 0) { + if (i + 1 >= argc) { + fprintf(stderr, "-repeat needs an extra parameter for the number of repetitions\n"); + exit(1); + } else { + repetitions = qToInt(argv[++i]); + repeatForever = repetitions < 0; + } } else if (strcmp(argv[i], "-nocrashhandler") == 0) { - QTest::noCrashHandler = true; + QTest::Internal::noCrashHandler = true; + } else if (strcmp(argv[i], "-skipblacklisted") == 0) { + QTest::skipBlacklisted = true; + } else if (strcmp(argv[i], "-throwonfail") == 0) { + QTest::setThrowOnFail(true); + } else if (strcmp(argv[i], "-nothrowonfail") == 0) { + QTest::setThrowOnFail(false); + } else if (strcmp(argv[i], "-throwonskip") == 0) { + QTest::setThrowOnSkip(true); + } else if (strcmp(argv[i], "-nothrowonskip") == 0) { + QTest::setThrowOnSkip(false); #if QT_CONFIG(valgrind) } else if (strcmp(argv[i], "-callgrind") == 0) { if (!QBenchmarkValgrindUtils::haveValgrind()) { @@ -1045,16 +998,21 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool #if defined(QT_USE_APPLE_UNIFIED_LOGGING) // Any explicitly requested loggers will be added by now, so we can check if they use stdout - const bool safeToAddAppleLogger = !AppleUnifiedLogger::willMirrorToStderr() || !QTestLog::loggerUsingStdout(); + const bool safeToAddAppleLogger = !AppleUnifiedLogger::preventsStderrLogging() || !QTestLog::loggerUsingStdout(); if (safeToAddAppleLogger && QAppleTestLogger::debugLoggingEnabled()) { QTestLog::addLogger(QTestLog::Apple, nullptr); - if (AppleUnifiedLogger::willMirrorToStderr() && !logFilename) + if (AppleUnifiedLogger::preventsStderrLogging() && !logFilename) addFallbackLogger = false; // Prevent plain test logger fallback below } #endif if (addFallbackLogger) QTestLog::addLogger(QTestLog::Plain, logFilename); + + if (repetitions != 1 && !QTestLog::isRepeatSupported()) { + fprintf(stderr, "-repeat is only supported with plain text logger\n"); + exit(1); + } } // Temporary, backwards compatibility, until qtdeclarative's use of it is converted @@ -1062,17 +1020,20 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) { qtest_qParseArgs(argc, const_cast<const char *const *>(argv), qml); } -QBenchmarkResult qMedian(const QList<QBenchmarkResult> &container) +static QList<QBenchmarkResult> qMedian(const QList<QList<QBenchmarkResult>> &container) { - const int count = container.count(); + const int count = container.size(); if (count == 0) - return QBenchmarkResult(); + return {}; if (count == 1) return container.front(); - QList<QBenchmarkResult> containerCopy = container; - std::sort(containerCopy.begin(), containerCopy.end()); + QList<QList<QBenchmarkResult>> containerCopy = container; + std::sort(containerCopy.begin(), containerCopy.end(), + [](const QList<QBenchmarkResult> &a, const QList<QBenchmarkResult> &b) { + return a.first() < b.first(); + }); const int middle = count / 2; @@ -1092,15 +1053,6 @@ struct QTestDataSetter } }; -namespace { - -qreal addResult(qreal current, const QBenchmarkResult& r) -{ - return current + r.value; -} - -} - void TestMethods::invokeTestOnData(int index) const { /* Benchmarking: for each median iteration*/ @@ -1108,27 +1060,30 @@ void TestMethods::invokeTestOnData(int index) const bool isBenchmark = false; int i = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0; - QList<QBenchmarkResult> results; + QList<QList<QBenchmarkResult>> resultsList; bool minimumTotalReached = false; do { QBenchmarkTestMethodData::current->beginDataRun(); + if (i < 0) + QBenchmarkTestMethodData::current->iterationCount = 1; /* Benchmarking: for each accumulation iteration*/ bool invokeOk; do { - if (m_initMethod.isValid()) - m_initMethod.invoke(QTest::currentTestObject, Qt::DirectConnection); + QTest::inTestFunction = true; + invokeTestMethodIfValid(m_initMethod); const bool initQuit = QTestResult::skipCurrentTest() || QTestResult::currentTestFailed(); if (!initQuit) { - QBenchmarkTestMethodData::current->result = QBenchmarkResult(); + QBenchmarkTestMethodData::current->results.clear(); QBenchmarkTestMethodData::current->resultAccepted = false; + QBenchmarkTestMethodData::current->valid = false; QBenchmarkGlobalData::current->context.tag = QLatin1StringView( QTestResult::currentDataTag() ? QTestResult::currentDataTag() : ""); - invokeOk = m_methods[index].invoke(QTest::currentTestObject, Qt::DirectConnection); + invokeOk = invokeTestMethodIfValid(m_methods[index]); if (!invokeOk) QTestResult::addFailure("Unable to execute slot", __FILE__, __LINE__); @@ -1137,11 +1092,11 @@ void TestMethods::invokeTestOnData(int index) const invokeOk = false; } + QTest::inTestFunction = false; QTestResult::finishedCurrentTestData(); if (!initQuit) { - if (m_cleanupMethod.isValid()) - m_cleanupMethod.invoke(QTest::currentTestObject, Qt::DirectConnection); + invokeTestMethodIfValid(m_cleanupMethod); // Process any deleteLater(), used by event-loop-based apps. // Fixes memleak reports. @@ -1164,26 +1119,29 @@ void TestMethods::invokeTestOnData(int index) const QBenchmarkTestMethodData::current->endDataRun(); if (!QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed()) { if (i > -1) // iteration -1 is the warmup iteration. - results.append(QBenchmarkTestMethodData::current->result); - - if (isBenchmark && QBenchmarkGlobalData::current->verboseOutput) { - if (i == -1) { - QTestLog::info(qPrintable( - QString::fromLatin1("warmup stage result : %1") - .arg(QBenchmarkTestMethodData::current->result.value)), nullptr, 0); - } else { - QTestLog::info(qPrintable( - QString::fromLatin1("accumulation stage result: %1") - .arg(QBenchmarkTestMethodData::current->result.value)), nullptr, 0); - } + resultsList.append(QBenchmarkTestMethodData::current->results); + + if (isBenchmark && QBenchmarkGlobalData::current->verboseOutput && + !QBenchmarkTestMethodData::current->results.isEmpty()) { + // we only print the first result + const QBenchmarkResult &first = QBenchmarkTestMethodData::current->results.constFirst(); + QString pattern = i < 0 ? "warmup stage result : %1"_L1 + : "accumulation stage result: %1"_L1; + QTestLog::info(qPrintable(pattern.arg(first.measurement.value)), nullptr, 0); } } - // Verify if the minimum total measurement is reached, if it was specified: + // Verify if the minimum total measurement (for the first measurement) + // was reached, if it was specified: if (QBenchmarkGlobalData::current->minimumTotal == -1) { minimumTotalReached = true; } else { - const qreal total = std::accumulate(results.begin(), results.end(), 0.0, addResult); + auto addResult = [](qreal current, const QList<QBenchmarkResult> &r) { + if (!r.isEmpty()) + current += r.first().measurement.value; + return current; + }; + const qreal total = std::accumulate(resultsList.begin(), resultsList.end(), 0.0, addResult); minimumTotalReached = (total >= QBenchmarkGlobalData::current->minimumTotal); } } while (isBenchmark @@ -1196,7 +1154,7 @@ void TestMethods::invokeTestOnData(int index) const QTestResult::finishedCurrentTestDataCleanup(); // Only report benchmark figures if the test passed if (testPassed && QBenchmarkTestMethodData::current->resultsAccepted()) - QTestLog::addBenchmarkResult(qMedian(results)); + QTestLog::addBenchmarkResults(qMedian(resultsList)); } } @@ -1204,17 +1162,30 @@ void TestMethods::invokeTestOnData(int index) const class WatchDog : public QThread { - enum Expectation { + enum Expectation : std::size_t { + // bits 0..1: state ThreadStart, TestFunctionStart, TestFunctionEnd, ThreadEnd, - }; - bool waitFor(std::unique_lock<QtPrivate::mutex> &m, Expectation e) + // bits 2..: generation + }; + static constexpr auto ExpectationMask = Expectation{ThreadStart | TestFunctionStart | TestFunctionEnd | ThreadEnd}; + static_assert(size_t(ExpectationMask) == 0x3); + static constexpr size_t GenerationShift = 2; + + static constexpr Expectation state(Expectation e) noexcept + { return Expectation{e & ExpectationMask}; } + static constexpr size_t generation(Expectation e) noexcept + { return e >> GenerationShift; } + static constexpr Expectation combine(Expectation e, size_t gen) noexcept + { return Expectation{e | (gen << GenerationShift)}; } + + bool waitFor(std::unique_lock<std::mutex> &m, Expectation e) { auto expectationChanged = [this, e] { return expecting.load(std::memory_order_relaxed) != e; }; - switch (e) { + switch (state(e)) { case TestFunctionEnd: return waitCondition.wait_for(m, defaultTimeout(), expectationChanged); case ThreadStart: @@ -1223,8 +1194,20 @@ class WatchDog : public QThread waitCondition.wait(m, expectationChanged); return true; } - Q_UNREACHABLE(); - return false; + Q_UNREACHABLE_RETURN(false); + } + + void setExpectation(Expectation e) + { + Q_ASSERT(generation(e) == 0); // no embedded generation allowed + const auto locker = qt_scoped_lock(mutex); + auto cur = expecting.load(std::memory_order_relaxed); + auto gen = generation(cur); + if (e == TestFunctionStart) + ++gen; + e = combine(e, gen); + expecting.store(e, std::memory_order_relaxed); + waitCondition.notify_all(); } public: @@ -1239,36 +1222,29 @@ public: ~WatchDog() { - { - const auto locker = qt_scoped_lock(mutex); - expecting.store(ThreadEnd, std::memory_order_relaxed); - waitCondition.notify_all(); - } + setExpectation(ThreadEnd); wait(); } void beginTest() { - const auto locker = qt_scoped_lock(mutex); - expecting.store(TestFunctionEnd, std::memory_order_relaxed); - waitCondition.notify_all(); + setExpectation(TestFunctionEnd); } void testFinished() { - const auto locker = qt_scoped_lock(mutex); - expecting.store(TestFunctionStart, std::memory_order_relaxed); - waitCondition.notify_all(); + setExpectation(TestFunctionStart); } void run() override { + CrashHandler::blockUnixSignals(); auto locker = qt_unique_lock(mutex); expecting.store(TestFunctionStart, std::memory_order_release); waitCondition.notify_all(); while (true) { Expectation e = expecting.load(std::memory_order_acquire); - switch (e) { + switch (state(e)) { case ThreadEnd: return; case ThreadStart: @@ -1277,8 +1253,8 @@ public: case TestFunctionEnd: if (Q_UNLIKELY(!waitFor(locker, e))) { fflush(stderr); - printTestRunTime(); - generateStackTrace(); + CrashHandler::printTestRunTime(); + CrashHandler::generateStackTrace(); qFatal("Test function timed out"); } } @@ -1286,8 +1262,8 @@ public: } private: - QtPrivate::mutex mutex; - QtPrivate::condition_variable waitCondition; + std::mutex mutex; + std::condition_variable waitCondition; std::atomic<Expectation> expecting; }; @@ -1332,7 +1308,7 @@ static void printUnknownDataTagError(QLatin1StringView name, QLatin1StringView t If the function was successfully called, true is returned, otherwise false. */ -bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDog) const +bool TestMethods::invokeTest(int index, QLatin1StringView tag, std::optional<WatchDog> &watchDog) const { QBenchmarkTestMethodData benchmarkData; QBenchmarkTestMethodData::current = &benchmarkData; @@ -1364,6 +1340,7 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo tag[global.size()] == ':'; }; bool foundFunction = false; + bool blacklisted = false; /* For each entry in the global data table, do: */ do { @@ -1372,7 +1349,7 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo if (curGlobalDataIndex == 0) { qsnprintf(member, 512, "%s_data()", name.constData()); - invokeMethod(QTest::currentTestObject, member); + invokeTestMethodIfExists(member); if (QTestResult::skipCurrentTest()) break; } @@ -1390,18 +1367,28 @@ bool TestMethods::invokeTest(int index, QLatin1StringView tag, WatchDog *watchDo if (dataTagMatches(tag, QLatin1StringView(dataTag(curDataIndex)), QLatin1StringView(globalDataTag(curGlobalDataIndex)))) { foundFunction = true; - QTestPrivate::checkBlackLists(name.constData(), dataTag(curDataIndex), - globalDataTag(curGlobalDataIndex)); - - QTestDataSetter s(curDataIndex >= dataCount ? nullptr : table.testData(curDataIndex)); - - QTestPrivate::qtestMouseButtons = Qt::NoButton; - if (watchDog) - watchDog->beginTest(); - QTest::lastMouseTimestamp += 500; // Maintain at least 500ms mouse event timestamps between each test function call - invokeTestOnData(index); - if (watchDog) - watchDog->testFinished(); + blacklisted = QTestPrivate::checkBlackLists(name.constData(), dataTag(curDataIndex), + globalDataTag(curGlobalDataIndex)); + if (blacklisted) + QTestResult::setBlacklistCurrentTest(true); + + if (blacklisted && skipBlacklisted) { + QTest::qSkip("Skipping blacklisted test since -skipblacklisted option is set.", + NULL, 0); + QTestResult::finishedCurrentTestData(); + QTestResult::finishedCurrentTestDataCleanup(); + } else { + QTestDataSetter s( + curDataIndex >= dataCount ? nullptr : table.testData(curDataIndex)); + + QTestPrivate::qtestMouseButtons = Qt::NoButton; + if (watchDog) + watchDog->beginTest(); + QTest::lastMouseTimestamp += 500; // Maintain at least 500ms mouse event timestamps between each test function call + invokeTestOnData(index); + if (watchDog) + watchDog->testFinished(); + } if (!tag.isEmpty() && !globalDataCount) break; @@ -1620,65 +1607,77 @@ char *toPrettyCString(const char *p, qsizetype length) } /*! + \fn char *toPrettyUnicode(QStringView string) \internal Returns the same QString but with only the ASCII characters still shown; everything else is replaced with \c {\uXXXX}. Similar to QDebug::putString(). */ + +constexpr qsizetype PrettyUnicodeMaxOutputSize = 256; +// escape sequence, closing quote, the three dots and NUL +constexpr qsizetype PrettyUnicodeMaxIncrement = sizeof(R"(\uXXXX"...)"); // includes NUL + +static char *writePrettyUnicodeChar(char16_t ch, char * const buffer) +{ + auto dst = buffer; + auto first = [&](int n) { Q_ASSERT(dst - buffer == n); return dst; }; + if (ch < 0x7f && ch >= 0x20 && ch != '\\' && ch != '"') { + *dst++ = ch; + return first(1); + } + + // write as an escape sequence + *dst++ = '\\'; + switch (ch) { + case 0x22: + case 0x5c: + *dst++ = uchar(ch); + break; + case 0x8: + *dst++ = 'b'; + break; + case 0xc: + *dst++ = 'f'; + break; + case 0xa: + *dst++ = 'n'; + break; + case 0xd: + *dst++ = 'r'; + break; + case 0x9: + *dst++ = 't'; + break; + default: + *dst++ = 'u'; + *dst++ = toHexUpper(ch >> 12); + *dst++ = toHexUpper(ch >> 8); + *dst++ = toHexUpper(ch >> 4); + *dst++ = toHexUpper(ch); + return first(6); + } + return first(2); +} + char *toPrettyUnicode(QStringView string) { auto p = string.utf16(); auto length = string.size(); // keep it simple for the vast majority of cases bool trimmed = false; - auto buffer = std::make_unique<char[]>(256); + auto buffer = std::make_unique<char[]>(PrettyUnicodeMaxOutputSize); const auto end = p + length; char *dst = buffer.get(); *dst++ = '"'; for ( ; p != end; ++p) { - if (dst - buffer.get() > 245) { - // plus the quote, the three dots and NUL, it's 250, 251 or 255 + if (dst - buffer.get() > PrettyUnicodeMaxOutputSize - PrettyUnicodeMaxIncrement) { trimmed = true; break; } - - if (*p < 0x7f && *p >= 0x20 && *p != '\\' && *p != '"') { - *dst++ = *p; - continue; - } - - // write as an escape sequence - // this means we may advance dst to buffer.data() + 246 or 250 - *dst++ = '\\'; - switch (*p) { - case 0x22: - case 0x5c: - *dst++ = uchar(*p); - break; - case 0x8: - *dst++ = 'b'; - break; - case 0xc: - *dst++ = 'f'; - break; - case 0xa: - *dst++ = 'n'; - break; - case 0xd: - *dst++ = 'r'; - break; - case 0x9: - *dst++ = 't'; - break; - default: - *dst++ = 'u'; - *dst++ = toHexUpper(*p >> 12); - *dst++ = toHexUpper(*p >> 8); - *dst++ = toHexUpper(*p >> 4); - *dst++ = toHexUpper(*p); - } + dst = writePrettyUnicodeChar(*p, dst); } *dst++ = '"'; @@ -1696,23 +1695,21 @@ void TestMethods::invokeTests(QObject *testObject) const const QMetaObject *metaObject = testObject->metaObject(); QTEST_ASSERT(metaObject); QTestResult::setCurrentTestFunction("initTestCase"); - if (m_initTestCaseDataMethod.isValid()) - m_initTestCaseDataMethod.invoke(testObject, Qt::DirectConnection); + invokeTestMethodIfValid(m_initTestCaseDataMethod, testObject); - QScopedPointer<WatchDog> watchDog; - if (!alreadyDebugging() + std::optional<WatchDog> watchDog = std::nullopt; + if (!CrashHandler::alreadyDebugging() #if QT_CONFIG(valgrind) && QBenchmarkGlobalData::current->mode() != QBenchmarkGlobalData::CallgrindChildProcess #endif ) { - watchDog.reset(new WatchDog); + watchDog.emplace(); } QSignalDumper::startDump(); if (!QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed()) { - if (m_initTestCaseMethod.isValid()) - m_initTestCaseMethod.invoke(testObject, Qt::DirectConnection); + invokeTestMethodIfValid(m_initTestCaseMethod, testObject); // finishedCurrentTestDataCleanup() resets QTestResult::currentTestFailed(), so use a local copy. const bool previousFailed = QTestResult::currentTestFailed(); @@ -1725,7 +1722,7 @@ void TestMethods::invokeTests(QObject *testObject) const const char *data = nullptr; if (i < QTest::testTags.size() && !QTest::testTags.at(i).isEmpty()) data = qstrdup(QTest::testTags.at(i).toLatin1().constData()); - const bool ok = invokeTest(i, QLatin1StringView(data), watchDog.data()); + const bool ok = invokeTest(i, QLatin1StringView(data), watchDog); delete [] data; if (!ok) break; @@ -1736,8 +1733,7 @@ void TestMethods::invokeTests(QObject *testObject) const QTestResult::setSkipCurrentTest(false); QTestResult::setBlacklistCurrentTest(false); QTestResult::setCurrentTestFunction("cleanupTestCase"); - if (m_cleanupTestCaseMethod.isValid()) - m_cleanupTestCaseMethod.invoke(testObject, Qt::DirectConnection); + invokeTestMethodIfValid(m_cleanupTestCaseMethod, testObject); QTestResult::finishedCurrentTestData(); // Restore skip state as it affects decision on whether we passed: QTestResult::setSkipCurrentTest(wasSkipped || QTestResult::skipCurrentTest()); @@ -1759,421 +1755,6 @@ bool reportResult(bool success, qxp::function_ref<const char *()> lhs, } // namespace QTest -namespace { -#if defined(Q_OS_WIN) - -// Helper class for resolving symbol names by dynamically loading "dbghelp.dll". -class DebugSymbolResolver -{ - Q_DISABLE_COPY_MOVE(DebugSymbolResolver) -public: - struct Symbol { - Symbol() : name(nullptr), address(0) {} - - const char *name; // Must be freed by caller. - DWORD64 address; - }; - - explicit DebugSymbolResolver(HANDLE process); - ~DebugSymbolResolver() { cleanup(); } - - bool isValid() const { return m_symFromAddr; } - - Symbol resolveSymbol(DWORD64 address) const; - -private: - // typedefs from DbgHelp.h/.dll - struct DBGHELP_SYMBOL_INFO { // SYMBOL_INFO - ULONG SizeOfStruct; - ULONG TypeIndex; // Type Index of symbol - ULONG64 Reserved[2]; - ULONG Index; - ULONG Size; - ULONG64 ModBase; // Base Address of module comtaining this symbol - ULONG Flags; - ULONG64 Value; // Value of symbol, ValuePresent should be 1 - ULONG64 Address; // Address of symbol including base address of module - ULONG Register; // register holding value or pointer to value - ULONG Scope; // scope of the symbol - ULONG Tag; // pdb classification - ULONG NameLen; // Actual length of name - ULONG MaxNameLen; - CHAR Name[1]; // Name of symbol - }; - - typedef BOOL (__stdcall *SymInitializeType)(HANDLE, PCSTR, BOOL); - typedef BOOL (__stdcall *SymFromAddrType)(HANDLE, DWORD64, PDWORD64, DBGHELP_SYMBOL_INFO *); - - void cleanup(); - - const HANDLE m_process; - HMODULE m_dbgHelpLib; - SymFromAddrType m_symFromAddr; -}; - -void DebugSymbolResolver::cleanup() -{ - if (m_dbgHelpLib) - FreeLibrary(m_dbgHelpLib); - m_dbgHelpLib = 0; - m_symFromAddr = nullptr; -} - -DebugSymbolResolver::DebugSymbolResolver(HANDLE process) - : m_process(process), m_dbgHelpLib(0), m_symFromAddr(nullptr) -{ - bool success = false; - m_dbgHelpLib = LoadLibraryW(L"dbghelp.dll"); - if (m_dbgHelpLib) { - SymInitializeType symInitialize = reinterpret_cast<SymInitializeType>( - reinterpret_cast<QFunctionPointer>(GetProcAddress(m_dbgHelpLib, "SymInitialize"))); - m_symFromAddr = reinterpret_cast<SymFromAddrType>( - reinterpret_cast<QFunctionPointer>(GetProcAddress(m_dbgHelpLib, "SymFromAddr"))); - success = symInitialize && m_symFromAddr && symInitialize(process, NULL, TRUE); - } - if (!success) - cleanup(); -} - -DebugSymbolResolver::Symbol DebugSymbolResolver::resolveSymbol(DWORD64 address) const -{ - // reserve additional buffer where SymFromAddr() will store the name - struct NamedSymbolInfo : public DBGHELP_SYMBOL_INFO { - enum { symbolNameLength = 255 }; - - char name[symbolNameLength + 1]; - }; - - Symbol result; - if (!isValid()) - return result; - NamedSymbolInfo symbolBuffer; - memset(&symbolBuffer, 0, sizeof(NamedSymbolInfo)); - symbolBuffer.MaxNameLen = NamedSymbolInfo::symbolNameLength; - symbolBuffer.SizeOfStruct = sizeof(DBGHELP_SYMBOL_INFO); - if (!m_symFromAddr(m_process, address, 0, &symbolBuffer)) - return result; - result.name = qstrdup(symbolBuffer.Name); - result.address = symbolBuffer.Address; - return result; -} - -class WindowsFaultHandler -{ -public: - WindowsFaultHandler() - { -# if !defined(Q_CC_MINGW) - _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_DEBUG); -# endif - SetErrorMode(SetErrorMode(0) | SEM_NOGPFAULTERRORBOX); - SetUnhandledExceptionFilter(windowsFaultHandler); - } - -private: - static LONG WINAPI windowsFaultHandler(struct _EXCEPTION_POINTERS *exInfo) - { - enum { maxStackFrames = 100 }; - char appName[MAX_PATH]; - if (!GetModuleFileNameA(NULL, appName, MAX_PATH)) - appName[0] = 0; - const int msecsFunctionTime = qRound(QTestLog::msecsFunctionTime()); - const int msecsTotalTime = qRound(QTestLog::msecsTotalTime()); - const void *exceptionAddress = exInfo->ExceptionRecord->ExceptionAddress; - fprintf(stderr, "A crash occurred in %s.\n", appName); - if (const char *name = QTest::currentTestFunction()) - fprintf(stderr, "While testing %s\n", name); - fprintf(stderr, "Function time: %dms Total time: %dms\n\n" - "Exception address: 0x%p\n" - "Exception code : 0x%lx\n", - msecsFunctionTime, msecsTotalTime, exceptionAddress, - exInfo->ExceptionRecord->ExceptionCode); - - DebugSymbolResolver resolver(GetCurrentProcess()); - if (resolver.isValid()) { - DebugSymbolResolver::Symbol exceptionSymbol = resolver.resolveSymbol(DWORD64(exceptionAddress)); - if (exceptionSymbol.name) { - fprintf(stderr, "Nearby symbol : %s\n", exceptionSymbol.name); - delete [] exceptionSymbol.name; - } - void *stack[maxStackFrames]; - fputs("\nStack:\n", stderr); - const unsigned frameCount = CaptureStackBackTrace(0, DWORD(maxStackFrames), stack, NULL); - for (unsigned f = 0; f < frameCount; ++f) { - DebugSymbolResolver::Symbol symbol = resolver.resolveSymbol(DWORD64(stack[f])); - if (symbol.name) { - fprintf(stderr, "#%3u: %s() - 0x%p\n", f + 1, symbol.name, (const void *)symbol.address); - delete [] symbol.name; - } else { - fprintf(stderr, "#%3u: Unable to obtain symbol\n", f + 1); - } - } - } - - fputc('\n', stderr); - - return EXCEPTION_EXECUTE_HANDLER; - } -}; -using FatalSignalHandler = WindowsFaultHandler; - -#elif defined(Q_OS_UNIX) && !defined(Q_OS_WASM) -class FatalSignalHandler -{ -public: -# define OUR_SIGNALS(F) \ - F(HUP) \ - F(INT) \ - F(QUIT) \ - F(ABRT) \ - F(ILL) \ - F(BUS) \ - F(FPE) \ - F(SEGV) \ - F(PIPE) \ - F(TERM) \ - /**/ -# define CASE_LABEL(S) case SIG ## S: return QT_STRINGIFY(S); -# define ENUMERATE_SIGNALS(S) SIG ## S, - static const char *signalName(int signum) noexcept - { - switch (signum) { - OUR_SIGNALS(CASE_LABEL) - } - -# if defined(__GLIBC_MINOR__) && (__GLIBC_MINOR__ >= 32 || __GLIBC__ > 2) - // get the other signal names from glibc 2.32 - // (accessing the sys_sigabbrev variable causes linker warnings) - if (const char *p = sigabbrev_np(signum)) - return p; -# endif - return "???"; - } - static constexpr std::array fatalSignals = { - OUR_SIGNALS(ENUMERATE_SIGNALS) - }; -# undef CASE_LABEL -# undef ENUMERATE_SIGNALS - - static constexpr std::array crashingSignals = { - // Crash signals are special, because if we return from the handler - // without adjusting the machine state, the same instruction that - // originally caused the crash will get re-executed and will thus cause - // the same crash again. This is useful if our parent process logs the - // exit result or if core dumps are enabled: the core file will point - // to the actual instruction that crashed. - SIGILL, SIGBUS, SIGFPE, SIGSEGV - }; - using OldActionsArray = std::array<struct sigaction, fatalSignals.size()>; - - FatalSignalHandler() - { - pauseOnCrash = qEnvironmentVariableIsSet("QTEST_PAUSE_ON_CRASH"); - struct sigaction act; - memset(&act, 0, sizeof(act)); - act.sa_handler = SIG_DFL; - oldActions().fill(act); - - // Remove the handler after it is invoked. - act.sa_flags = SA_RESETHAND | setupAlternateStack(); - -# ifdef SA_SIGINFO - act.sa_flags |= SA_SIGINFO; - act.sa_sigaction = FatalSignalHandler::actionHandler; -# else - act.sa_handler = FatalSignalHandler::regularHandler; -# endif - - // Block all fatal signals in our signal handler so we don't try to close - // the testlog twice. - sigemptyset(&act.sa_mask); - for (int signal : fatalSignals) - sigaddset(&act.sa_mask, signal); - - for (size_t i = 0; i < fatalSignals.size(); ++i) - sigaction(fatalSignals[i], &act, &oldActions()[i]); - } - - ~FatalSignalHandler() - { - // Restore the default signal handlers in place of ours. - // If ours has been replaced, leave the replacement alone. - auto isOurs = [](const struct sigaction &old) { -# ifdef SA_SIGINFO - return (old.sa_flags & SA_SIGINFO) && old.sa_sigaction == FatalSignalHandler::actionHandler; -# else - return old.sa_handler == FatalSignalHandler::regularHandler; -# endif - }; - struct sigaction action; - - for (size_t i = 0; i < fatalSignals.size(); ++i) { - struct sigaction &act = oldActions()[i]; - if (act.sa_flags == 0 && act.sa_handler == SIG_DFL) - continue; // Already the default - if (sigaction(fatalSignals[i], nullptr, &action)) - continue; // Failed to query present handler - if (isOurs(action)) - sigaction(fatalSignals[i], &act, nullptr); - } - - freeAlternateStack(); - } - -private: - Q_DISABLE_COPY_MOVE(FatalSignalHandler) - - static OldActionsArray &oldActions() - { - Q_CONSTINIT static OldActionsArray oldActions {}; - return oldActions; - } - - auto alternateStackSize() - { - struct R { size_t size, pageSize; }; - static constexpr size_t MinStackSize = 32 * 1024; - size_t pageSize = sysconf(_SC_PAGESIZE); - size_t size = SIGSTKSZ; - if (size < MinStackSize) { - size = MinStackSize; - } else { - // round up to a page - size = (size + pageSize - 1) & -pageSize; - } - - return R{ size + pageSize, pageSize }; - } - - int setupAlternateStack() - { - // tvOS/watchOS both define SA_ONSTACK (in sys/signal.h) but mark sigaltstack() as - // unavailable (__WATCHOS_PROHIBITED __TVOS_PROHIBITED in signal.h) -# if defined(SA_ONSTACK) && !defined(Q_OS_TVOS) && !defined(Q_OS_WATCHOS) - // Let the signal handlers use an alternate stack - // This is necessary if SIGSEGV is to catch a stack overflow - auto r = alternateStackSize(); - int flags = MAP_PRIVATE | MAP_ANONYMOUS; -# ifdef MAP_STACK - flags |= MAP_STACK; -# endif - alternateStackBase = mmap(nullptr, r.size, PROT_READ | PROT_WRITE, flags, -1, 0); - if (alternateStackBase == MAP_FAILED) - return 0; - - // mark the bottom page inaccessible, to catch a handler stack overflow - (void) mprotect(alternateStackBase, r.pageSize, PROT_NONE); - - stack_t stack; - stack.ss_flags = 0; - stack.ss_size = r.size - r.pageSize; - stack.ss_sp = static_cast<char *>(alternateStackBase) + r.pageSize; - sigaltstack(&stack, nullptr); - return SA_ONSTACK; -# else - return 0; -# endif - } - - void freeAlternateStack() - { -# if defined(SA_ONSTACK) && !defined(Q_OS_TVOS) && !defined(Q_OS_WATCHOS) - if (alternateStackBase != MAP_FAILED) { - stack_t stack = {}; - stack.ss_flags = SS_DISABLE; - sigaltstack(&stack, nullptr); - munmap(alternateStackBase, alternateStackSize().size); - } -# endif - } - - template <typename T> static - std::enable_if_t<sizeof(std::declval<T>().si_pid) + sizeof(std::declval<T>().si_uid) >= 1> - printSentSignalInfo(T *info) - { - writeToStderr(" sent by PID ", asyncSafeToString(info->si_pid), - " UID ", asyncSafeToString(info->si_uid)); - } - static void printSentSignalInfo(...) {} - - template <typename T> static - std::enable_if_t<sizeof(std::declval<T>().si_addr) >= 1> printCrashingSignalInfo(T *info) - { - using HexString = std::array<char, sizeof(quintptr) * 2>; - auto toHexString = [](quintptr u, HexString &&r = {}) { - int shift = sizeof(quintptr) * 8 - 4; - for (size_t i = 0; i < sizeof(quintptr) * 2; ++i, shift -= 4) - r[i] = QtMiscUtils::toHexLower(u >> shift); - struct iovec vec; - vec.iov_base = r.data(); - vec.iov_len = r.size(); - return vec; - }; - writeToStderr(", code ", asyncSafeToString(info->si_code), - ", for address 0x", toHexString(quintptr(info->si_addr))); - } - static void printCrashingSignalInfo(...) {} - - static void actionHandler(int signum, siginfo_t *info, void * /* ucontext */) - { - writeToStderr("Received signal ", asyncSafeToString(signum), - " (SIG", signalName(signum), ")"); - - bool isCrashingSignal = - std::find(crashingSignals.begin(), crashingSignals.end(), signum) != crashingSignals.end(); - if (isCrashingSignal && (!info || info->si_code <= 0)) - isCrashingSignal = false; // wasn't sent by the kernel, so it's not really a crash - if (isCrashingSignal) - printCrashingSignalInfo(info); - else if (info && (info->si_code == SI_USER || info->si_code == SI_QUEUE)) - printSentSignalInfo(info); - - printTestRunTime(); - if (signum != SIGINT) { - generateStackTrace(); - if (pauseOnCrash) { - writeToStderr("Pausing process ", asyncSafeToString(getpid()), - " for debugging\n"); - raise(SIGSTOP); - } - } - - // chain back to the previous handler, if any - for (size_t i = 0; i < fatalSignals.size(); ++i) { - struct sigaction &act = oldActions()[i]; - if (signum != fatalSignals[i]) - continue; - - // restore the handler (if SA_RESETHAND hasn't done the job for us) - if (SA_RESETHAND == 0 || act.sa_handler != SIG_DFL || act.sa_flags) - (void) sigaction(signum, &act, nullptr); - - if (!isCrashingSignal) - raise(signum); - - // signal is blocked, so it'll be delivered when we return - return; - } - - // we shouldn't reach here! - std::abort(); - } - - [[maybe_unused]] static void regularHandler(int signum) - { - actionHandler(signum, nullptr, nullptr); - } - - void *alternateStackBase = MAP_FAILED; - static bool pauseOnCrash; -}; -bool FatalSignalHandler::pauseOnCrash = false; -#else // Q_OS_WASM or weird systems -class FatalSignalHandler {}; -#endif // Q_OS_* choice - -} // unnamed namespace - static void initEnvironment() { qputenv("QT_QTESTLIB_RUNNING", "1"); @@ -2226,6 +1807,14 @@ int QTest::qExec(QObject *testObject, int argc, char **argv) qInit(testObject, argc, argv); int ret = qRun(); qCleanup(); + +#if defined(Q_OS_WASM) + EM_ASM({ + if (typeof Module != "undefined" && typeof Module.notifyTestFinished != "undefined") + Module.notifyTestFinished($0); + }, ret); +#endif // Q_OS_WASM + return ret; } @@ -2234,7 +1823,7 @@ int QTest::qExec(QObject *testObject, int argc, char **argv) void QTest::qInit(QObject *testObject, int argc, char **argv) { initEnvironment(); - maybeDisableCoreDump(); + CrashHandler::maybeDisableCoreDump(); QBenchmarkGlobalData::current = new QBenchmarkGlobalData; #if defined(Q_OS_MACOS) @@ -2270,10 +1859,7 @@ void QTest::qInit(QObject *testObject, int argc, char **argv) #if QT_CONFIG(valgrind) if (QBenchmarkGlobalData::current->mode() != QBenchmarkGlobalData::CallgrindParentProcess) #endif - { - QTestTable::globalTestTable(); QTestLog::startLogging(); - } } /*! \internal @@ -2304,15 +1890,15 @@ int QTest::qRun() } else #endif { - std::optional<FatalSignalHandler> handler; - prepareStackTrace(); - if (!noCrashHandler) + std::optional<CrashHandler::FatalSignalHandler> handler; + CrashHandler::prepareStackTrace(); + if (!Internal::noCrashHandler) handler.emplace(); bool seenBad = false; TestMethods::MetaMethods commandLineMethods; commandLineMethods.reserve(static_cast<size_t>(QTest::testFunctions.size())); - for (const QString &tf : qAsConst(QTest::testFunctions)) { + for (const QString &tf : std::as_const(QTest::testFunctions)) { const QByteArray tfB = tf.toLatin1(); const QByteArray signature = tfB + QByteArrayLiteral("()"); QMetaMethod m = TestMethods::findMethod(currentTestObject, signature.constData()); @@ -2338,7 +1924,12 @@ int QTest::qRun() return 1; } TestMethods test(currentTestObject, std::move(commandLineMethods)); - test.invokeTests(currentTestObject); + + while (QTestLog::failCount() == 0 && (repeatForever || repetitions-- > 0)) { + QTestTable::globalTestTable(); + test.invokeTests(currentTestObject); + QTestTable::clearGlobalTestTable(); + } } #ifndef QT_NO_EXCEPTIONS @@ -2375,10 +1966,7 @@ void QTest::qCleanup() #if QT_CONFIG(valgrind) if (QBenchmarkGlobalData::current->mode() != QBenchmarkGlobalData::CallgrindParentProcess) #endif - { QTestLog::stopLogging(); - QTestTable::clearGlobalTestTable(); - } delete QBenchmarkGlobalData::current; QBenchmarkGlobalData::current = nullptr; @@ -2388,6 +1976,34 @@ void QTest::qCleanup() #endif } +#if QT_CONFIG(batch_test_support) || defined(Q_QDOC) +/*! + Registers the test \a name, with entry function \a entryFunction, in a + central test case registry for the current binary. + + The \a name will be listed when running the batch test binary with no + parameters. Running the test binary with the argv[1] of \a name will result + in \a entryFunction being called. + + \since 6.5 +*/ +void QTest::qRegisterTestCase(const QString &name, TestEntryFunction entryFunction) +{ + QTest::TestRegistry::instance()->registerTest(name, entryFunction); +} + +QList<QString> QTest::qGetTestCaseNames() +{ + return QTest::TestRegistry::instance()->getAllTestNames(); +} + +QTest::TestEntryFunction QTest::qGetTestCaseEntryFunction(const QString& name) +{ + return QTest::TestRegistry::instance()->getTestEntryFunction(name); +} + +#endif // QT_CONFIG(batch_test_support) + /*! \overload \since 4.4 @@ -2397,7 +2013,7 @@ void QTest::qCleanup() */ int QTest::qExec(QObject *testObject, const QStringList &arguments) { - const int argc = arguments.count(); + const int argc = arguments.size(); QVarLengthArray<char *> argv(argc); QList<QByteArray> args; @@ -2478,6 +2094,39 @@ void QTest::qCaught(const char *expected, const char *what, const char *file, in qFail(message().toUtf8().constData(), file, line); } +/*! + \internal + + Contains the implementation of the catch(...) block of + QVERIFY_THROWS_EXCEPTION. + + The function inspects std::current_exception() by rethrowing it using + std::rethrow_exception(). + + The function must be called from a catch handler. + + If the exception inherits std::exception, its what() message is logged and + this function returns normally. The caller of this function must then + execute a \c{QTEST_FAIL_ACTION} to exit from the test function. + + Otherwise, a message saying an unknown exception was caught is logged and + this function rethrows the exception, skipping the \c{QTEST_FAIL_ACTION} + that follows this function call in the caller. +*/ +void QTest::qCaught(const char *expected, const char *file, int line) +{ + try { + // let's see what the cat brought us: + std::rethrow_exception(std::current_exception()); + } catch (const std::exception &e) { + qCaught(expected, e.what(), file, line); + } catch (...) { + qCaught(expected, nullptr, file, line); + throw; + } + // caller shall invoke `QTEST_FAIL_ACTION` if control reached here +} + #if QT_DEPRECATED_SINCE(6, 3) /*! @@ -2637,30 +2286,36 @@ QSharedPointer<QTemporaryDir> QTest::qExtractTestData(const QString &dirName) return result; } - QDirIterator it(resourcePath, QDirIterator::Subdirectories); - if (!it.hasNext()) { - qWarning("Resource directory '%s' is empty.", qPrintable(resourcePath)); - return result; - } - - while (it.hasNext()) { - QFileInfo fileInfo = it.nextFileInfo(); - - if (!fileInfo.isDir()) { - const QString destination = dataPath + u'/' + QStringView{fileInfo.filePath()}.mid(resourcePath.length()); + bool isResourceDirEmpty = true; + for (const auto &dirEntry : QDirListing(resourcePath, QDirListing::IteratorFlag::Recursive)) { + isResourceDirEmpty = false; + if (!dirEntry.isDir()) { + const QString &filePath = dirEntry.filePath(); + const QString destination = + dataPath + u'/' + QStringView{filePath}.sliced(resourcePath.size()); QFileInfo destinationFileInfo(destination); QDir().mkpath(destinationFileInfo.path()); - if (!QFile::copy(fileInfo.filePath(), destination)) { - qWarning("Failed to copy '%s'.", qPrintable(fileInfo.filePath())); + QFile file(filePath); + if (!file.copy(destination)) { + qWarning("Failed to copy '%ls': %ls.", qUtf16Printable(filePath), + qUtf16Printable(file.errorString())); return result; } - if (!QFile::setPermissions(destination, QFile::ReadUser | QFile::WriteUser | QFile::ReadGroup)) { - qWarning("Failed to set permissions on '%s'.", qPrintable(destination)); + + file.setFileName(destination); + if (!file.setPermissions(QFile::ReadUser | QFile::WriteUser | QFile::ReadGroup)) { + qWarning("Failed to set permissions on '%ls': %ls.", qUtf16Printable(destination), + qUtf16Printable(file.errorString())); return result; } } } + if (isResourceDirEmpty) { + qWarning("Resource directory '%s' is empty.", qPrintable(resourcePath)); + return result; + } + result = std::move(tempDir); return result; @@ -2854,9 +2509,13 @@ void QTest::addColumnInternal(int id, const char *name) } /*! - Appends a new row to the current test data. \a dataTag is the name of - the testdata that will appear in the test output. Returns a QTestData reference - that can be used to stream in data. + Appends a new row to the current test data. + + The test output will identify the test run with this test data using the + name \a dataTag. + + Returns a QTestData reference that can be used to stream in data, one value + for each column in the table. Example: \snippet code/src_qtestlib_qtestcase.cpp 20 @@ -2867,14 +2526,15 @@ void QTest::addColumnInternal(int id, const char *name) See \l {Chapter 2: Data Driven Testing}{Data Driven Testing} for a more extensive example. - \sa addColumn(), QFETCH() + \sa addRow(), addColumn(), QFETCH() */ QTestData &QTest::newRow(const char *dataTag) { QTEST_ASSERT_X(dataTag, "QTest::newRow()", "Data tag cannot be null"); QTestTable *tbl = QTestTable::currentTestTable(); QTEST_ASSERT_X(tbl, "QTest::newRow()", "Cannot add testdata outside of a _data slot."); - QTEST_ASSERT_X(tbl->elementCount(), "QTest::newRow()", "Must add columns before attempting to add rows."); + QTEST_ASSERT_X(tbl->elementCount(), "QTest::newRow()", + "Must add columns before attempting to add rows."); return *tbl->newData(dataTag); } @@ -2882,13 +2542,17 @@ QTestData &QTest::newRow(const char *dataTag) /*! \since 5.9 - Appends a new row to the current test data. The function's arguments are passed - to qsnprintf() for formatting according to \a format. See the qvsnprintf() - documentation for caveats and limitations. + Appends a new row to the current test data. - The formatted string will appear as the name of this test data in the test output. + The function's arguments are passed to qsnprintf() for formatting according + to \a format. See the qvsnprintf() documentation for caveats and + limitations. - Returns a QTestData reference that can be used to stream in data. + The test output will identify the test run with this test data using the + name that results from this formatting. + + Returns a QTestData reference that can be used to stream in data, one value + for each column in the table. Example: \snippet code/src_qtestlib_qtestcase.cpp addRow @@ -2899,14 +2563,15 @@ QTestData &QTest::newRow(const char *dataTag) See \l {Chapter 2: Data Driven Testing}{Data Driven Testing} for a more extensive example. - \sa addColumn(), QFETCH() + \sa newRow(), addColumn(), QFETCH() */ QTestData &QTest::addRow(const char *format, ...) { QTEST_ASSERT_X(format, "QTest::addRow()", "Format string cannot be null"); QTestTable *tbl = QTestTable::currentTestTable(); QTEST_ASSERT_X(tbl, "QTest::addRow()", "Cannot add testdata outside of a _data slot."); - QTEST_ASSERT_X(tbl->elementCount(), "QTest::addRow()", "Must add columns before attempting to add rows."); + QTEST_ASSERT_X(tbl->elementCount(), "QTest::addRow()", + "Must add columns before attempting to add rows."); char buf[1024]; @@ -2975,13 +2640,46 @@ const char *QTest::currentDataTag() } /*! - Returns \c true if the current test function failed, otherwise false. + Returns \c true if the current test function has failed, otherwise false. + + \sa QTest::currentTestResolved() */ bool QTest::currentTestFailed() { return QTestResult::currentTestFailed(); } +/*! + \since 6.5 + Returns \c true if the current test function has failed or skipped. + + This applies if the test has failed or exercised a skip. When it is true, + the test function should return early. In particular, the \c{QTRY_*} macros + and the test event loop terminate their loops early if executed during the + test function (but not its cleanup()). After a test has called a helper + function that uses this module's macros, it can use this function to test + whether to return early. + + \sa QTest::currentTestFailed() +*/ +bool QTest::currentTestResolved() +{ + return QTestResult::currentTestFailed() || QTestResult::skipCurrentTest(); +} + +/*! + \internal + \since 6.4 + Returns \c true during the run of the test-function and its set-up. + + Used by the \c{QTRY_*} macros and \l QTestEventLoop to check whether to + return when QTest::currentTestResolved() is true. +*/ +bool QTest::runningTest() +{ + return QTest::inTestFunction; +} + /*! \internal */ QObject *QTest::testObject() |