QTest: add opt-in changing QCOMPARE etc to exit with throw, not return

Add exception classes and use them to change the control flow for
QTEST_{FAIL,SKIP}_ACTION from return'ing from just the immediate
function to the full way to the QTestLib infrastructure, here we
filter them out.

There are three modes:
- If QT_NO_EXCEPTION, then we return
  - If QTEST_THROW_ON_... is also defined, #error out
- Otherwise, if QTEST_THROW_ON_... is defined, always throw
- Otherwise, the decision is made at runtime (with defaults read from
  QTEST_THROW_ON_... environment variables).

Three selftests depend on the old behavior, as they explicitly check
that multiple FAIL SKIP etc emerge, which the new framework, of
course, prevents. Locally disable throwing at the test function level.

Add initial docs and enable exceptions in all of the selftest
subprograms to facilitate switching between the two runtime-selectable
modes.

[ChangeLog][QtTest] Added QTEST_THROW_ON_FAIL and QTEST_THROW_ON_SKIP
C++ macros and environment variables that, when defined, change how
QCOMPARE/QVERIFY/QSKIP etc exit the test function on failure. Instead
of a return, exiting only the immediately-surrounding function, they
throw a special exception instead, thereby exiting from subfunctions
of the test function, all the way to QtTestLib.

Fixes: QTBUG-66320
Change-Id: I96c38d2a1dcdd9de84942cf448a8bbf3ab6d3679
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>

3 files changed, 258 insertions, 1 deletions

diff --git a/src/testlib/doc/src/qttestlib-manual.qdoc b/src/testlib/doc/src/qttestlib-manual.qdoc
index 5c7b298fb9..712de24a49 100644
--- a/src/testlib/doc/src/qttestlib-manual.qdoc
+++ b/src/testlib/doc/src/qttestlib-manual.qdoc
@@ -415,6 +415,12 @@
     to e.g. debug an unstable or intermittent failure in a test, by
     launching the test in a debugger. Support for this variable was
     added in Qt 6.1.
+    \li \c QTEST_THROW_ON_FAIL (since 6.8) \br
+    Setting this variable to a non-zero value will cause QCOMPARE()/QVERIFY()
+    etc to throw on failure (as opposed to just returning from the
+    immediately-surrounding function scope).
+    \li \c QTEST_THROW_ON_SKIP (since 6.8) \br
+    Same as \c QTEST_THROW_ON_SKIP, except affecting QSKIP().
     \endlist
 
     \section1 Creating a Benchmark
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index 313e9c45c4..33b1d06398 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -458,9 +458,193 @@ namespace QTestPrivate
     Q_TESTLIB_EXPORT Qt::MouseButtons qtestMouseButtons = Qt::NoButton;
 }
 
+namespace {
+
+class TestFailedException : public std::exception // clazy:exclude=copyable-polymorphic
+{
+public:
+    TestFailedException() = default;
+    ~TestFailedException() override = default;
+
+    const char *what() const noexcept override { return "QtTest: test failed"; }
+};
+
+class TestSkippedException : public std::exception // clazy:exclude=copyable-polymorphic
+{
+public:
+    TestSkippedException() = default;
+    ~TestSkippedException() override = default;
+
+    const char *what() const noexcept override { return "QtTest: test was skipped"; }
+};
+
+} // unnamed namespace
+
 namespace QTest
 {
 
+void Internal::throwOnFail() { throw TestFailedException(); }
+void Internal::throwOnSkip() { throw TestSkippedException(); }
+
+Q_CONSTINIT static QBasicAtomicInt g_throwOnFail = Q_BASIC_ATOMIC_INITIALIZER(0);
+Q_CONSTINIT static QBasicAtomicInt g_throwOnSkip = Q_BASIC_ATOMIC_INITIALIZER(0);
+
+void Internal::maybeThrowOnFail()
+{
+    if (g_throwOnFail.loadRelaxed() > 0)
+        Internal::throwOnFail();
+}
+
+void Internal::maybeThrowOnSkip()
+{
+    if (g_throwOnSkip.loadRelaxed() > 0)
+        Internal::throwOnSkip();
+}
+
+/*!
+    \since 6.8
+    \macro QTEST_THROW_ON_FAIL
+    \relates <QTest>
+
+    When defined, QCOMPARE()/QVERIFY() etc always throw on failure.
+    QTest::throwOnFail() then no longer has any effect.
+*/
+
+/*!
+    \since 6.8
+    \macro QTEST_THROW_ON_SKIP
+    \relates <QTest>
+
+    When defined, QSKIP() always throws. QTest::throwOnSkip() then no longer
+    has any effect.
+*/
+
+/*!
+    \since 6.8
+    \class QTest::ThrowOnFailEnabler
+    \inmodule QtTestLib
+
+    RAII class around setThrowOnFail().
+*/
+/*!
+    \fn QTest::ThrowOnFailEnabler::ThrowOnFailEnabler()
+
+    Constructor. Calls \c{setThrowOnFail(true)}.
+*/
+/*!
+    \fn QTest::ThrowOnFailEnabler::~ThrowOnFailEnabler()
+
+    Destructor. Calls \c{setThrowOnFail(false)}.
+*/
+
+/*!
+    \since 6.8
+    \class QTest::ThrowOnFailDisabler
+    \inmodule QtTestLib
+
+    RAII class around setThrowOnFail().
+*/
+/*!
+    \fn QTest::ThrowOnFailDisabler::ThrowOnFailDisabler()
+
+    Constructor. Calls \c{setThrowOnFail(false)}.
+*/
+/*!
+    \fn QTest::ThrowOnFailDisabler::~ThrowOnFailDisabler()
+
+    Destructor. Calls \c{setThrowOnFail(true)}.
+*/
+
+/*!
+    \since 6.8
+    \class QTest::ThrowOnSkipEnabler
+    \inmodule QtTestLib
+
+    RAII class around setThrowOnSkip().
+*/
+/*!
+    \fn QTest::ThrowOnSkipEnabler::ThrowOnSkipEnabler()
+
+    Constructor. Calls \c{setThrowOnSkip(true)}.
+*/
+/*!
+    \fn QTest::ThrowOnSkipEnabler::~ThrowOnSkipEnabler()
+
+    Destructor. Calls \c{setThrowOnSkip(false)}.
+*/
+
+/*!
+    \since 6.8
+    \class QTest::ThrowOnSkipDisabler
+    \inmodule QtTestLib
+
+    RAII class around setThrowOnSkip().
+*/
+/*!
+    \fn QTest::ThrowOnSkipDisabler::ThrowOnSkipDisabler()
+
+    Constructor. Calls \c{setThrowOnSkip(false)}.
+*/
+/*!
+    \fn QTest::ThrowOnSkipDisabler::~ThrowOnSkipDisabler()
+
+    Destructor. Calls \c{setThrowOnSkip(true)}.
+*/
+
+/*!
+    \since 6.8
+
+    Enables (\a enable = \c true) or disables (\ enable = \c false) throwing on
+    QCOMPARE()/QVERIFY() failures (as opposed to just returning from the
+    immediately-surrounding function context).
+
+    The feature is reference-counted: If you call this function \e{N} times
+    with \c{true}, you need to call it \e{N} times with \c{false} to get back
+    to where you started.
+
+    The default is \c{false}, unless the \l{Qt Test Environment Variables}
+    {QTEST_THROW_ON_FAIL environment variable} is set.
+
+    This call has no effect when the \l{QTEST_THROW_ON_FAIL} C++ macro is
+    defined.
+
+    \note You must compile your tests with exceptions enabled to use this
+    feature.
+
+    \sa setThrowOnSkip(), ThrowOnFailEnabler, ThrowOnFailDisabler, QTEST_THROW_ON_FAIL
+*/
+void setThrowOnFail(bool enable) noexcept
+{
+    g_throwOnFail.fetchAndAddRelaxed(enable ? 1 : -1);
+}
+
+/*!
+    \since 6.8
+
+    Enables (\a enable = \c true) or disables (\ enable = \c false) throwing on
+    QSKIP() (as opposed to just returning from the immediately-surrounding
+    function context).
+
+    The feature is reference-counted: If you call this function \e{N} times
+    with \c{true}, you need to call it \e{N} times with \c{false} to get back
+    to where you started.
+
+    The default is \c{false}, unless the \l{Qt Test Environment Variables}
+    {QTEST_THROW_ON_SKIP environment variable} is set.
+
+    This call has no effect when the \l{QTEST_THROW_ON_SKIP} C++ macro is
+    defined.
+
+    \note You must compile your tests with exceptions enabled to use this
+    feature.
+
+    \sa setThrowOnFail(), ThrowOnSkipEnabler, ThrowOnSkipDisabler, QTEST_THROW_ON_SKIP
+*/
+void setThrowOnSkip(bool enable) noexcept
+{
+    g_throwOnSkip.fetchAndAddRelaxed(enable ? 1 : -1);
+}
+
 QString Internal::formatTryTimeoutDebugMessage(q_no_char8_t::QUtf8StringView expr, int timeout, int actual)
 {
     return "QTestLib: This test case check (\"%1\") failed because the requested timeout (%2 ms) "
@@ -546,7 +730,14 @@ static bool skipBlacklisted = false;
 
 static bool invokeTestMethodIfValid(QMetaMethod m, QObject *obj = QTest::currentTestObject)
 {
-    return m.isValid() && m.invoke(obj, Qt::DirectConnection);
+    if (!m.isValid())
+        return false;
+    bool ok = true;
+    try { ok = m.invoke(obj, Qt ::DirectConnection); }
+    catch (const TestFailedException &) {}  // ignore (used for control flow)
+    catch (const TestSkippedException &) {} // ditto
+    // every other exception is someone else's problem
+    return ok;
 }
 
 static void invokeTestMethodIfExists(const char *methodName, QObject *obj = QTest::currentTestObject)
@@ -720,6 +911,11 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
     QTest::testFunctions.clear();
     QTest::testTags.clear();
 
+    if (qEnvironmentVariableIsSet("QTEST_THROW_ON_FAIL"))
+        QTest::setThrowOnFail(true);
+    if (qEnvironmentVariableIsSet("QTEST_THROW_ON_SKIP"))
+        QTest::setThrowOnSkip(true);
+
 #if defined(Q_OS_DARWIN) && defined(HAVE_XCTEST)
     if (QXcodeTestLogger::canLogTestProgress())
         logFormat = QTestLog::XCTest;
diff --git a/src/testlib/qtestcase.h b/src/testlib/qtestcase.h
index 62eb579986..ae926ab632 100644
--- a/src/testlib/qtestcase.h
+++ b/src/testlib/qtestcase.h
@@ -23,6 +23,26 @@
 
 QT_BEGIN_NAMESPACE
 
+#ifndef QT_NO_EXCEPTIONS
+
+#ifdef QTEST_THROW_ON_FAILURE
+# define QTEST_FAIL_ACTION QTest::Internal::throwOnFail()
+#else
+# define QTEST_FAIL_ACTION do { QTest::Internal::maybeThrowOnFail(); return; } while (false)
+#endif
+
+#ifdef QTEST_THROW_ON_SKIP
+# define QTEST_SKIP_ACTION QTest::Internal::throwOnSkip()
+#else
+# define QTEST_SKIP_ACTION do { QTest::Internal::maybeThrowOnSkip(); return; } while (false)
+#endif
+
+#else
+# if defined(QTEST_THROW_ON_FAILURE) || defined(QTEST_THROW_ON_SKIP)
+#  error QTEST_THROW_ON_FAILURE/SKIP require exception support enabled.
+# endif
+#endif // QT_NO_EXCEPTIONS
+
 #ifndef QTEST_FAIL_ACTION
 # define QTEST_FAIL_ACTION return
 #endif
@@ -300,6 +320,11 @@ namespace QTest
 {
     namespace Internal {
 
+    [[noreturn]] Q_TESTLIB_EXPORT void throwOnFail();
+    [[noreturn]] Q_TESTLIB_EXPORT void throwOnSkip();
+    Q_TESTLIB_EXPORT void maybeThrowOnFail();
+    Q_TESTLIB_EXPORT void maybeThrowOnSkip();
+
     Q_TESTLIB_EXPORT QString formatTryTimeoutDebugMessage(q_no_char8_t::QUtf8StringView expr, int timeout, int actual);
 
     template<typename T> // Output registered enums
@@ -385,6 +410,36 @@ namespace QTest
 #endif  // QT_CONFIG(batch_test_support)
 
     Q_TESTLIB_EXPORT void setMainSourcePath(const char *file, const char *builddir = nullptr);
+    Q_TESTLIB_EXPORT void setThrowOnFail(bool enable) noexcept;
+    Q_TESTLIB_EXPORT void setThrowOnSkip(bool enable) noexcept;
+
+    class ThrowOnFailEnabler {
+        Q_DISABLE_COPY_MOVE(ThrowOnFailEnabler)
+    public:
+        ThrowOnFailEnabler() { setThrowOnFail(true); }
+        ~ThrowOnFailEnabler() { setThrowOnFail(false); }
+    };
+
+    class ThrowOnSkipEnabler {
+        Q_DISABLE_COPY_MOVE(ThrowOnSkipEnabler)
+    public:
+        ThrowOnSkipEnabler() { setThrowOnSkip(true); }
+        ~ThrowOnSkipEnabler() { setThrowOnSkip(false); }
+    };
+
+    class ThrowOnFailDisabler {
+        Q_DISABLE_COPY_MOVE(ThrowOnFailDisabler)
+    public:
+        ThrowOnFailDisabler() { setThrowOnFail(false); }
+        ~ThrowOnFailDisabler() { setThrowOnFail(true); }
+    };
+
+    class ThrowOnSkipDisabler {
+        Q_DISABLE_COPY_MOVE(ThrowOnSkipDisabler)
+    public:
+        ThrowOnSkipDisabler() { setThrowOnSkip(false); }
+        ~ThrowOnSkipDisabler() { setThrowOnSkip(true); }
+    };
 
     Q_TESTLIB_EXPORT bool qVerify(bool statement, const char *statementStr, const char *description,
                                  const char *file, int line);