Add QTest::failOnWarning

This solves the long-standing problem of not being able to easily
fail a test when a certain warning is output.

[ChangeLog][QtTest] Added QTest::failOnWarning. When called in a test
function, any warning that matches the given pattern will cause a test
failure. The test will continue execution when a failure is added.
All patterns are cleared at the end of each test function.

Fixes: QTBUG-70029
Change-Id: I5763f8d4acf1cee8178be43a503619fbfb0f4f36
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

5 files changed, 123 insertions, 0 deletions

diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index c95712ec97..4d0b30a844 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -2127,6 +2127,68 @@ void QTest::ignoreMessage(QtMsgType type, const QRegularExpression &messagePatte
 }
 #endif // QT_CONFIG(regularexpression)
 
+/*!
+    \since 6.3
+    \overload failOnWarning()
+
+    Appends a test failure to the test log if the \a message is output.
+
+    \sa failOnWarning()
+*/
+void QTest::failOnWarning(const char *message)
+{
+    return QTestLog::failOnWarning(message);
+}
+
+#if QT_CONFIG(regularexpression)
+/*!
+    \since 6.3
+
+    Appends a test failure to the test log for each warning that matches
+    \a messagePattern.
+
+    The test function will continue execution when a failure is added. To abort
+    the test instead, you can check \l currentTestFailed() and return early if
+    it's \c true.
+
+    For each warning, the first pattern that matches will cause a failure,
+    and the remaining patterns will be ignored.
+
+    All patterns are cleared at the end of each test function.
+
+    \code
+    void FileTest::loadFiles()
+    {
+        QTest::failOnWarning(QRegularExpression("^Failed to load"));
+
+        // Each of these will cause a test failure:
+        qWarning() << "Failed to load image";
+        qWarning() << "Failed to load video";
+    }
+    \endcode
+
+    To fail every test that triggers a given warning, pass a suitable regular
+    expression to this function in \l {Creating a Test}{init()}:
+
+    \code
+    void FileTest::init()
+    {
+        QTest::failOnWarning(QRegularExpression(".?"));
+    }
+    \endcode
+
+    \note \l ignoreMessage() takes precedence over this function, so any
+    warnings that match a pattern given to both \c ignoreMessage() and
+    \c failOnWarning() will be ignored.
+
+    \sa {Qt Test Environment Variables}{QTEST_FATAL_FAIL}
+*/
+void QTest::failOnWarning(const QRegularExpression &messagePattern)
+{
+    QTestLog::failOnWarning(messagePattern);
+}
+#endif // QT_CONFIG(regularexpression)
+
 /*! \internal
  */
 
diff --git a/src/testlib/qtestcase.h b/src/testlib/qtestcase.h
index c5a90c8e1b..eea99e26e4 100644
--- a/src/testlib/qtestcase.h
+++ b/src/testlib/qtestcase.h
@@ -331,6 +331,10 @@ namespace QTest
 #if QT_CONFIG(regularexpression)
     Q_TESTLIB_EXPORT void ignoreMessage(QtMsgType type, const QRegularExpression &messagePattern);
 #endif
+    Q_TESTLIB_EXPORT void failOnWarning(const char *message);
+#if QT_CONFIG(regularexpression)
+    Q_TESTLIB_EXPORT void failOnWarning(const QRegularExpression &messagePattern);
+#endif
 
 #if QT_CONFIG(temporaryfile)
     Q_TESTLIB_EXPORT QSharedPointer<QTemporaryDir> qExtractTestData(const QString &dirName);
diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp
index cebe548ec6..7b13f01e24 100644
--- a/src/testlib/qtestlog.cpp
+++ b/src/testlib/qtestlog.cpp
@@ -68,6 +68,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <limits.h>
+#include <vector>
 
 #include <vector>
 #include <memory>
@@ -171,6 +172,8 @@ namespace QTest {
 
     static IgnoreResultList *ignoreResultList = nullptr;
 
+    static std::vector<QVariant> failOnWarningList;
+
     Q_GLOBAL_STATIC(std::vector<std::unique_ptr<QAbstractTestLogger>>, loggers)
 
     static int verbosity = 0;
@@ -205,6 +208,32 @@ namespace QTest {
         return false;
     }
 
+    static bool handleFailOnWarning(const QMessageLogContext &context, const QString &message)
+    {
+        // failOnWarnings can be called multiple times per test function, so let
+        // each call cause a failure if required.
+        for (const auto &pattern : failOnWarningList) {
+            if (pattern.metaType() == QMetaType::fromType<QString>()) {
+                if (message != pattern.toString())
+                    continue;
+            }
+#if QT_CONFIG(regularexpression)
+            else if (pattern.metaType() == QMetaType::fromType<QRegularExpression>()) {
+                if (!message.contains(pattern.toRegularExpression()))
+                    continue;
+            }
+#endif
+
+            const size_t maxMsgLen = 1024;
+            char msg[maxMsgLen] = {'\0'};
+            qsnprintf(msg, maxMsgLen, "Received a warning that resulted in a failure:\n%s",
+                      qPrintable(message));
+            QTestResult::addFailure(msg, context.file, context.line);
+            return true;
+        }
+        return false;
+    }
+
     static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message)
     {
         static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings);
@@ -220,6 +249,9 @@ namespace QTest {
             return;
         }
 
+        if (type == QtWarningMsg && handleFailOnWarning(context, message))
+            return;
+
         if (type != QtFatalMsg) {
             if (counter.loadRelaxed() <= 0)
                 return;
@@ -313,6 +345,11 @@ void QTestLog::clearIgnoreMessages()
     QTest::IgnoreResultList::clearList(QTest::ignoreResultList);
 }
 
+void QTestLog::clearFailOnWarnings()
+{
+    QTest::failOnWarningList.clear();
+}
+
 void QTestLog::addPass(const char *msg)
 {
     if (printAvailableTags)
@@ -546,6 +583,20 @@ void QTestLog::ignoreMessage(QtMsgType type, const QRegularExpression &expressio
 }
 #endif // QT_CONFIG(regularexpression)
 
+void QTestLog::failOnWarning(const char *msg)
+{
+    QTest::failOnWarningList.push_back(QString::fromUtf8(msg));
+}
+
+#if QT_CONFIG(regularexpression)
+void QTestLog::failOnWarning(const QRegularExpression &expression)
+{
+    QTEST_ASSERT(expression.isValid());
+
+    QTest::failOnWarningList.push_back(QVariant::fromValue(expression));
+}
+#endif // QT_CONFIG(regularexpression)
+
 void QTestLog::setMaxWarnings(int m)
 {
     QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2;
diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h
index c422aadbf8..cc24360bed 100644
--- a/src/testlib/qtestlog_p.h
+++ b/src/testlib/qtestlog_p.h
@@ -105,9 +105,14 @@ public:
 #ifndef QT_NO_REGULAREXPRESSION
     static void ignoreMessage(QtMsgType type, const QRegularExpression &expression);
 #endif
+    static void failOnWarning(const char *msg);
+#ifndef QT_NO_REGULAREXPRESSION
+    static void failOnWarning(const QRegularExpression &expression);
+#endif
     static int unhandledIgnoreMessages();
     static void printUnhandledIgnoreMessages();
     static void clearIgnoreMessages();
+    static void clearFailOnWarnings();
 
     static void warn(const char *msg, const char *file, int line);
     static void info(const char *msg, const char *file, int line);
diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp
index 899f5358b5..ae40164392 100644
--- a/src/testlib/qtestresult.cpp
+++ b/src/testlib/qtestresult.cpp
@@ -191,6 +191,7 @@ void QTestResult::finishedCurrentTestData()
         addFailure("Not all expected messages were received");
     }
     QTestLog::clearIgnoreMessages();
+    QTestLog::clearFailOnWarnings();
 }
 
 /*!