diff options
Diffstat (limited to 'src/testlib/qtestlog.cpp')
| -rw-r--r-- | src/testlib/qtestlog.cpp | 227 |
1 files changed, 163 insertions, 64 deletions
diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp index 7e311d273e..4b6df54e91 100644 --- a/src/testlib/qtestlog.cpp +++ b/src/testlib/qtestlog.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the QtTest module of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <QtTest/qtestassert.h> @@ -60,6 +24,7 @@ #include <QtCore/qbytearray.h> #include <QtCore/qelapsedtimer.h> #include <QtCore/qlist.h> +#include <QtCore/qmutex.h> #include <QtCore/qvariant.h> #if QT_CONFIG(regularexpression) #include <QtCore/QRegularExpression> @@ -68,9 +33,15 @@ #include <stdlib.h> #include <string.h> #include <limits.h> +#include <vector> + +#include <vector> +#include <memory> QT_BEGIN_NAMESPACE +using namespace Qt::StringLiterals; + static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage) { #ifdef __COVERAGESCANNER__ @@ -96,10 +67,10 @@ static void saveCoverageTool(const char * appname, bool testfailed, bool install #endif } -static QElapsedTimer elapsedFunctionTime; -static QElapsedTimer elapsedTotalTime; +Q_CONSTINIT static QElapsedTimer elapsedFunctionTime; +Q_CONSTINIT static QElapsedTimer elapsedTotalTime; -#define FOREACH_TEST_LOGGER for (QAbstractTestLogger *logger : *QTest::loggers()) +#define FOREACH_TEST_LOGGER for (const auto &logger : std::as_const(*QTest::loggers())) namespace QTest { @@ -107,6 +78,7 @@ namespace QTest { int passes = 0; int skips = 0; int blacklists = 0; + enum { Unresolved, Passed, Skipped, Suppressed, Failed } currentTestState; struct IgnoreResultList { @@ -143,8 +115,8 @@ namespace QTest { // ignore an optional whitespace at the end of str // (the space was added automatically by ~QDebug() until Qt 5.3, // so autotests still might expect it) - if (expected.endsWith(QLatin1Char(' '))) - return actual == QStringView{expected}.left(expected.length() - 1); + if (expected.endsWith(u' ')) + return actual == QStringView{expected}.left(expected.size() - 1); return false; } @@ -167,8 +139,11 @@ namespace QTest { }; static IgnoreResultList *ignoreResultList = nullptr; + Q_CONSTINIT static QBasicMutex mutex; - Q_GLOBAL_STATIC(QList<QAbstractTestLogger *>, loggers) + static std::vector<QVariant> failOnWarningList; + + Q_GLOBAL_STATIC(std::vector<std::unique_ptr<QAbstractTestLogger>>, loggers) static int verbosity = 0; static int maxWarnings = 2002; @@ -178,6 +153,8 @@ namespace QTest { static bool handleIgnoredMessage(QtMsgType type, const QString &message) { + const QMutexLocker mutexLocker(&QTest::mutex); + if (!ignoreResultList) return false; IgnoreResultList *last = nullptr; @@ -187,10 +164,8 @@ namespace QTest { // remove the item from the list if (last) last->next = list->next; - else if (list->next) - ignoreResultList = list->next; else - ignoreResultList = nullptr; + ignoreResultList = list->next; delete list; return true; @@ -202,14 +177,40 @@ namespace QTest { return false; } + static bool handleFailOnWarning(const QMessageLogContext &context, const QString &message) + { + // failOnWarning can be called multiple times per test function, so let + // each call cause a failure if required. + for (const auto &pattern : failOnWarningList) { + if (pattern.metaType() == QMetaType::fromType<QString>()) { + if (message != pattern.toString()) + continue; + } +#if QT_CONFIG(regularexpression) + else if (pattern.metaType() == QMetaType::fromType<QRegularExpression>()) { + if (!message.contains(pattern.toRegularExpression())) + continue; + } +#endif + + const size_t maxMsgLen = 1024; + char msg[maxMsgLen] = {'\0'}; + qsnprintf(msg, maxMsgLen, "Received a warning that resulted in a failure:\n%s", + qPrintable(message)); + QTestResult::addFailure(msg, context.file, context.line); + return true; + } + return false; + } + static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message) { static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings); - if (QTestLog::loggerCount() == 0) { + if (!QTestLog::hasLoggers()) { // if this goes wrong, something is seriously broken. qInstallMessageHandler(oldMessageHandler); - QTEST_ASSERT(QTestLog::loggerCount() != 0); + QTEST_ASSERT(QTestLog::hasLoggers()); } if (handleIgnoredMessage(type, message)) { @@ -217,6 +218,9 @@ namespace QTest { return; } + if (type == QtWarningMsg && handleFailOnWarning(context, message)) + return; + if (type != QtFatalMsg) { if (counter.loadRelaxed() <= 0) return; @@ -268,6 +272,7 @@ void QTestLog::enterTestData(QTestData *data) int QTestLog::unhandledIgnoreMessages() { + const QMutexLocker mutexLocker(&QTest::mutex); int i = 0; QTest::IgnoreResultList *list = QTest::ignoreResultList; while (list) { @@ -288,14 +293,16 @@ void QTestLog::leaveTestFunction() void QTestLog::printUnhandledIgnoreMessages() { + const QMutexLocker mutexLocker(&QTest::mutex); QString message; QTest::IgnoreResultList *list = QTest::ignoreResultList; while (list) { if (list->pattern.userType() == QMetaType::QString) { - message = QStringLiteral("Did not receive message: \"") + list->pattern.toString() + QLatin1Char('"'); + message = "Did not receive message: \"%1\""_L1.arg(list->pattern.toString()); } else { #if QT_CONFIG(regularexpression) - message = QStringLiteral("Did not receive any message matching: \"") + list->pattern.toRegularExpression().pattern() + QLatin1Char('"'); + message = "Did not receive any message matching: \"%1\""_L1.arg( + list->pattern.toRegularExpression().pattern()); #endif } FOREACH_TEST_LOGGER @@ -307,17 +314,32 @@ void QTestLog::printUnhandledIgnoreMessages() void QTestLog::clearIgnoreMessages() { + const QMutexLocker mutexLocker(&QTest::mutex); QTest::IgnoreResultList::clearList(QTest::ignoreResultList); } +void QTestLog::clearFailOnWarnings() +{ + QTest::failOnWarningList.clear(); +} + +void QTestLog::clearCurrentTestState() +{ + clearIgnoreMessages(); + clearFailOnWarnings(); + QTest::currentTestState = QTest::Unresolved; +} + void QTestLog::addPass(const char *msg) { if (printAvailableTags) return; QTEST_ASSERT(msg); + Q_ASSERT(QTest::currentTestState == QTest::Unresolved); ++QTest::passes; + QTest::currentTestState = QTest::Passed; FOREACH_TEST_LOGGER logger->addIncident(QAbstractTestLogger::Pass, msg); @@ -327,8 +349,18 @@ void QTestLog::addFail(const char *msg, const char *file, int line) { QTEST_ASSERT(msg); - ++QTest::fails; + if (QTest::currentTestState == QTest::Unresolved) { + ++QTest::fails; + } else { + // After an XPASS/Continue, or fail or skip in a function the test + // calls, we can subsequently fail. + Q_ASSERT(QTest::currentTestState == QTest::Failed + || QTest::currentTestState == QTest::Skipped); + } + // It is up to particular loggers to decide whether to report such + // subsequent failures; they may carry useful information. + QTest::currentTestState = QTest::Failed; FOREACH_TEST_LOGGER logger->addIncident(QAbstractTestLogger::Fail, msg, file, line); } @@ -347,8 +379,16 @@ void QTestLog::addXPass(const char *msg, const char *file, int line) { QTEST_ASSERT(msg); - ++QTest::fails; + if (QTest::currentTestState == QTest::Unresolved) { + ++QTest::fails; + } else { + // After an XPASS/Continue, we can subsequently XPASS again. + // Likewise after a fail or skip in a function called by the test. + Q_ASSERT(QTest::currentTestState == QTest::Failed + || QTest::currentTestState == QTest::Skipped); + } + QTest::currentTestState = QTest::Failed; FOREACH_TEST_LOGGER logger->addIncident(QAbstractTestLogger::XPass, msg, file, line); } @@ -356,8 +396,10 @@ void QTestLog::addXPass(const char *msg, const char *file, int line) void QTestLog::addBPass(const char *msg) { QTEST_ASSERT(msg); + Q_ASSERT(QTest::currentTestState == QTest::Unresolved); - ++QTest::blacklists; + ++QTest::blacklists; // Not passes ? + QTest::currentTestState = QTest::Suppressed; FOREACH_TEST_LOGGER logger->addIncident(QAbstractTestLogger::BlacklistedPass, msg); @@ -367,8 +409,16 @@ void QTestLog::addBFail(const char *msg, const char *file, int line) { QTEST_ASSERT(msg); - ++QTest::blacklists; + if (QTest::currentTestState == QTest::Unresolved) { + ++QTest::blacklists; + } else { + // After a BXPASS/Continue, we can subsequently fail. + // Likewise after a fail or skip in a function called by a test. + Q_ASSERT(QTest::currentTestState == QTest::Suppressed + || QTest::currentTestState == QTest::Skipped); + } + QTest::currentTestState = QTest::Suppressed; FOREACH_TEST_LOGGER logger->addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line); } @@ -377,8 +427,16 @@ void QTestLog::addBXPass(const char *msg, const char *file, int line) { QTEST_ASSERT(msg); - ++QTest::blacklists; + if (QTest::currentTestState == QTest::Unresolved) { + ++QTest::blacklists; + } else { + // After a BXPASS/Continue, we may BXPASS again. + // Likewise after a fail or skip in a function called by a test. + Q_ASSERT(QTest::currentTestState == QTest::Suppressed + || QTest::currentTestState == QTest::Skipped); + } + QTest::currentTestState = QTest::Suppressed; FOREACH_TEST_LOGGER logger->addIncident(QAbstractTestLogger::BlacklistedXPass, msg, file, line); } @@ -397,16 +455,27 @@ void QTestLog::addSkip(const char *msg, const char *file, int line) { QTEST_ASSERT(msg); - ++QTest::skips; + if (QTest::currentTestState == QTest::Unresolved) { + ++QTest::skips; + QTest::currentTestState = QTest::Skipped; + } else { + // After an B?XPASS/Continue, we might subsequently skip. + // Likewise after a skip in a function called by a test. + Q_ASSERT(QTest::currentTestState == QTest::Suppressed + || QTest::currentTestState == QTest::Failed + || QTest::currentTestState == QTest::Skipped); + } + // It is up to particular loggers to decide whether to report such + // subsequent skips; they may carry useful information. FOREACH_TEST_LOGGER - logger->addMessage(QAbstractTestLogger::Skip, QString::fromUtf8(msg), file, line); + logger->addIncident(QAbstractTestLogger::Skip, msg, file, line); } -void QTestLog::addBenchmarkResult(const QBenchmarkResult &result) +void QTestLog::addBenchmarkResults(const QList<QBenchmarkResult> &results) { FOREACH_TEST_LOGGER - logger->addBenchmarkResult(result); + logger->addBenchmarkResults(results); } void QTestLog::startLogging() @@ -423,7 +492,6 @@ void QTestLog::stopLogging() qInstallMessageHandler(QTest::oldMessageHandler); FOREACH_TEST_LOGGER { logger->stopLogging(); - delete logger; } QTest::loggers()->clear(); saveCoverageTool(QTestResult::currentAppName(), failCount() != 0, QTestLog::installedTestCoverage()); @@ -484,12 +552,27 @@ void QTestLog::addLogger(LogMode mode, const char *filename) void QTestLog::addLogger(QAbstractTestLogger *logger) { QTEST_ASSERT(logger); - QTest::loggers()->append(logger); + QTest::loggers()->emplace_back(logger); } -int QTestLog::loggerCount() +bool QTestLog::hasLoggers() { - return QTest::loggers()->size(); + return !QTest::loggers()->empty(); +} + +/*! + \internal + + Returns true if all loggers support repeated test runs +*/ +bool QTestLog::isRepeatSupported() +{ + FOREACH_TEST_LOGGER { + if (!logger->isRepeatSupported()) + return false; + } + + return true; } bool QTestLog::loggerUsingStdout() @@ -532,6 +615,7 @@ void QTestLog::ignoreMessage(QtMsgType type, const char *msg) { QTEST_ASSERT(msg); + const QMutexLocker mutexLocker(&QTest::mutex); QTest::IgnoreResultList::append(QTest::ignoreResultList, type, QString::fromUtf8(msg)); } @@ -540,10 +624,25 @@ void QTestLog::ignoreMessage(QtMsgType type, const QRegularExpression &expressio { QTEST_ASSERT(expression.isValid()); + const QMutexLocker mutexLocker(&QTest::mutex); QTest::IgnoreResultList::append(QTest::ignoreResultList, type, QVariant(expression)); } #endif // QT_CONFIG(regularexpression) +void QTestLog::failOnWarning(const char *msg) +{ + QTest::failOnWarningList.push_back(QString::fromUtf8(msg)); +} + +#if QT_CONFIG(regularexpression) +void QTestLog::failOnWarning(const QRegularExpression &expression) +{ + QTEST_ASSERT(expression.isValid()); + + QTest::failOnWarningList.push_back(QVariant::fromValue(expression)); +} +#endif // QT_CONFIG(regularexpression) + void QTestLog::setMaxWarnings(int m) { QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2; |