path: root/src/testlib/qtestlog.cpp

// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

#include <QtTest/qtestassert.h>

#include <QtTest/private/qtestlog_p.h>
#include <QtTest/private/qtestresult_p.h>
#include <QtTest/private/qabstracttestlogger_p.h>
#include <QtTest/private/qplaintestlogger_p.h>
#include <QtTest/private/qcsvbenchmarklogger_p.h>
#include <QtTest/private/qjunittestlogger_p.h>
#include <QtTest/private/qxmltestlogger_p.h>
#include <QtTest/private/qteamcitylogger_p.h>
#include <QtTest/private/qtaptestlogger_p.h>
#if defined(HAVE_XCTEST)
#include <QtTest/private/qxctestlogger_p.h>
#endif

#if defined(Q_OS_DARWIN)
#include <QtTest/private/qappletestlogger_p.h>
#endif

#include <QtCore/qatomic.h>
#include <QtCore/qbytearray.h>
#include <QtCore/qelapsedtimer.h>
#include <QtCore/qlist.h>
#include <QtCore/qmutex.h>
#include <QtCore/qvariant.h>
#if QT_CONFIG(regularexpression)
#include <QtCore/QRegularExpression>
#endif

#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <vector>

#include <vector>
#include <memory>

QT_BEGIN_NAMESPACE

using namespace Qt::StringLiterals;

static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage)
{
#ifdef __COVERAGESCANNER__
#  if QT_CONFIG(testlib_selfcover)
    __coveragescanner_teststate(QTestLog::failCount() > 0 ? "FAILED" :
                                QTestLog::passCount() > 0 ? "PASSED" : "SKIPPED");
#  else
    if (!installedTestCoverage)
        return;
    // install again to make sure the filename is correct.
    // without this, a plugin or similar may have changed the filename.
    __coveragescanner_install(appname);
    __coveragescanner_teststate(testfailed ? "FAILED" : "PASSED");
    __coveragescanner_save();
    __coveragescanner_testname("");
    __coveragescanner_clear();
    unsetenv("QT_TESTCOCOON_ACTIVE");
#  endif // testlib_selfcover
#else
    Q_UNUSED(appname);
    Q_UNUSED(testfailed);
    Q_UNUSED(installedTestCoverage);
#endif
}

Q_CONSTINIT static QElapsedTimer elapsedFunctionTime;
Q_CONSTINIT static QElapsedTimer elapsedTotalTime;

#define FOREACH_TEST_LOGGER for (const auto &logger : std::as_const(*QTest::loggers()))

namespace QTest {

    int fails = 0;
    int passes = 0;
    int skips = 0;
    int blacklists = 0;
    enum { Unresolved, Passed, Skipped, Suppressed, Failed } currentTestState;

    struct IgnoreResultList
    {
        inline IgnoreResultList(QtMsgType tp, const QVariant &patternIn)
            : type(tp), pattern(patternIn) {}

        static inline void clearList(IgnoreResultList *&list)
        {
            while (list) {
                IgnoreResultList *current = list;
                list = list->next;
                delete current;
            }
        }

        static void append(IgnoreResultList *&list, QtMsgType type, const QVariant &patternIn)
        {
            QTest::IgnoreResultList *item = new QTest::IgnoreResultList(type, patternIn);

            if (!list) {
                list = item;
                return;
            }
            IgnoreResultList *last = list;
            for ( ; last->next; last = last->next) ;
            last->next = item;
        }

        static bool stringsMatch(const QString &expected, const QString &actual)
        {
            if (expected == actual)
                return true;

            // ignore an optional whitespace at the end of str
            // (the space was added automatically by ~QDebug() until Qt 5.3,
            //  so autotests still might expect it)
            if (expected.endsWith(u' '))
                return actual == QStringView{expected}.left(expected.size() - 1);

            return false;
        }

        inline bool matches(QtMsgType tp, const QString &message) const
        {
            return tp == type
                   && (pattern.userType() == QMetaType::QString ?
                       stringsMatch(pattern.toString(), message) :
#if QT_CONFIG(regularexpression)
                       pattern.toRegularExpression().match(message).hasMatch());
#else
                       false);
#endif
        }

        QtMsgType type;
        QVariant pattern;
        IgnoreResultList *next = nullptr;
    };

    static IgnoreResultList *ignoreResultList = nullptr;
    Q_CONSTINIT static QBasicMutex mutex;

    static std::vector<QVariant> failOnWarningList;

    Q_GLOBAL_STATIC(std::vector<std::unique_ptr<QAbstractTestLogger>>, loggers)

    static int verbosity = 0;
    static int maxWarnings = 2002;
    static bool installedTestCoverage = true;

    static QtMessageHandler oldMessageHandler;

    static bool handleIgnoredMessage(QtMsgType type, const QString &message)
    {
        const QMutexLocker mutexLocker(&QTest::mutex);

        if (!ignoreResultList)
            return false;
        IgnoreResultList *last = nullptr;
        IgnoreResultList *list = ignoreResultList;
        while (list) {
            if (list->matches(type, message)) {
                // remove the item from the list
                if (last)
                    last->next = list->next;
                else
                    ignoreResultList = list->next;

                delete list;
                return true;
            }

            last = list;
            list = list->next;
        }
        return false;
    }

    static bool handleFailOnWarning(const QMessageLogContext &context, const QString &message)
    {
        // failOnWarning can be called multiple times per test function, so let
        // each call cause a failure if required.
        for (const auto &pattern : failOnWarningList) {
            if (pattern.metaType() == QMetaType::fromType<QString>()) {
                if (message != pattern.toString())
                    continue;
            }
#if QT_CONFIG(regularexpression)
            else if (pattern.metaType() == QMetaType::fromType<QRegularExpression>()) {
                if (!message.contains(pattern.toRegularExpression()))
                    continue;
            }
#endif

            const size_t maxMsgLen = 1024;
            char msg[maxMsgLen] = {'\0'};
            qsnprintf(msg, maxMsgLen, "Received a warning that resulted in a failure:\n%s",
                      qPrintable(message));
            QTestResult::addFailure(msg, context.file, context.line);
            return true;
        }
        return false;
    }

    static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message)
    {
        static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings);

        if (!QTestLog::hasLoggers()) {
            // if this goes wrong, something is seriously broken.
            qInstallMessageHandler(oldMessageHandler);
            QTEST_ASSERT(QTestLog::hasLoggers());
        }

        if (handleIgnoredMessage(type, message)) {
            // the message is expected, so just swallow it.
            return;
        }

        if (type == QtWarningMsg && handleFailOnWarning(context, message))
            return;

        if (type != QtFatalMsg) {
            if (counter.loadRelaxed() <= 0)
                return;

            if (!counter.deref()) {
                FOREACH_TEST_LOGGER {
                    logger->addMessage(QAbstractTestLogger::Warn,
                        QStringLiteral("Maximum amount of warnings exceeded. Use -maxwarnings to override."));
                }
                return;
            }
        }

        FOREACH_TEST_LOGGER
            logger->addMessage(type, context, message);

        if (type == QtFatalMsg) {
             /* Right now, we're inside the custom message handler and we're
             * being qt_message_output in qglobal.cpp. After we return from
             * this function, it will proceed with calling exit() and abort()
             * and hence crash. Therefore, we call these logging functions such
             * that we wrap up nicely, and in particular produce well-formed XML. */
            QTestResult::addFailure("Received a fatal error.", context.file, context.line);
            QTestLog::leaveTestFunction();
            QTestLog::stopLogging();
        }
    }
}

void QTestLog::enterTestFunction(const char* function)
{
    elapsedFunctionTime.restart();
    if (printAvailableTags)
        return;

    QTEST_ASSERT(function);

    FOREACH_TEST_LOGGER
        logger->enterTestFunction(function);
}

void QTestLog::enterTestData(QTestData *data)
{
    QTEST_ASSERT(data);

    FOREACH_TEST_LOGGER
        logger->enterTestData(data);
}

int QTestLog::unhandledIgnoreMessages()
{
    const QMutexLocker mutexLocker(&QTest::mutex);
    int i = 0;
    QTest::IgnoreResultList *list = QTest::ignoreResultList;
    while (list) {
        ++i;
        list = list->next;
    }
    return i;
}

void QTestLog::leaveTestFunction()
{
    if (printAvailableTags)
        return;

    FOREACH_TEST_LOGGER
        logger->leaveTestFunction();
}

void QTestLog::printUnhandledIgnoreMessages()
{
    const QMutexLocker mutexLocker(&QTest::mutex);
    QString message;
    QTest::IgnoreResultList *list = QTest::ignoreResultList;
    while (list) {
        if (list->pattern.userType() == QMetaType::QString) {
            message = "Did not receive message: \"%1\""_L1.arg(list->pattern.toString());
        } else {
#if QT_CONFIG(regularexpression)
            message = "Did not receive any message matching: \"%1\""_L1.arg(
                    list->pattern.toRegularExpression().pattern());
#endif
        }
        FOREACH_TEST_LOGGER
            logger->addMessage(QAbstractTestLogger::Info, message);

        list = list->next;
    }
}

void QTestLog::clearIgnoreMessages()
{
    const QMutexLocker mutexLocker(&QTest::mutex);
    QTest::IgnoreResultList::clearList(QTest::ignoreResultList);
}

void QTestLog::clearFailOnWarnings()
{
    QTest::failOnWarningList.clear();
}

void QTestLog::clearCurrentTestState()
{
    clearIgnoreMessages();
    clearFailOnWarnings();
    QTest::currentTestState = QTest::Unresolved;
}

void QTestLog::addPass(const char *msg)
{
    if (printAvailableTags)
        return;

    QTEST_ASSERT(msg);
    Q_ASSERT(QTest::currentTestState == QTest::Unresolved);

    ++QTest::passes;
    QTest::currentTestState = QTest::Passed;

    FOREACH_TEST_LOGGER
        logger->addIncident(QAbstractTestLogger::Pass, msg);
}

void QTestLog::addFail(const char *msg, const char *file, int line)
{
    QTEST_ASSERT(msg);

    if (QTest::currentTestState == QTest::Unresolved) {
        ++QTest::fails;
    } else {
        // After an XPASS/Continue, or fail or skip in a function the test
        // calls, we can subsequently fail.
        Q_ASSERT(QTest::currentTestState == QTest::Failed
                 || QTest::currentTestState == QTest::Skipped);
    }
    // It is up to particular loggers to decide whether to report such
    // subsequent failures; they may carry useful information.

    QTest::currentTestState = QTest::Failed;
    FOREACH_TEST_LOGGER
        logger->addIncident(QAbstractTestLogger::Fail, msg, file, line);
}

void QTestLog::addXFail(const char *msg, const char *file, int line)
{
    QTEST_ASSERT(msg);

    // Will be counted in addPass() if we get there.

    FOREACH_TEST_LOGGER
        logger->addIncident(QAbstractTestLogger::XFail, msg, file, line);
}

void QTestLog::addXPass(const char *msg, const char *file, int line)
{
    QTEST_ASSERT(msg);

    if (QTest::currentTestState == QTest::Unresolved) {
        ++QTest::fails;
    } else {
        // After an XPASS/Continue, we can subsequently XPASS again.
        // Likewise after a fail or skip in a function called by the test.
        Q_ASSERT(QTest::currentTestState == QTest::Failed
                 || QTest::currentTestState == QTest::Skipped);
    }

    QTest::currentTestState = QTest::Failed;
    FOREACH_TEST_LOGGER
        logger->addIncident(QAbstractTestLogger::XPass, msg, file, line);
}

void QTestLog::addBPass(const char *msg)
{
    QTEST_ASSERT(msg);
    Q_ASSERT(QTest::currentTestState == QTest::Unresolved);

    ++QTest::blacklists; // Not passes ?
    QTest::currentTestState = QTest::Suppressed;

    FOREACH_TEST_LOGGER
        logger->addIncident(QAbstractTestLogger::BlacklistedPass, msg);
}

void QTestLog::addBFail(const char *msg, const char *file, int line)
{
    QTEST_ASSERT(msg);

    if (QTest::currentTestState == QTest::Unresolved) {
        ++QTest::blacklists;
    } else {
        // After a BXPASS/Continue, we can subsequently fail.
        // Likewise after a fail or skip in a function called by a test.
        Q_ASSERT(QTest::currentTestState == QTest::Suppressed
                 || QTest::currentTestState == QTest::Skipped);
    }

    QTest::currentTestState = QTest::Suppressed;
    FOREACH_TEST_LOGGER
        logger->addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line);
}

void QTestLog::addBXPass(const char *msg, const char *file, int line)
{
    QTEST_ASSERT(msg);

    if (QTest::currentTestState == QTest::Unresolved) {
        ++QTest::blacklists;
    } else {
        // After a BXPASS/Continue, we may BXPASS again.
        // Likewise after a fail or skip in a function called by a test.
        Q_ASSERT(QTest::currentTestState == QTest::Suppressed
                 || QTest::currentTestState == QTest::Skipped);
    }

    QTest::currentTestState = QTest::Suppressed;
    FOREACH_TEST_LOGGER
        logger->addIncident(QAbstractTestLogger::BlacklistedXPass, msg, file, line);
}

void QTestLog::addBXFail(const char *msg, const char *file, int line)
{
    QTEST_ASSERT(msg);

    // Will be counted in addBPass() if we get there.

    FOREACH_TEST_LOGGER
        logger->addIncident(QAbstractTestLogger::BlacklistedXFail, msg, file, line);
}

void QTestLog::addSkip(const char *msg, const char *file, int line)
{
    QTEST_ASSERT(msg);

    if (QTest::currentTestState == QTest::Unresolved) {
        ++QTest::skips;
        QTest::currentTestState = QTest::Skipped;
    } else {
        // After an B?XPASS/Continue, we might subsequently skip.
        // Likewise after a skip in a function called by a test.
        Q_ASSERT(QTest::currentTestState == QTest::Suppressed
                 || QTest::currentTestState == QTest::Failed
                 || QTest::currentTestState == QTest::Skipped);
    }
    // It is up to particular loggers to decide whether to report such
    // subsequent skips; they may carry useful information.

    FOREACH_TEST_LOGGER
        logger->addIncident(QAbstractTestLogger::Skip, msg, file, line);
}

void QTestLog::addBenchmarkResults(const QList<QBenchmarkResult> &results)
{
    FOREACH_TEST_LOGGER
        logger->addBenchmarkResults(results);
}

void QTestLog::startLogging()
{
    elapsedTotalTime.start();
    elapsedFunctionTime.start();
    FOREACH_TEST_LOGGER
        logger->startLogging();
    QTest::oldMessageHandler = qInstallMessageHandler(QTest::messageHandler);
}

void QTestLog::stopLogging()
{
    qInstallMessageHandler(QTest::oldMessageHandler);
    FOREACH_TEST_LOGGER {
        logger->stopLogging();
    }
    QTest::loggers()->clear();
    saveCoverageTool(QTestResult::currentAppName(), failCount() != 0, QTestLog::installedTestCoverage());
}

void QTestLog::addLogger(LogMode mode, const char *filename)
{
    if (filename && strcmp(filename, "-") == 0)
        filename = nullptr;

    QAbstractTestLogger *logger = nullptr;
    switch (mode) {
    case QTestLog::Plain:
        logger = new QPlainTestLogger(filename);
        break;
    case QTestLog::CSV:
        logger = new QCsvBenchmarkLogger(filename);
        break;
    case QTestLog::XML:
        logger = new QXmlTestLogger(QXmlTestLogger::Complete, filename);
        break;
    case QTestLog::LightXML:
        logger = new QXmlTestLogger(QXmlTestLogger::Light, filename);
        break;
    case QTestLog::JUnitXML:
        logger = new QJUnitTestLogger(filename);
        break;
    case QTestLog::TeamCity:
        logger = new QTeamCityLogger(filename);
        break;
    case QTestLog::TAP:
        logger = new QTapTestLogger(filename);
        break;
#if defined(QT_USE_APPLE_UNIFIED_LOGGING)
    case QTestLog::Apple:
        logger = new QAppleTestLogger;
        break;
#endif
#if defined(HAVE_XCTEST)
    case QTestLog::XCTest:
        logger = new QXcodeTestLogger;
        break;
#endif
    }

    QTEST_ASSERT(logger);
    addLogger(logger);
}

/*!
    \internal

    Adds a new logger to the set of loggers that will be used
    to report incidents and messages during testing.

    The function takes ownership of the logger.
*/
void QTestLog::addLogger(QAbstractTestLogger *logger)
{
    QTEST_ASSERT(logger);
    QTest::loggers()->emplace_back(logger);
}

bool QTestLog::hasLoggers()
{
    return !QTest::loggers()->empty();
}

/*!
    \internal

    Returns true if all loggers support repeated test runs
*/
bool QTestLog::isRepeatSupported()
{
    FOREACH_TEST_LOGGER {
        if (!logger->isRepeatSupported())
            return false;
    }

    return true;
}

bool QTestLog::loggerUsingStdout()
{
    FOREACH_TEST_LOGGER {
        if (logger->isLoggingToStdout())
            return true;
    }

    return false;
}

void QTestLog::warn(const char *msg, const char *file, int line)
{
    QTEST_ASSERT(msg);

    FOREACH_TEST_LOGGER
        logger->addMessage(QAbstractTestLogger::Warn, QString::fromUtf8(msg), file, line);
}

void QTestLog::info(const char *msg, const char *file, int line)
{
    QTEST_ASSERT(msg);

    FOREACH_TEST_LOGGER
        logger->addMessage(QAbstractTestLogger::Info, QString::fromUtf8(msg), file, line);
}

void QTestLog::setVerboseLevel(int level)
{
    QTest::verbosity = level;
}

int QTestLog::verboseLevel()
{
    return QTest::verbosity;
}

void QTestLog::ignoreMessage(QtMsgType type, const char *msg)
{
    QTEST_ASSERT(msg);

    const QMutexLocker mutexLocker(&QTest::mutex);
    QTest::IgnoreResultList::append(QTest::ignoreResultList, type, QString::fromUtf8(msg));
}

#if QT_CONFIG(regularexpression)
void QTestLog::ignoreMessage(QtMsgType type, const QRegularExpression &expression)
{
    QTEST_ASSERT(expression.isValid());

    const QMutexLocker mutexLocker(&QTest::mutex);
    QTest::IgnoreResultList::append(QTest::ignoreResultList, type, QVariant(expression));
}
#endif // QT_CONFIG(regularexpression)

void QTestLog::failOnWarning(const char *msg)
{
    QTest::failOnWarningList.push_back(QString::fromUtf8(msg));
}

#if QT_CONFIG(regularexpression)
void QTestLog::failOnWarning(const QRegularExpression &expression)
{
    QTEST_ASSERT(expression.isValid());

    QTest::failOnWarningList.push_back(QVariant::fromValue(expression));
}
#endif // QT_CONFIG(regularexpression)

void QTestLog::setMaxWarnings(int m)
{
    QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2;
}

bool QTestLog::printAvailableTags = false;

void QTestLog::setPrintAvailableTagsMode()
{
    printAvailableTags = true;
}

int QTestLog::passCount()
{
    return QTest::passes;
}

int QTestLog::failCount()
{
    return QTest::fails;
}

int QTestLog::skipCount()
{
    return QTest::skips;
}

int QTestLog::blacklistCount()
{
    return QTest::blacklists;
}

int QTestLog::totalCount()
{
    return passCount() + failCount() + skipCount() + blacklistCount();
}

void QTestLog::resetCounters()
{
    QTest::passes = 0;
    QTest::fails = 0;
    QTest::skips = 0;
}

void QTestLog::setInstalledTestCoverage(bool installed)
{
    QTest::installedTestCoverage = installed;
}

bool QTestLog::installedTestCoverage()
{
    return QTest::installedTestCoverage;
}

qint64 QTestLog::nsecsTotalTime()
{
    return elapsedTotalTime.nsecsElapsed();
}

qint64 QTestLog::nsecsFunctionTime()
{
    return elapsedFunctionTime.nsecsElapsed();
}

QT_END_NAMESPACE

#include "moc_qtestlog_p.cpp"