diff options
Diffstat (limited to 'src/testlib/qabstracttestlogger.cpp')
| -rw-r--r-- | src/testlib/qabstracttestlogger.cpp | 248 |
1 files changed, 146 insertions, 102 deletions
diff --git a/src/testlib/qabstracttestlogger.cpp b/src/testlib/qabstracttestlogger.cpp index fda350ea13..de6fb63560 100644 --- a/src/testlib/qabstracttestlogger.cpp +++ b/src/testlib/qabstracttestlogger.cpp @@ -1,44 +1,9 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the QtTest module of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <QtTest/private/qabstracttestlogger_p.h> #include <QtTest/qtestassert.h> +#include <qbenchmark_p.h> #include <qtestresult_p.h> #include <QtCore/qbytearray.h> @@ -75,26 +40,67 @@ QT_BEGIN_NAMESPACE /*! \enum QAbstractTestLogger::IncidentTypes - \value Pass - \value XFail - \value Fail - \value XPass - \value BlacklistedPass - \value BlacklistedFail - \value BlacklistedXPass - \value BlacklistedXFail + \value Pass The test ran to completion successfully. + \value XFail The test failed a check that is known to fail; this failure + does not prevent successful completion of the test and may be + followed by further checks. + \value Fail The test fails. + \value XPass A check which was expected to fail actually passed. This is + counted as a failure, as it means whatever caused the known failure + no longer does, so the test needs an update. + \value Skip The current test ended prematurely, skipping some checks. + \value BlacklistedPass As Pass but the test was blacklisted. + \value BlacklistedXFail As XFail but the test was blacklisted. + \value BlacklistedFail As Fail but the test was blacklisted. + \value BlacklistedXPass As XPass but the test was blacklisted. + + A test may also skip (see \l {QAbstractTestLogger::}{MessageTypes}). The + first of skip, Fail, XPass or the blacklisted equivalents of the last two to + arise is decisive for the outcome of the test: loggers which should only + report one outcome should thus record that as the outcome and either ignore + later incidents (or skips) in the same run of the test function or map them + to some form of message. + + \note tests can be "blacklisted" when they are known to fail + unreliably. When testing is used to decide whether a change to the code + under test is acceptable, such failures are not automatic grounds for + rejecting the change, if the unreliable failure was known before the + change. QTest::qExec(), as a result, only returns a failing status code if + some non-blacklisted test failed. Logging backends may reasonably report a + blacklisted result just as they would report the non-blacklisted equivalent, + optionally with some annotation to indicate that the result should not be + taken as damning evidence against recent changes to the code under test. + + \sa QAbstractTestLogger::addIncident() */ /*! \enum QAbstractTestLogger::MessageTypes - \value Warn - \value QWarning - \value QDebug - \value QCritical - \value QFatal - \value Skip - \value Info - \value QInfo + + The members whose names begin with \c Q describe messages that originate in + calls, by the test or code under test, to Qt logging functions (implemented + as macros) whose names are similar, with a \c q in place of the leading \c + Q. The other members describe messages generated internally by QtTest. + + \value QInfo An informational message from qInfo(). + \value QWarning A warning from qWarning(). + \value QDebug A debug message from qDebug(). + \value QCritical A critical error from qCritical(). + \value QFatal A fatal error from qFatal(), or an unrecognised message from + the Qt logging functions. + \value Info Messages QtTest generates as requested by the \c{-v1} or \c{-v2} + command-line option being specified when running the test. + \value Warn A warning generated internally by QtTest + + \note For these purposes, some utilities provided by QtTestlib as helper + functions to facilitate testing - such as \l QSignalSpy, \l + QTestAccessibility, \l QTest::qExtractTestData(), and the facilities to + deliver artificial mouse and keyboard events - are treated as test code, + rather than internal to QtTest; they call \l qWarning() and friends rather + than using the internal logging infrastructure, so that \l + QTest::ignoreMessage() can be used to anticipate the messages. + + \sa QAbstractTestLogger::addMessage() */ /*! @@ -102,7 +108,7 @@ QT_BEGIN_NAMESPACE Derived classes should pass this base-constructor the \a filename of the file to which they shall log test results, or \nullptr to write to standard - output. The protected member \c stream is set to the open file desriptor. + output. The protected member \c stream is set to the open file descriptor. */ QAbstractTestLogger::QAbstractTestLogger(const char *filename) { @@ -142,7 +148,20 @@ QAbstractTestLogger::~QAbstractTestLogger() } /*! - Returns true precisely if the \c output stream is standard output. + Returns true if the logger supports repeated test runs. + + Repetition of test runs is disabled by default, and can be enabled only for + test loggers that support it. Even if the logger may create syntactically + correct test reports, log-file analyzers may assume that test names are + unique within one report file. +*/ +bool QAbstractTestLogger::isRepeatSupported() const +{ + return false; +} + +/*! + Returns true if the \c output stream is standard output. */ bool QAbstractTestLogger::isLoggingToStdout() const { @@ -191,7 +210,7 @@ void QAbstractTestLogger::outputString(const char *msg) Called before the start of a test run. This virtual method is called before the first tests are run. A logging - implementation might open a file, write some preamble or prepare in other + implementation might open a file, write some preamble, or prepare in other ways, such as setting up initial values of variables. It can use the usual Qt logging infrastucture, since it is also called before QtTest installs its own custom message handler. @@ -207,7 +226,7 @@ void QAbstractTestLogger::startLogging() This virtual method is called after all tests have run. A logging implementation might collate information gathered from the run, write a - summary or close a file. It can use the usual Qt logging infrastucture, + summary, or close a file. It can use the usual Qt logging infrastucture, since it is also called after QtTest has restored the default message handler it replaced with its own custom message handler. @@ -217,6 +236,12 @@ void QAbstractTestLogger::stopLogging() { } +void QAbstractTestLogger::addBenchmarkResults(const QList<QBenchmarkResult> &result) +{ + for (const auto &m : result) + addBenchmarkResult(m); +} + /*! \fn void QAbstractTestLogger::enterTestFunction(const char *function) @@ -225,7 +250,7 @@ void QAbstractTestLogger::stopLogging() function. It is likewise called for \c{initTestCase()} at the start of testing, after \l startLogging(), and for \c{cleanupTestCase()} at the end of testing, in each case passing the name of the function. It is also called - with \nullptr as \function after the last of these functions, or in the + with \nullptr as \a function after the last of these functions, or in the event of an early end to testing, before \l stopLogging(). For data-driven test functions, this is called only once, before the data @@ -246,8 +271,8 @@ void QAbstractTestLogger::stopLogging() Every logging implementation must implement this method. In some cases it may be called more than once without an intervening call to \l - enterTestFunction(); in such cases, the implementation should ignore the - second (and later) calls. + enterTestFunction(). In such cases, the implementation should ignore these + later calls, until the next call to enterTestFunction(). \sa enterTestFunction(), enterTestData() */ @@ -269,20 +294,21 @@ void QAbstractTestLogger::stopLogging() /*! \fn void QAbstractTestLogger::addIncident(IncidentTypes type, const char *description, const char *file, int line) - This virtual method is called when an event occurs that bears on whether the - test passes or fails. The \a type indicates whether this was a pass or a - fail, whether a failure was expected and whether the test being run is - blacklisted. The \a description may be empty (for a pass), a message - describing the failure or, for an expected failure, the explanation of why a - failure was expected. Where the location in code of the incident is known, - it is indicated by \a file and \a line; otherwise, these are \a nullptr and - 0, respectively. + This virtual method is called when an event occurs that relates to the + resolution of the test. The \a type indicates whether this was a pass, a + fail or a skip, whether a failure was expected, and whether the test being + run is blacklisted. The \a description may be empty (for a pass) or a + message describing the nature of the incident. Where the location in code of + the incident is known, it is indicated by \a file and \a line; otherwise, + these are \a nullptr and 0, respectively. Every logging implementation must implement this method. Note that there are circumstances where more than one incident may be reported, in this way, for - a single run of a test on a single dataset. It is (for now) the - implementation's responsibility to recognize such cases and decide what to - do about them. + a single run of a test on a single dataset. It is the implementation's + responsibility to recognize such cases and decide what to do about them. For + purposes of counting resolutions of tests in the "Totals" report at the end + of a test run, QtTest considers the first incident (excluding XFail and its + blacklisted variant) decisive. \sa addMessage(), addBenchmarkResult() */ @@ -302,17 +328,17 @@ void QAbstractTestLogger::stopLogging() \fn void QAbstractTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line) This virtual method is called, via its \c QtMsgType overload, from the - custom message handler QtTest installs. It is also used by the - implementations of QSKIP(), to warn about various situations detected by - QtTest itself, such as \e failure to see a message anticipated by - QTest::ignoreMessage() and, particularly when verbosity options have been - enabled via the command-line, to log some extra information. + custom message handler QtTest installs. It is also used to + warn about various situations detected by QtTest itself, such + as \e failure to see a message anticipated by QTest::ignoreMessage() and, + particularly when verbosity options have been enabled via the command-line, + to log some extra information. Every logging implementation must implement this method. The \a type indicates the category of message and the \a message is the content to be reported. When the message is associated with specific code, the name of the - \a file and \a line number within it are also supplied (otherwise, these are - \nullptr and 0, respectively). + \a file and \a line number within it are also supplied; otherwise, these are + \nullptr and 0, respectively. \sa QTest::ignoreMessage(), addIncident() */ @@ -326,10 +352,10 @@ void QAbstractTestLogger::stopLogging() messages have previously been processed. (The limiting number of messages is controlled by the -maxwarnings option to a test and defaults to 2002.) - Logging implementation should not normally need to over-ride this method. + Logging implementations should not normally need to override this method. The base implementation converts \a type to the matching \l MessageType, - formats the given \a message suitably for the specified \a context and - forwards the converted type and formatted message to the overload taking + formats the given \a message suitably for the specified \a context, and + forwards the converted type and formatted message to the overload that takes MessageType and QString. \sa QTest::ignoreMessage(), addIncident() @@ -345,8 +371,7 @@ void QAbstractTestLogger::addMessage(QtMsgType type, const QMessageLogContext &c case QtWarningMsg: return QAbstractTestLogger::QWarning; case QtFatalMsg: return QAbstractTestLogger::QFatal; } - Q_UNREACHABLE(); - return QAbstractTestLogger::QFatal; + Q_UNREACHABLE_RETURN(QAbstractTestLogger::QFatal); }(); QString formattedMessage = qFormatLogMessage(type, context, message); @@ -357,10 +382,13 @@ void QAbstractTestLogger::addMessage(QtMsgType type, const QMessageLogContext &c addMessage(messageType, formattedMessage); } -namespace QTest +namespace { + constexpr int MAXSIZE = 1024 * 1024 * 2; +} -extern void filter_unprintable(char *str); +namespace QTest +{ /*! \fn int QTest::qt_asprintf(QTestCharBuffer *buf, const char *format, ...); @@ -368,33 +396,27 @@ extern void filter_unprintable(char *str); */ int qt_asprintf(QTestCharBuffer *str, const char *format, ...) { - static const int MAXSIZE = 1024*1024*2; - Q_ASSERT(str); - int size = str->size(); + Q_ASSERT(size > 0); va_list ap; int res = 0; - for (;;) { + do { va_start(ap, format); res = qvsnprintf(str->data(), size, format, ap); va_end(ap); - str->data()[size - 1] = '\0'; - if (res >= 0 && res < size) { - // We succeeded - break; - } - // buffer wasn't big enough, try again. + // vsnprintf() reliably '\0'-terminates + Q_ASSERT(res < 0 || str->data()[res < size ? res : size - 1] == '\0'); // Note, we're assuming that a result of -1 is always due to running out of space. - size *= 2; - if (size > MAXSIZE) { + if (res >= 0 && res < size) // Success break; - } - if (!str->reset(size)) - break; // out of memory - take what we have - } + + // Buffer wasn't big enough, try again: + size *= 2; + // If too large or out of memory, take what we have: + } while (size <= MAXSIZE && str->reset(size)); return res; } @@ -421,6 +443,28 @@ void generateTestIdentifier(QTestCharBuffer *identifier, int parts) globalDataTag, tagFiller, dataTag, testFuctionEnd); } +// strcat() for QTestCharBuffer objects: +bool appendCharBuffer(QTestCharBuffer *accumulator, const QTestCharBuffer &more) +{ + const auto bufsize = [](const QTestCharBuffer &buf) -> int { + const int max = buf.size(); + return max > 0 ? int(qstrnlen(buf.constData(), max)) : 0; + }; + const int extra = bufsize(more); + if (extra <= 0) + return true; // Nothing to do, fatuous success + + const int oldsize = bufsize(*accumulator); + const int newsize = oldsize + extra + 1; // 1 for final '\0' + if (newsize > MAXSIZE || !accumulator->resize(newsize)) + return false; // too big or unable to grow + + char *tail = accumulator->data() + oldsize; + memcpy(tail, more.constData(), extra); + tail[extra] = '\0'; + return true; +} + } QT_END_NAMESPACE |