diff options
Diffstat (limited to 'src/testlib')
-rw-r--r-- | src/testlib/doc/snippets/code/doc_src_qtestlib.cpp | 30 | ||||
-rw-r--r-- | src/testlib/doc/src/qttestlib-manual.qdoc | 12 | ||||
-rw-r--r-- | src/testlib/qabstracttestlogger.cpp | 23 | ||||
-rw-r--r-- | src/testlib/qabstracttestlogger_p.h | 8 | ||||
-rw-r--r-- | src/testlib/qappletestlogger.cpp | 157 | ||||
-rw-r--r-- | src/testlib/qappletestlogger_p.h | 95 | ||||
-rw-r--r-- | src/testlib/qplaintestlogger.cpp | 18 | ||||
-rw-r--r-- | src/testlib/qsignalspy.h | 8 | ||||
-rw-r--r-- | src/testlib/qtaptestlogger.cpp | 254 | ||||
-rw-r--r-- | src/testlib/qtaptestlogger_p.h | 85 | ||||
-rw-r--r-- | src/testlib/qtestcase.cpp | 8 | ||||
-rw-r--r-- | src/testlib/qtestcase.h | 16 | ||||
-rw-r--r-- | src/testlib/qtestlog.cpp | 33 | ||||
-rw-r--r-- | src/testlib/qtestlog_p.h | 6 | ||||
-rw-r--r-- | src/testlib/qtestresult.cpp | 2 | ||||
-rw-r--r-- | src/testlib/qtestsystem.h | 4 | ||||
-rw-r--r-- | src/testlib/qxctestlogger.mm | 10 | ||||
-rw-r--r-- | src/testlib/qxctestlogger_p.h | 2 | ||||
-rw-r--r-- | src/testlib/testlib.pro | 9 |
19 files changed, 735 insertions, 45 deletions
diff --git a/src/testlib/doc/snippets/code/doc_src_qtestlib.cpp b/src/testlib/doc/snippets/code/doc_src_qtestlib.cpp index 0dc45bef76..de301b8df9 100644 --- a/src/testlib/doc/snippets/code/doc_src_qtestlib.cpp +++ b/src/testlib/doc/snippets/code/doc_src_qtestlib.cpp @@ -48,19 +48,41 @@ ** ****************************************************************************/ +#include <QtTest> + //! [0] class MyFirstTest: public QObject { Q_OBJECT + +private: + bool myCondition() + { + return true; + } + private slots: void initTestCase() - { qDebug("called before everything else"); } + { + qDebug("Called before everything else."); + } + void myFirstTest() - { QVERIFY(1 == 1); } + { + QVERIFY(true); // check that a condition is satisfied + QCOMPARE(1, 1); // compare two values + } + void mySecondTest() - { QVERIFY(1 != 2); } + { + QVERIFY(myCondition()); + QVERIFY(1 != 2); + } + void cleanupTestCase() - { qDebug("called after myFirstTest and mySecondTest"); } + { + qDebug("Called after myFirstTest and mySecondTest."); + } }; //! [0] diff --git a/src/testlib/doc/src/qttestlib-manual.qdoc b/src/testlib/doc/src/qttestlib-manual.qdoc index 9bd210d71f..9c6bdf60a4 100644 --- a/src/testlib/doc/src/qttestlib-manual.qdoc +++ b/src/testlib/doc/src/qttestlib-manual.qdoc @@ -69,8 +69,8 @@ \li Qt Test supports benchmarking and provides several measurement back-ends. \row \li \b {IDE friendly} - \li Qt Test outputs messages that can be interpreted by Visual - Studio and KDevelop. + \li Qt Test outputs messages that can be interpreted by Qt Creator, Visual + Studio, and KDevelop. \row \li \b Thread-safety \li The error reporting is thread safe and atomic. @@ -175,7 +175,7 @@ \list \li \c -o \e{filename,format} \br Writes output to the specified file, in the specified format (one of - \c txt, \c xml, \c lightxml or \c xunitxml). The special filename \c - + \c txt, \c xml, \c lightxml, \c xunitxml or \c tap). The special filename \c - may be used to log to standard output. \li \c -o \e filename \br Writes output to the specified file. @@ -192,6 +192,8 @@ benchmarks, since it suppresses normal pass/fail messages. \li \c -teamcity \br Outputs results in TeamCity format. + \li \c -tap \br + Outputs results in Test Anything Protocol (TAP) format. \endlist The first version of the \c -o option may be repeated in order to log @@ -199,8 +201,8 @@ option can log test results to standard output. If the first version of the \c -o option is used, neither the second version - of the \c -o option nor the \c -txt, \c -xml, \c -lightxml, \c -teamcity - or \c -xunitxml options should be used. + of the \c -o option nor the \c -txt, \c -xml, \c -lightxml, \c -teamcity, + \c -xunitxml or \c -tap options should be used. If neither version of the \c -o option is used, test results will be logged to standard output. If no format option is used, test results will be logged in diff --git a/src/testlib/qabstracttestlogger.cpp b/src/testlib/qabstracttestlogger.cpp index 63c165b9dc..2b54cd410b 100644 --- a/src/testlib/qabstracttestlogger.cpp +++ b/src/testlib/qabstracttestlogger.cpp @@ -39,6 +39,7 @@ #include <QtTest/private/qabstracttestlogger_p.h> #include <QtTest/qtestassert.h> +#include <qtestresult_p.h> #include <QtCore/qbytearray.h> #include <QtCore/qstring.h> @@ -189,4 +190,26 @@ int qt_asprintf(QTestCharBuffer *str, const char *format, ...) } +namespace QTestPrivate +{ + +void generateTestIdentifier(QTestCharBuffer *identifier, int parts) +{ + const char *testObject = parts & TestObject ? QTestResult::currentTestObjectName() : ""; + const char *testFunction = parts & TestFunction ? (QTestResult::currentTestFunction() ? QTestResult::currentTestFunction() : "UnknownTestFunc") : ""; + const char *objectFunctionFiller = parts & TestObject && parts & (TestFunction | TestDataTag) ? "::" : ""; + const char *testFuctionStart = parts & TestFunction ? "(" : ""; + const char *testFuctionEnd = parts & TestFunction ? ")" : ""; + + const char *dataTag = (parts & TestDataTag) && QTestResult::currentDataTag() ? QTestResult::currentDataTag() : ""; + const char *globalDataTag = (parts & TestDataTag) && QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : ""; + const char *tagFiller = (dataTag[0] && globalDataTag[0]) ? ":" : ""; + + QTest::qt_asprintf(identifier, "%s%s%s%s%s%s%s%s", + testObject, objectFunctionFiller, testFunction, testFuctionStart, + globalDataTag, tagFiller, dataTag, testFuctionEnd); +} + +} + QT_END_NAMESPACE diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h index c4083b43f4..018361b81e 100644 --- a/src/testlib/qabstracttestlogger_p.h +++ b/src/testlib/qabstracttestlogger_p.h @@ -58,6 +58,7 @@ QT_BEGIN_NAMESPACE class QBenchmarkResult; +class QTestData; class QAbstractTestLogger { @@ -91,6 +92,8 @@ public: virtual void enterTestFunction(const char *function) = 0; virtual void leaveTestFunction() = 0; + virtual void enterTestData(QTestData *) {} + virtual void addIncident(IncidentTypes type, const char *description, const char *file = 0, int line = 0) = 0; virtual void addBenchmarkResult(const QBenchmarkResult &result) = 0; @@ -175,6 +178,11 @@ namespace QTest int qt_asprintf(QTestCharBuffer *buf, const char *format, ...); } +namespace QTestPrivate +{ + enum IdentifierPart { TestObject = 0x1, TestFunction = 0x2, TestDataTag = 0x4, AllParts = 0xFFFF }; + void generateTestIdentifier(QTestCharBuffer *identifier, int parts = AllParts); +} QT_END_NAMESPACE diff --git a/src/testlib/qappletestlogger.cpp b/src/testlib/qappletestlogger.cpp new file mode 100644 index 0000000000..e970f008d2 --- /dev/null +++ b/src/testlib/qappletestlogger.cpp @@ -0,0 +1,157 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qappletestlogger_p.h" + +#include <QPair> + +QT_BEGIN_NAMESPACE + +#if defined(QT_USE_APPLE_UNIFIED_LOGGING) + +using namespace QTestPrivate; + +bool QAppleTestLogger::debugLoggingEnabled() +{ + // Debug-level messages are only captured in memory when debug logging is + // enabled through a configuration change, which can happen automatically + // when running inside Xcode, or with the Console application open. + if (__builtin_available(macOS 10.12, iOS 10, tvOS 10, watchOS 3, *)) + return os_log_type_enabled(OS_LOG_DEFAULT, OS_LOG_TYPE_DEBUG); + + return false; +} + +QAppleTestLogger::QAppleTestLogger(QAbstractTestLogger *logger) + : QAbstractTestLogger(nullptr) + , m_logger(logger) +{ +} + +static QAppleLogActivity testFunctionActivity; + +void QAppleTestLogger::enterTestFunction(const char *function) +{ + // Re-create activity each time + testFunctionActivity = QT_APPLE_LOG_ACTIVITY("Running test function").enter(); + + if (__builtin_available(macOS 10.12, iOS 10, tvOS 10, watchOS 3, *)) { + QTestCharBuffer testIdentifier; + QTestPrivate::generateTestIdentifier(&testIdentifier); + QString identifier = QString::fromLatin1(testIdentifier.data()); + QMessageLogContext context(nullptr, 0, nullptr, "qt.test.enter"); + QString message = identifier; + if (AppleUnifiedLogger::messageHandler(QtDebugMsg, context, message, identifier)) + return; // AUL already printed to stderr + } + + m_logger->enterTestFunction(function); +} + +void QAppleTestLogger::leaveTestFunction() +{ + m_logger->leaveTestFunction(); + testFunctionActivity.leave(); +} + +typedef QPair<QtMsgType, const char *> IncidentClassification; +static IncidentClassification incidentTypeToClassification(QAbstractTestLogger::IncidentTypes type) +{ + switch (type) { + case QAbstractTestLogger::Pass: + return IncidentClassification(QtInfoMsg, "pass"); + case QAbstractTestLogger::XFail: + return IncidentClassification(QtInfoMsg, "xfail"); + case QAbstractTestLogger::Fail: + return IncidentClassification(QtCriticalMsg, "fail"); + case QAbstractTestLogger::XPass: + return IncidentClassification(QtInfoMsg, "xpass"); + case QAbstractTestLogger::BlacklistedPass: + return IncidentClassification(QtWarningMsg, "bpass"); + case QAbstractTestLogger::BlacklistedFail: + return IncidentClassification(QtInfoMsg, "bfail"); + } + return IncidentClassification(QtFatalMsg, nullptr); +} + +void QAppleTestLogger::addIncident(IncidentTypes type, const char *description, + const char *file, int line) +{ + if (__builtin_available(macOS 10.12, iOS 10, tvOS 10, watchOS 3, *)) { + IncidentClassification incidentClassification = incidentTypeToClassification(type); + + QTestCharBuffer category; + QTest::qt_asprintf(&category, "qt.test.%s", incidentClassification.second); + QMessageLogContext context(file, line, /* function = */ nullptr, category.data()); + + QTestCharBuffer subsystemBuffer; + // It would be nice to have the data tag as part of the subsystem too, but that + // will for some tests results in hundreds of thousands of log objects being + // created, so we limit the subsystem to test functions, which we can hope + // are reasonably limited. + generateTestIdentifier(&subsystemBuffer, TestObject | TestFunction); + QString subsystem = QString::fromLatin1(subsystemBuffer.data()); + + // We still want the full identifier as part of the message though + QTestCharBuffer testIdentifier; + generateTestIdentifier(&testIdentifier); + QString message = QString::fromLatin1(testIdentifier.data()); + if (qstrlen(description)) + message += QLatin1Char('\n') % QString::fromLatin1(description); + + if (AppleUnifiedLogger::messageHandler(incidentClassification.first, context, message, subsystem)) + return; // AUL already printed to stderr + } + + m_logger->addIncident(type, description, file, line); +} + +void QAppleTestLogger::addMessage(QtMsgType type, const QMessageLogContext &context, const QString &message) +{ + if (__builtin_available(macOS 10.12, iOS 10, tvOS 10, watchOS 3, *)) { + if (AppleUnifiedLogger::messageHandler(type, context, message)) + return; // AUL already printed to stderr + } + + m_logger->addMessage(type, context, message); +} + +#endif // QT_USE_APPLE_UNIFIED_LOGGING + +QT_END_NAMESPACE diff --git a/src/testlib/qappletestlogger_p.h b/src/testlib/qappletestlogger_p.h new file mode 100644 index 0000000000..5a45fad7a0 --- /dev/null +++ b/src/testlib/qappletestlogger_p.h @@ -0,0 +1,95 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QAPPLETESTLOGGER_P_H +#define QAPPLETESTLOGGER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtTest/private/qabstracttestlogger_p.h> + +#include <QtCore/private/qcore_mac_p.h> + +QT_BEGIN_NAMESPACE + +#if defined(QT_USE_APPLE_UNIFIED_LOGGING) +class QAppleTestLogger : public QAbstractTestLogger +{ +public: + static bool debugLoggingEnabled(); + + QAppleTestLogger(QAbstractTestLogger *logger); + + void startLogging() override + { m_logger->startLogging(); } + void stopLogging() override + { m_logger->stopLogging(); } + + void enterTestFunction(const char *function) override; + void leaveTestFunction() override; + + void addIncident(IncidentTypes type, const char *description, + const char *file = 0, int line = 0) override; + void addMessage(QtMsgType, const QMessageLogContext &, + const QString &) override; + + void addBenchmarkResult(const QBenchmarkResult &result) override + { m_logger->addBenchmarkResult(result); } + + void addMessage(MessageTypes type, const QString &message, + const char *file = 0, int line = 0) override + { m_logger->addMessage(type, message, file, line); } + +private: + QScopedPointer<QAbstractTestLogger> m_logger; +}; +#endif + +QT_END_NAMESPACE + +#endif // QAPPLETESTLOGGER_P_H diff --git a/src/testlib/qplaintestlogger.cpp b/src/testlib/qplaintestlogger.cpp index e5226b7820..853515f2d9 100644 --- a/src/testlib/qplaintestlogger.cpp +++ b/src/testlib/qplaintestlogger.cpp @@ -221,18 +221,6 @@ void QPlainTestLogger::outputMessage(const char *str) outputString(str); } -static void testIdentifier(QTestCharBuffer *identifier) -{ - const char *testObject = QTestResult::currentTestObjectName(); - const char *testFunction = QTestResult::currentTestFunction() ? QTestResult::currentTestFunction() : "UnknownTestFunc"; - - const char *dataTag = QTestResult::currentDataTag() ? QTestResult::currentDataTag() : ""; - const char *globalDataTag = QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : ""; - const char *tagFiller = (dataTag[0] && globalDataTag[0]) ? ":" : ""; - - QTest::qt_asprintf(identifier, "%s::%s(%s%s%s)", testObject, testFunction, globalDataTag, tagFiller, dataTag); -} - void QPlainTestLogger::printMessage(const char *type, const char *msg, const char *file, int line) { QTEST_ASSERT(type); @@ -251,10 +239,10 @@ void QPlainTestLogger::printMessage(const char *type, const char *msg, const cha } const char *msgFiller = msg[0] ? " " : ""; - QTestCharBuffer testIdent; - testIdentifier(&testIdent); + QTestCharBuffer testIdentifier; + QTestPrivate::generateTestIdentifier(&testIdentifier); QTest::qt_asprintf(&messagePrefix, "%s: %s%s%s%s\n", - type, testIdent.data(), msgFiller, msg, failureLocation.data()); + type, testIdentifier.data(), msgFiller, msg, failureLocation.data()); // In colored mode, printf above stripped our nonprintable control characters. // Put them back. diff --git a/src/testlib/qsignalspy.h b/src/testlib/qsignalspy.h index 824dc6aaed..218a26ec5c 100644 --- a/src/testlib/qsignalspy.h +++ b/src/testlib/qsignalspy.h @@ -59,11 +59,7 @@ public: explicit QSignalSpy(const QObject *obj, const char *aSignal) : m_waiting(false) { -#ifdef Q_CC_BOR - const int memberOffset = QObject::staticMetaObject.methodCount(); -#else static const int memberOffset = QObject::staticMetaObject.methodCount(); -#endif if (!obj) { qWarning("QSignalSpy: Cannot spy on a null object"); return; @@ -104,11 +100,7 @@ public: QSignalSpy(const typename QtPrivate::FunctionPointer<Func>::Object *obj, Func signal0) : m_waiting(false) { -#ifdef Q_CC_BOR - const int memberOffset = QObject::staticMetaObject.methodCount(); -#else static const int memberOffset = QObject::staticMetaObject.methodCount(); -#endif if (!obj) { qWarning("QSignalSpy: Cannot spy on a null object"); return; diff --git a/src/testlib/qtaptestlogger.cpp b/src/testlib/qtaptestlogger.cpp new file mode 100644 index 0000000000..37ab89ac91 --- /dev/null +++ b/src/testlib/qtaptestlogger.cpp @@ -0,0 +1,254 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qtaptestlogger_p.h" + +#include "qtestlog_p.h" +#include "qtestresult_p.h" +#include "qtestassert.h" + +#include <QtCore/qregularexpression.h> + +QT_BEGIN_NAMESPACE + +QTapTestLogger::QTapTestLogger(const char *filename) + : QAbstractTestLogger(filename) + , m_wasExpectedFail(false) +{ +} + +QTapTestLogger::~QTapTestLogger() +{ +} + +void QTapTestLogger::startLogging() +{ + QAbstractTestLogger::startLogging(); + + QTestCharBuffer preamble; + QTest::qt_asprintf(&preamble, "TAP version 13\n" + // By convention, test suite names are output as diagnostics lines + // This is a pretty poor convention, as consumers will then treat + // actual diagnostics, e.g. qDebug, as test suite names o_O + "# %s\n", QTestResult::currentTestObjectName()); + outputString(preamble.data()); +} + +void QTapTestLogger::stopLogging() +{ + const int total = QTestLog::totalCount(); + + QTestCharBuffer testPlanAndStats; + QTest::qt_asprintf(&testPlanAndStats, + "1..%d\n" + "# tests %d\n" + "# pass %d\n" + "# fail %d\n", + total, total, QTestLog::passCount(), QTestLog::failCount()); + outputString(testPlanAndStats.data()); + + QAbstractTestLogger::stopLogging(); +} + +void QTapTestLogger::enterTestFunction(const char *function) +{ + Q_UNUSED(function); + m_wasExpectedFail = false; +} + +void QTapTestLogger::enterTestData(QTestData *data) +{ + Q_UNUSED(data); + m_wasExpectedFail = false; +} + +using namespace QTestPrivate; + +void QTapTestLogger::outputTestLine(bool ok, int testNumber, QTestCharBuffer &directive) +{ + QTestCharBuffer testIdentifier; + QTestPrivate::generateTestIdentifier(&testIdentifier, TestFunction | TestDataTag); + + QTestCharBuffer testLine; + QTest::qt_asprintf(&testLine, "%s %d - %s%s\n", + ok ? "ok" : "not ok", testNumber, testIdentifier.data(), directive.data()); + + outputString(testLine.data()); +} + +void QTapTestLogger::addIncident(IncidentTypes type, const char *description, + const char *file, int line) +{ + if (m_wasExpectedFail && type == Pass) { + // XFail comes with a corresponding Pass incident, but we only want + // to emit a single test point for it, so skip the this pass. + return; + } + + bool ok = type == Pass || type == XPass || type == BlacklistedPass; + + QTestCharBuffer directive; + if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass) + // We treat expected or blacklisted failures/passes as TODO-failures/passes, + // which should be treated as soft issues by consumers. Not all do though :/ + QTest::qt_asprintf(&directive, " # TODO %s", description); + + int testNumber = QTestLog::totalCount(); + if (type == XFail) { + // The global test counter hasn't been updated yet for XFail + testNumber += 1; + } + + outputTestLine(ok, testNumber, directive); + + if (!ok) { + // All failures need a diagnostics sections to not confuse consumers + + // The indent needs to be two spaces for maximum compatibility + #define YAML_INDENT " " + + outputString(YAML_INDENT "---\n"); + + if (type != XFail) { + // This is fragile, but unfortunately testlib doesn't plumb + // the expected and actual values to the loggers (yet). + static QRegularExpression verifyRegex( + QLatin1Literal("^'(?<actualexpression>.*)' returned (?<actual>\\w+).+\\((?<message>.*)\\)$")); + + static QRegularExpression comparRegex( + QLatin1Literal("^(?<message>.*)\n" + "\\s*Actual\\s+\\((?<actualexpression>.*)\\)\\s*: (?<actual>.*)\n" + "\\s*Expected\\s+\\((?<expectedexpresssion>.*)\\)\\s*: (?<expected>.*)$")); + + QString descriptionString = QString::fromUtf8(description); + QRegularExpressionMatch match = verifyRegex.match(descriptionString); + if (!match.hasMatch()) + match = comparRegex.match(descriptionString); + + if (match.hasMatch()) { + bool isVerify = match.regularExpression() == verifyRegex; + QString message = match.captured(QLatin1Literal("message")); + QString expected; + QString actual; + + if (isVerify) { + QString expression = QLatin1Literal(" (") + % match.captured(QLatin1Literal("actualexpression")) % QLatin1Char(')') ; + actual = match.captured(QLatin1Literal("actual")).toLower() % expression; + expected = (actual.startsWith(QLatin1Literal("true")) ? QLatin1Literal("false") : QLatin1Literal("true")) % expression; + if (message.isEmpty()) + message = QLatin1Literal("Verification failed"); + } else { + expected = match.captured(QLatin1Literal("expected")) + % QLatin1Literal(" (") % match.captured(QLatin1Literal("expectedexpresssion")) % QLatin1Char(')'); + actual = match.captured(QLatin1Literal("actual")) + % QLatin1Literal(" (") % match.captured(QLatin1Literal("actualexpression")) % QLatin1Char(')'); + } + + QTestCharBuffer diagnosticsYamlish; + QTest::qt_asprintf(&diagnosticsYamlish, + YAML_INDENT "type: %s\n" + YAML_INDENT "message: %s\n" + + // Some consumers understand 'wanted/found', while others need + // 'expected/actual', so we do both for maximum compatibility. + YAML_INDENT "wanted: %s\n" + YAML_INDENT "found: %s\n" + YAML_INDENT "expected: %s\n" + YAML_INDENT "actual: %s\n", + + isVerify ? "QVERIFY" : "QCOMPARE", + qPrintable(message), + qPrintable(expected), qPrintable(actual), + qPrintable(expected), qPrintable(actual) + ); + + outputString(diagnosticsYamlish.data()); + } else { + QTestCharBuffer unparsableDescription; + QTest::qt_asprintf(&unparsableDescription, + YAML_INDENT "# %s\n", description); + outputString(unparsableDescription.data()); + } + } + + if (file) { + QTestCharBuffer location; + QTest::qt_asprintf(&location, + // The generic 'at' key is understood by most consumers. + YAML_INDENT "at: %s::%s() (%s:%d)\n" + + // The file and line keys are for consumers that are able + // to read more granular location info. + YAML_INDENT "file: %s\n" + YAML_INDENT "line: %d\n", + + QTestResult::currentTestObjectName(), + QTestResult::currentTestFunction(), + file, line, file, line + ); + outputString(location.data()); + } + + outputString(YAML_INDENT "...\n"); + } + + m_wasExpectedFail = type == XFail; +} + +void QTapTestLogger::addMessage(MessageTypes type, const QString &message, + const char *file, int line) +{ + Q_UNUSED(file); + Q_UNUSED(line); + + if (type == Skip) { + QTestCharBuffer directive; + QTest::qt_asprintf(&directive, " # SKIP %s", message.toUtf8().constData()); + outputTestLine(/* ok = */ true, QTestLog::totalCount(), directive); + return; + } + + QTestCharBuffer diagnostics; + QTest::qt_asprintf(&diagnostics, "# %s\n", qPrintable(message)); + outputString(diagnostics.data()); +} + +QT_END_NAMESPACE + diff --git a/src/testlib/qtaptestlogger_p.h b/src/testlib/qtaptestlogger_p.h new file mode 100644 index 0000000000..b51343e4fe --- /dev/null +++ b/src/testlib/qtaptestlogger_p.h @@ -0,0 +1,85 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QTAPTESTLOGGER_P_H +#define QTAPTESTLOGGER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtTest/private/qabstracttestlogger_p.h> + +QT_BEGIN_NAMESPACE + +class QTapTestLogger : public QAbstractTestLogger +{ +public: + QTapTestLogger(const char *filename); + ~QTapTestLogger(); + + void startLogging() override; + void stopLogging() override; + + void enterTestFunction(const char *) override; + void leaveTestFunction() override {} + + void enterTestData(QTestData *data) override; + + void addIncident(IncidentTypes type, const char *description, + const char *file = 0, int line = 0) override; + void addMessage(MessageTypes type, const QString &message, + const char *file = 0, int line = 0) override; + + void addBenchmarkResult(const QBenchmarkResult &) override {}; +private: + void outputTestLine(bool ok, int testNumber, QTestCharBuffer &directive); + bool m_wasExpectedFail; +}; + +QT_END_NAMESPACE + +#endif // QTAPTESTLOGGER_P_H diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index adf4b9e1ef..0866176b6b 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -526,6 +526,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) " xml : XML document\n" " lightxml : A stream of XML tags\n" " teamcity : TeamCity format\n" + " tap : Test Anything Protocol\n" "\n" " *** Multiple loggers can be specified, but at most one can log to stdout.\n" "\n" @@ -537,6 +538,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) " -xml : Output results as XML document\n" " -lightxml : Output results as stream of XML tags\n" " -teamcity : Output results in TeamCity format\n" + " -tap : Output results in Test Anything Protocol format\n" "\n" " *** If no output file is specified, stdout is assumed.\n" " *** If no output format is specified, -txt is assumed.\n" @@ -624,6 +626,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) logFormat = QTestLog::LightXML; } else if (strcmp(argv[i], "-teamcity") == 0) { logFormat = QTestLog::TeamCity; + } else if (strcmp(argv[i], "-tap") == 0) { + logFormat = QTestLog::TAP; } else if (strcmp(argv[i], "-silent") == 0) { QTestLog::setVerboseLevel(-1); } else if (strcmp(argv[i], "-v1") == 0) { @@ -658,8 +662,10 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) logFormat = QTestLog::XunitXML; else if (strcmp(format, "teamcity") == 0) logFormat = QTestLog::TeamCity; + else if (strcmp(format, "tap") == 0) + logFormat = QTestLog::TAP; else { - fprintf(stderr, "output format must be one of txt, csv, lightxml, xml, teamcity or xunitxml\n"); + fprintf(stderr, "output format must be one of txt, csv, lightxml, xml, tap, teamcity or xunitxml\n"); exit(1); } if (strcmp(filename, "-") == 0 && QTestLog::loggerUsingStdout()) { diff --git a/src/testlib/qtestcase.h b/src/testlib/qtestcase.h index 54669c11de..4bf816a850 100644 --- a/src/testlib/qtestcase.h +++ b/src/testlib/qtestcase.h @@ -253,6 +253,22 @@ namespace QTest return nullptr; } + template<typename F> // Output QFlags of registered enumerations + inline typename std::enable_if<QtPrivate::IsQEnumHelper<F>::Value, char*>::type toString(QFlags<F> f) + { + const QMetaEnum me = QMetaEnum::fromType<F>(); + return qstrdup(me.valueToKeys(int(f)).constData()); + } + + template <typename F> // Fallback: Output hex value + inline typename std::enable_if<!QtPrivate::IsQEnumHelper<F>::Value, char*>::type toString(QFlags<F> f) + { + const size_t space = 3 + 2 * sizeof(unsigned); // 2 for 0x, two hex digits per byte, 1 for '\0' + char *msg = new char[space]; + qsnprintf(msg, space, "0x%x", unsigned(f)); + return msg; + } + } // namespace Internal template<typename T> diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp index 6260b9e3fd..1268730cc6 100644 --- a/src/testlib/qtestlog.cpp +++ b/src/testlib/qtestlog.cpp @@ -47,10 +47,15 @@ #include <QtTest/private/qxunittestlogger_p.h> #include <QtTest/private/qxmltestlogger_p.h> #include <QtTest/private/qteamcitylogger_p.h> +#include <QtTest/private/qtaptestlogger_p.h> #if defined(HAVE_XCTEST) #include <QtTest/private/qxctestlogger_p.h> #endif +#if defined(Q_OS_DARWIN) +#include <QtTest/private/qappletestlogger_p.h> +#endif + #include <QtCore/qatomic.h> #include <QtCore/qbytearray.h> #include <QtCore/QElapsedTimer> @@ -211,6 +216,11 @@ namespace QTest { FOREACH_LOGGER(logger->leaveTestFunction()); } + static void enterTestData(QTestData *data) + { + FOREACH_LOGGER(logger->enterTestData(data)); + } + static void addIncident(QAbstractTestLogger::IncidentTypes type, const char *description, const char *file = 0, int line = 0) { @@ -339,6 +349,12 @@ void QTestLog::enterTestFunction(const char* function) QTest::TestLoggers::enterTestFunction(function); } +void QTestLog::enterTestData(QTestData *data) +{ + QTEST_ASSERT(data); + QTest::TestLoggers::enterTestData(data); +} + int QTestLog::unhandledIgnoreMessages() { int i = 0; @@ -498,12 +514,24 @@ void QTestLog::addLogger(LogMode mode, const char *filename) case QTestLog::TeamCity: logger = new QTeamCityLogger(filename); break; + case QTestLog::TAP: + logger = new QTapTestLogger(filename); + break; #if defined(HAVE_XCTEST) case QTestLog::XCTest: logger = new QXcodeTestLogger; break; #endif } + +#if defined(QT_USE_APPLE_UNIFIED_LOGGING) + // Logger that also feeds messages to AUL. It needs to wrap the existing + // logger, as it needs to be able to short circuit the existing logger + // in case AUL prints to stderr. + if (QAppleTestLogger::debugLoggingEnabled()) + logger = new QAppleTestLogger(logger); +#endif + QTEST_ASSERT(logger); QTest::TestLoggers::addLogger(logger); } @@ -591,6 +619,11 @@ int QTestLog::blacklistCount() return QTest::blacklists; } +int QTestLog::totalCount() +{ + return passCount() + failCount() + skipCount() + blacklistCount(); +} + void QTestLog::resetCounters() { QTest::passes = 0; diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h index 3d28795188..600c078ce2 100644 --- a/src/testlib/qtestlog_p.h +++ b/src/testlib/qtestlog_p.h @@ -57,12 +57,13 @@ QT_BEGIN_NAMESPACE class QBenchmarkResult; class QRegularExpression; +class QTestData; class Q_TESTLIB_EXPORT QTestLog { public: enum LogMode { - Plain = 0, XML, LightXML, XunitXML, CSV, TeamCity, + Plain = 0, XML, LightXML, XunitXML, CSV, TeamCity, TAP, #if defined(HAVE_XCTEST) XCTest #endif @@ -71,6 +72,8 @@ public: static void enterTestFunction(const char* function); static void leaveTestFunction(); + static void enterTestData(QTestData *data); + static void addPass(const char *msg); static void addFail(const char *msg, const char *file, int line); static void addXFail(const char *msg, const char *file, int line); @@ -110,6 +113,7 @@ public: static int failCount(); static int skipCount(); static int blacklistCount(); + static int totalCount(); static void resetCounters(); diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp index 219190d5da..88e3407c90 100644 --- a/src/testlib/qtestresult.cpp +++ b/src/testlib/qtestresult.cpp @@ -110,6 +110,8 @@ void QTestResult::setCurrentTestData(QTestData *data) { QTest::currentTestData = data; QTest::failed = false; + if (data) + QTestLog::enterTestData(data); } void QTestResult::setCurrentTestFunction(const char *func) diff --git a/src/testlib/qtestsystem.h b/src/testlib/qtestsystem.h index daa0d7aea0..df62b392d4 100644 --- a/src/testlib/qtestsystem.h +++ b/src/testlib/qtestsystem.h @@ -74,10 +74,8 @@ namespace QTest QCoreApplication::sendPostedEvents(nullptr, QEvent::DeferredDelete); remaining = deadline.remainingTime(); - if (remaining > 0) { + if (remaining > 0) QTest::qSleep(qMin(10, remaining)); - remaining = deadline.remainingTime(); - } if (predicate()) return true; diff --git a/src/testlib/qxctestlogger.mm b/src/testlib/qxctestlogger.mm index 62fd73070f..9fa9da2fdd 100644 --- a/src/testlib/qxctestlogger.mm +++ b/src/testlib/qxctestlogger.mm @@ -200,7 +200,7 @@ private: [autoreleasepool release]; } -+ (id)defaultTestSuite ++ (QTestLibTests *)defaultTestSuite { return [[QtTestLibTests alloc] initWithName:@"QtTestLib"]; } @@ -255,7 +255,7 @@ static XCTestSuiteRun *s_qtTestSuiteRun = 0; @implementation QtTestLibTest -- (id)initWithInvocation:(NSInvocation *)invocation +- (instancetype)initWithInvocation:(NSInvocation *)invocation { if (self = [super initWithInvocation:invocation]) { // The test object name and function name are used by XCTest after QtTestLib has @@ -322,7 +322,7 @@ QXcodeTestLogger *QXcodeTestLogger::s_currentTestLogger = 0; QXcodeTestLogger::QXcodeTestLogger() : QAbstractTestLogger(0) - , m_testRuns([[NSMutableArray arrayWithCapacity:2] retain]) + , m_testRuns([[NSMutableArray<XCTestRun *> arrayWithCapacity:2] retain]) { Q_ASSERT(!s_currentTestLogger); @@ -383,11 +383,11 @@ static bool isTestFunctionInActiveScope(const char *function) Q_ASSERT(activeScope == Selected); - static NSArray *forcedTests = [@[ @"initTestCase", @"initTestCase_data", @"cleanupTestCase" ] retain]; + static NSArray<NSString *> *forcedTests = [@[ @"initTestCase", @"initTestCase_data", @"cleanupTestCase" ] retain]; if ([forcedTests containsObject:[NSString stringWithUTF8String:function]]) return true; - static NSArray *testsInScope = [[testScope componentsSeparatedByString:@","] retain]; + static NSArray<NSString *> *testsInScope = [[testScope componentsSeparatedByString:@","] retain]; bool inScope = [testsInScope containsObject:[NSString stringWithFormat:@"%s/%s", QTestResult::currentTestObjectName(), function]]; diff --git a/src/testlib/qxctestlogger_p.h b/src/testlib/qxctestlogger_p.h index 1b641f18af..8baa5aa27f 100644 --- a/src/testlib/qxctestlogger_p.h +++ b/src/testlib/qxctestlogger_p.h @@ -90,7 +90,7 @@ private: void pushTestRunForTest(XCTest *test, bool start); XCTestRun *popTestRun(); - NSMutableArray *m_testRuns; + NSMutableArray<XCTestRun *> *m_testRuns; static QXcodeTestLogger *s_currentTestLogger; }; diff --git a/src/testlib/testlib.pro b/src/testlib/testlib.pro index 06a6f8b8e5..46b61dac07 100644 --- a/src/testlib/testlib.pro +++ b/src/testlib/testlib.pro @@ -39,7 +39,8 @@ HEADERS = \ qtesttouch.h \ qtestblacklist_p.h \ qtesthelpers_p.h \ - qttestglobal.h + qttestglobal.h \ + qtaptestlogger_p.h SOURCES = \ qtestcase.cpp \ @@ -65,7 +66,8 @@ SOURCES = \ qtestmouse.cpp \ qtestxunitstreamer.cpp \ qxunittestlogger.cpp \ - qtestblacklist.cpp + qtestblacklist.cpp \ + qtaptestlogger.cpp qtConfig(itemmodeltester) { HEADERS += \ @@ -84,6 +86,9 @@ embedded:QMAKE_CXXFLAGS += -fno-rtti mac { LIBS += -framework Security + SOURCES += qappletestlogger.cpp + HEADERS += qappletestlogger_p.h + macos { HEADERS += qtestutil_macos_p.h OBJECTIVE_SOURCES += qtestutil_macos.mm |