diff options
Diffstat (limited to 'src/testlib')
-rw-r--r-- | src/testlib/doc/snippets/code/doc_src_qtestlib.cpp | 30 | ||||
-rw-r--r-- | src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp | 20 | ||||
-rw-r--r-- | src/testlib/doc/src/qttestlib-manual.qdoc | 12 | ||||
-rw-r--r-- | src/testlib/qabstracttestlogger.cpp | 23 | ||||
-rw-r--r-- | src/testlib/qabstracttestlogger_p.h | 8 | ||||
-rw-r--r-- | src/testlib/qappletestlogger.cpp | 149 | ||||
-rw-r--r-- | src/testlib/qappletestlogger_p.h | 95 | ||||
-rw-r--r-- | src/testlib/qplaintestlogger.cpp | 18 | ||||
-rw-r--r-- | src/testlib/qsignalspy.h | 8 | ||||
-rw-r--r-- | src/testlib/qtaptestlogger.cpp | 254 | ||||
-rw-r--r-- | src/testlib/qtaptestlogger_p.h | 85 | ||||
-rw-r--r-- | src/testlib/qtest.h | 21 | ||||
-rw-r--r-- | src/testlib/qtestcase.cpp | 73 | ||||
-rw-r--r-- | src/testlib/qtestcase.h | 21 | ||||
-rw-r--r-- | src/testlib/qtestcase.qdoc | 99 | ||||
-rw-r--r-- | src/testlib/qtesteventloop.h | 5 | ||||
-rw-r--r-- | src/testlib/qtestlog.cpp | 33 | ||||
-rw-r--r-- | src/testlib/qtestlog_p.h | 6 | ||||
-rw-r--r-- | src/testlib/qtestresult.cpp | 2 | ||||
-rw-r--r-- | src/testlib/qtestsystem.h | 105 | ||||
-rw-r--r-- | src/testlib/qxctestlogger.mm | 10 | ||||
-rw-r--r-- | src/testlib/qxctestlogger_p.h | 2 | ||||
-rw-r--r-- | src/testlib/testlib.pro | 9 |
23 files changed, 810 insertions, 278 deletions
diff --git a/src/testlib/doc/snippets/code/doc_src_qtestlib.cpp b/src/testlib/doc/snippets/code/doc_src_qtestlib.cpp index 0dc45bef76..de301b8df9 100644 --- a/src/testlib/doc/snippets/code/doc_src_qtestlib.cpp +++ b/src/testlib/doc/snippets/code/doc_src_qtestlib.cpp @@ -48,19 +48,41 @@ ** ****************************************************************************/ +#include <QtTest> + //! [0] class MyFirstTest: public QObject { Q_OBJECT + +private: + bool myCondition() + { + return true; + } + private slots: void initTestCase() - { qDebug("called before everything else"); } + { + qDebug("Called before everything else."); + } + void myFirstTest() - { QVERIFY(1 == 1); } + { + QVERIFY(true); // check that a condition is satisfied + QCOMPARE(1, 1); // compare two values + } + void mySecondTest() - { QVERIFY(1 != 2); } + { + QVERIFY(myCondition()); + QVERIFY(1 != 2); + } + void cleanupTestCase() - { qDebug("called after myFirstTest and mySecondTest"); } + { + qDebug("Called after myFirstTest and mySecondTest."); + } }; //! [0] diff --git a/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp b/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp index 990b7a38d7..202f87af52 100644 --- a/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp +++ b/src/testlib/doc/snippets/code/src_qtestlib_qtestcase.cpp @@ -177,13 +177,6 @@ namespace MyNamespace { } //! [toString-overload] -//! [17] -int i = 0; -while (myNetworkServerNotResponding() && i++ < 50) - QTest::qWait(250); -//! [17] - - //! [18] MyTestObject test1; QTest::qExec(&test1); @@ -245,11 +238,6 @@ void MyTestClass::cleanup() QTest::qSleep(250); //! [23] -//! [24] -QWidget widget; -widget.show(); -QTest::qWaitForWindowShown(&widget); -//! [24] //! [25] QTouchDevice *dev = QTest::createTouchDevice(); @@ -306,13 +294,5 @@ QTest::keyClick(myWindow, Qt::Key_Escape); QTest::keyClick(myWindow, Qt::Key_Escape, Qt::ShiftModifier, 200); //! [29] -//! [30] -MyObject obj; -obj.startup(); -QTest::qWaitFor([&]() { - return obj.isReady(); -}, 3000); -//! [30] - } diff --git a/src/testlib/doc/src/qttestlib-manual.qdoc b/src/testlib/doc/src/qttestlib-manual.qdoc index 363ec17c6c..71b4541313 100644 --- a/src/testlib/doc/src/qttestlib-manual.qdoc +++ b/src/testlib/doc/src/qttestlib-manual.qdoc @@ -69,8 +69,8 @@ \li Qt Test supports benchmarking and provides several measurement back-ends. \row \li \b {IDE friendly} - \li Qt Test outputs messages that can be interpreted by Visual - Studio and KDevelop. + \li Qt Test outputs messages that can be interpreted by Qt Creator, Visual + Studio, and KDevelop. \row \li \b Thread-safety \li The error reporting is thread safe and atomic. @@ -177,7 +177,7 @@ \list \li \c -o \e{filename,format} \br Writes output to the specified file, in the specified format (one of - \c txt, \c xml, \c lightxml or \c xunitxml). The special filename \c - + \c txt, \c xml, \c lightxml, \c xunitxml or \c tap). The special filename \c - may be used to log to standard output. \li \c -o \e filename \br Writes output to the specified file. @@ -194,6 +194,8 @@ benchmarks, since it suppresses normal pass/fail messages. \li \c -teamcity \br Outputs results in TeamCity format. + \li \c -tap \br + Outputs results in Test Anything Protocol (TAP) format. \endlist The first version of the \c -o option may be repeated in order to log @@ -201,8 +203,8 @@ option can log test results to standard output. If the first version of the \c -o option is used, neither the second version - of the \c -o option nor the \c -txt, \c -xml, \c -lightxml, \c -teamcity - or \c -xunitxml options should be used. + of the \c -o option nor the \c -txt, \c -xml, \c -lightxml, \c -teamcity, + \c -xunitxml or \c -tap options should be used. If neither version of the \c -o option is used, test results will be logged to standard output. If no format option is used, test results will be logged in diff --git a/src/testlib/qabstracttestlogger.cpp b/src/testlib/qabstracttestlogger.cpp index 63c165b9dc..2b54cd410b 100644 --- a/src/testlib/qabstracttestlogger.cpp +++ b/src/testlib/qabstracttestlogger.cpp @@ -39,6 +39,7 @@ #include <QtTest/private/qabstracttestlogger_p.h> #include <QtTest/qtestassert.h> +#include <qtestresult_p.h> #include <QtCore/qbytearray.h> #include <QtCore/qstring.h> @@ -189,4 +190,26 @@ int qt_asprintf(QTestCharBuffer *str, const char *format, ...) } +namespace QTestPrivate +{ + +void generateTestIdentifier(QTestCharBuffer *identifier, int parts) +{ + const char *testObject = parts & TestObject ? QTestResult::currentTestObjectName() : ""; + const char *testFunction = parts & TestFunction ? (QTestResult::currentTestFunction() ? QTestResult::currentTestFunction() : "UnknownTestFunc") : ""; + const char *objectFunctionFiller = parts & TestObject && parts & (TestFunction | TestDataTag) ? "::" : ""; + const char *testFuctionStart = parts & TestFunction ? "(" : ""; + const char *testFuctionEnd = parts & TestFunction ? ")" : ""; + + const char *dataTag = (parts & TestDataTag) && QTestResult::currentDataTag() ? QTestResult::currentDataTag() : ""; + const char *globalDataTag = (parts & TestDataTag) && QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : ""; + const char *tagFiller = (dataTag[0] && globalDataTag[0]) ? ":" : ""; + + QTest::qt_asprintf(identifier, "%s%s%s%s%s%s%s%s", + testObject, objectFunctionFiller, testFunction, testFuctionStart, + globalDataTag, tagFiller, dataTag, testFuctionEnd); +} + +} + QT_END_NAMESPACE diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h index c4083b43f4..018361b81e 100644 --- a/src/testlib/qabstracttestlogger_p.h +++ b/src/testlib/qabstracttestlogger_p.h @@ -58,6 +58,7 @@ QT_BEGIN_NAMESPACE class QBenchmarkResult; +class QTestData; class QAbstractTestLogger { @@ -91,6 +92,8 @@ public: virtual void enterTestFunction(const char *function) = 0; virtual void leaveTestFunction() = 0; + virtual void enterTestData(QTestData *) {} + virtual void addIncident(IncidentTypes type, const char *description, const char *file = 0, int line = 0) = 0; virtual void addBenchmarkResult(const QBenchmarkResult &result) = 0; @@ -175,6 +178,11 @@ namespace QTest int qt_asprintf(QTestCharBuffer *buf, const char *format, ...); } +namespace QTestPrivate +{ + enum IdentifierPart { TestObject = 0x1, TestFunction = 0x2, TestDataTag = 0x4, AllParts = 0xFFFF }; + void generateTestIdentifier(QTestCharBuffer *identifier, int parts = AllParts); +} QT_END_NAMESPACE diff --git a/src/testlib/qappletestlogger.cpp b/src/testlib/qappletestlogger.cpp new file mode 100644 index 0000000000..2c1005ad80 --- /dev/null +++ b/src/testlib/qappletestlogger.cpp @@ -0,0 +1,149 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qappletestlogger_p.h" + +#include <QPair> + +QT_BEGIN_NAMESPACE + +#if defined(QT_USE_APPLE_UNIFIED_LOGGING) + +using namespace QTestPrivate; + +bool QAppleTestLogger::debugLoggingEnabled() +{ + // Debug-level messages are only captured in memory when debug logging is + // enabled through a configuration change, which can happen automatically + // when running inside Xcode, or with the Console application open. + return os_log_type_enabled(OS_LOG_DEFAULT, OS_LOG_TYPE_DEBUG); +} + +QAppleTestLogger::QAppleTestLogger(QAbstractTestLogger *logger) + : QAbstractTestLogger(nullptr) + , m_logger(logger) +{ +} + +static QAppleLogActivity testFunctionActivity; + +void QAppleTestLogger::enterTestFunction(const char *function) +{ + // Re-create activity each time + testFunctionActivity = QT_APPLE_LOG_ACTIVITY("Running test function").enter(); + + QTestCharBuffer testIdentifier; + QTestPrivate::generateTestIdentifier(&testIdentifier); + QString identifier = QString::fromLatin1(testIdentifier.data()); + QMessageLogContext context(nullptr, 0, nullptr, "qt.test.enter"); + QString message = identifier; + if (AppleUnifiedLogger::messageHandler(QtDebugMsg, context, message, identifier)) + return; // AUL already printed to stderr + + m_logger->enterTestFunction(function); +} + +void QAppleTestLogger::leaveTestFunction() +{ + m_logger->leaveTestFunction(); + testFunctionActivity.leave(); +} + +typedef QPair<QtMsgType, const char *> IncidentClassification; +static IncidentClassification incidentTypeToClassification(QAbstractTestLogger::IncidentTypes type) +{ + switch (type) { + case QAbstractTestLogger::Pass: + return IncidentClassification(QtInfoMsg, "pass"); + case QAbstractTestLogger::XFail: + return IncidentClassification(QtInfoMsg, "xfail"); + case QAbstractTestLogger::Fail: + return IncidentClassification(QtCriticalMsg, "fail"); + case QAbstractTestLogger::XPass: + return IncidentClassification(QtInfoMsg, "xpass"); + case QAbstractTestLogger::BlacklistedPass: + return IncidentClassification(QtWarningMsg, "bpass"); + case QAbstractTestLogger::BlacklistedFail: + return IncidentClassification(QtInfoMsg, "bfail"); + } + return IncidentClassification(QtFatalMsg, nullptr); +} + +void QAppleTestLogger::addIncident(IncidentTypes type, const char *description, + const char *file, int line) +{ + + IncidentClassification incidentClassification = incidentTypeToClassification(type); + + QTestCharBuffer category; + QTest::qt_asprintf(&category, "qt.test.%s", incidentClassification.second); + QMessageLogContext context(file, line, /* function = */ nullptr, category.data()); + + QTestCharBuffer subsystemBuffer; + // It would be nice to have the data tag as part of the subsystem too, but that + // will for some tests results in hundreds of thousands of log objects being + // created, so we limit the subsystem to test functions, which we can hope + // are reasonably limited. + generateTestIdentifier(&subsystemBuffer, TestObject | TestFunction); + QString subsystem = QString::fromLatin1(subsystemBuffer.data()); + + // We still want the full identifier as part of the message though + QTestCharBuffer testIdentifier; + generateTestIdentifier(&testIdentifier); + QString message = QString::fromLatin1(testIdentifier.data()); + if (qstrlen(description)) + message += QLatin1Char('\n') % QString::fromLatin1(description); + + if (AppleUnifiedLogger::messageHandler(incidentClassification.first, context, message, subsystem)) + return; // AUL already printed to stderr + + m_logger->addIncident(type, description, file, line); +} + +void QAppleTestLogger::addMessage(QtMsgType type, const QMessageLogContext &context, const QString &message) +{ + if (AppleUnifiedLogger::messageHandler(type, context, message)) + return; // AUL already printed to stderr + + m_logger->addMessage(type, context, message); +} + +#endif // QT_USE_APPLE_UNIFIED_LOGGING + +QT_END_NAMESPACE diff --git a/src/testlib/qappletestlogger_p.h b/src/testlib/qappletestlogger_p.h new file mode 100644 index 0000000000..5a45fad7a0 --- /dev/null +++ b/src/testlib/qappletestlogger_p.h @@ -0,0 +1,95 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QAPPLETESTLOGGER_P_H +#define QAPPLETESTLOGGER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtTest/private/qabstracttestlogger_p.h> + +#include <QtCore/private/qcore_mac_p.h> + +QT_BEGIN_NAMESPACE + +#if defined(QT_USE_APPLE_UNIFIED_LOGGING) +class QAppleTestLogger : public QAbstractTestLogger +{ +public: + static bool debugLoggingEnabled(); + + QAppleTestLogger(QAbstractTestLogger *logger); + + void startLogging() override + { m_logger->startLogging(); } + void stopLogging() override + { m_logger->stopLogging(); } + + void enterTestFunction(const char *function) override; + void leaveTestFunction() override; + + void addIncident(IncidentTypes type, const char *description, + const char *file = 0, int line = 0) override; + void addMessage(QtMsgType, const QMessageLogContext &, + const QString &) override; + + void addBenchmarkResult(const QBenchmarkResult &result) override + { m_logger->addBenchmarkResult(result); } + + void addMessage(MessageTypes type, const QString &message, + const char *file = 0, int line = 0) override + { m_logger->addMessage(type, message, file, line); } + +private: + QScopedPointer<QAbstractTestLogger> m_logger; +}; +#endif + +QT_END_NAMESPACE + +#endif // QAPPLETESTLOGGER_P_H diff --git a/src/testlib/qplaintestlogger.cpp b/src/testlib/qplaintestlogger.cpp index e5226b7820..853515f2d9 100644 --- a/src/testlib/qplaintestlogger.cpp +++ b/src/testlib/qplaintestlogger.cpp @@ -221,18 +221,6 @@ void QPlainTestLogger::outputMessage(const char *str) outputString(str); } -static void testIdentifier(QTestCharBuffer *identifier) -{ - const char *testObject = QTestResult::currentTestObjectName(); - const char *testFunction = QTestResult::currentTestFunction() ? QTestResult::currentTestFunction() : "UnknownTestFunc"; - - const char *dataTag = QTestResult::currentDataTag() ? QTestResult::currentDataTag() : ""; - const char *globalDataTag = QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : ""; - const char *tagFiller = (dataTag[0] && globalDataTag[0]) ? ":" : ""; - - QTest::qt_asprintf(identifier, "%s::%s(%s%s%s)", testObject, testFunction, globalDataTag, tagFiller, dataTag); -} - void QPlainTestLogger::printMessage(const char *type, const char *msg, const char *file, int line) { QTEST_ASSERT(type); @@ -251,10 +239,10 @@ void QPlainTestLogger::printMessage(const char *type, const char *msg, const cha } const char *msgFiller = msg[0] ? " " : ""; - QTestCharBuffer testIdent; - testIdentifier(&testIdent); + QTestCharBuffer testIdentifier; + QTestPrivate::generateTestIdentifier(&testIdentifier); QTest::qt_asprintf(&messagePrefix, "%s: %s%s%s%s\n", - type, testIdent.data(), msgFiller, msg, failureLocation.data()); + type, testIdentifier.data(), msgFiller, msg, failureLocation.data()); // In colored mode, printf above stripped our nonprintable control characters. // Put them back. diff --git a/src/testlib/qsignalspy.h b/src/testlib/qsignalspy.h index 824dc6aaed..218a26ec5c 100644 --- a/src/testlib/qsignalspy.h +++ b/src/testlib/qsignalspy.h @@ -59,11 +59,7 @@ public: explicit QSignalSpy(const QObject *obj, const char *aSignal) : m_waiting(false) { -#ifdef Q_CC_BOR - const int memberOffset = QObject::staticMetaObject.methodCount(); -#else static const int memberOffset = QObject::staticMetaObject.methodCount(); -#endif if (!obj) { qWarning("QSignalSpy: Cannot spy on a null object"); return; @@ -104,11 +100,7 @@ public: QSignalSpy(const typename QtPrivate::FunctionPointer<Func>::Object *obj, Func signal0) : m_waiting(false) { -#ifdef Q_CC_BOR - const int memberOffset = QObject::staticMetaObject.methodCount(); -#else static const int memberOffset = QObject::staticMetaObject.methodCount(); -#endif if (!obj) { qWarning("QSignalSpy: Cannot spy on a null object"); return; diff --git a/src/testlib/qtaptestlogger.cpp b/src/testlib/qtaptestlogger.cpp new file mode 100644 index 0000000000..37ab89ac91 --- /dev/null +++ b/src/testlib/qtaptestlogger.cpp @@ -0,0 +1,254 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qtaptestlogger_p.h" + +#include "qtestlog_p.h" +#include "qtestresult_p.h" +#include "qtestassert.h" + +#include <QtCore/qregularexpression.h> + +QT_BEGIN_NAMESPACE + +QTapTestLogger::QTapTestLogger(const char *filename) + : QAbstractTestLogger(filename) + , m_wasExpectedFail(false) +{ +} + +QTapTestLogger::~QTapTestLogger() +{ +} + +void QTapTestLogger::startLogging() +{ + QAbstractTestLogger::startLogging(); + + QTestCharBuffer preamble; + QTest::qt_asprintf(&preamble, "TAP version 13\n" + // By convention, test suite names are output as diagnostics lines + // This is a pretty poor convention, as consumers will then treat + // actual diagnostics, e.g. qDebug, as test suite names o_O + "# %s\n", QTestResult::currentTestObjectName()); + outputString(preamble.data()); +} + +void QTapTestLogger::stopLogging() +{ + const int total = QTestLog::totalCount(); + + QTestCharBuffer testPlanAndStats; + QTest::qt_asprintf(&testPlanAndStats, + "1..%d\n" + "# tests %d\n" + "# pass %d\n" + "# fail %d\n", + total, total, QTestLog::passCount(), QTestLog::failCount()); + outputString(testPlanAndStats.data()); + + QAbstractTestLogger::stopLogging(); +} + +void QTapTestLogger::enterTestFunction(const char *function) +{ + Q_UNUSED(function); + m_wasExpectedFail = false; +} + +void QTapTestLogger::enterTestData(QTestData *data) +{ + Q_UNUSED(data); + m_wasExpectedFail = false; +} + +using namespace QTestPrivate; + +void QTapTestLogger::outputTestLine(bool ok, int testNumber, QTestCharBuffer &directive) +{ + QTestCharBuffer testIdentifier; + QTestPrivate::generateTestIdentifier(&testIdentifier, TestFunction | TestDataTag); + + QTestCharBuffer testLine; + QTest::qt_asprintf(&testLine, "%s %d - %s%s\n", + ok ? "ok" : "not ok", testNumber, testIdentifier.data(), directive.data()); + + outputString(testLine.data()); +} + +void QTapTestLogger::addIncident(IncidentTypes type, const char *description, + const char *file, int line) +{ + if (m_wasExpectedFail && type == Pass) { + // XFail comes with a corresponding Pass incident, but we only want + // to emit a single test point for it, so skip the this pass. + return; + } + + bool ok = type == Pass || type == XPass || type == BlacklistedPass; + + QTestCharBuffer directive; + if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass) + // We treat expected or blacklisted failures/passes as TODO-failures/passes, + // which should be treated as soft issues by consumers. Not all do though :/ + QTest::qt_asprintf(&directive, " # TODO %s", description); + + int testNumber = QTestLog::totalCount(); + if (type == XFail) { + // The global test counter hasn't been updated yet for XFail + testNumber += 1; + } + + outputTestLine(ok, testNumber, directive); + + if (!ok) { + // All failures need a diagnostics sections to not confuse consumers + + // The indent needs to be two spaces for maximum compatibility + #define YAML_INDENT " " + + outputString(YAML_INDENT "---\n"); + + if (type != XFail) { + // This is fragile, but unfortunately testlib doesn't plumb + // the expected and actual values to the loggers (yet). + static QRegularExpression verifyRegex( + QLatin1Literal("^'(?<actualexpression>.*)' returned (?<actual>\\w+).+\\((?<message>.*)\\)$")); + + static QRegularExpression comparRegex( + QLatin1Literal("^(?<message>.*)\n" + "\\s*Actual\\s+\\((?<actualexpression>.*)\\)\\s*: (?<actual>.*)\n" + "\\s*Expected\\s+\\((?<expectedexpresssion>.*)\\)\\s*: (?<expected>.*)$")); + + QString descriptionString = QString::fromUtf8(description); + QRegularExpressionMatch match = verifyRegex.match(descriptionString); + if (!match.hasMatch()) + match = comparRegex.match(descriptionString); + + if (match.hasMatch()) { + bool isVerify = match.regularExpression() == verifyRegex; + QString message = match.captured(QLatin1Literal("message")); + QString expected; + QString actual; + + if (isVerify) { + QString expression = QLatin1Literal(" (") + % match.captured(QLatin1Literal("actualexpression")) % QLatin1Char(')') ; + actual = match.captured(QLatin1Literal("actual")).toLower() % expression; + expected = (actual.startsWith(QLatin1Literal("true")) ? QLatin1Literal("false") : QLatin1Literal("true")) % expression; + if (message.isEmpty()) + message = QLatin1Literal("Verification failed"); + } else { + expected = match.captured(QLatin1Literal("expected")) + % QLatin1Literal(" (") % match.captured(QLatin1Literal("expectedexpresssion")) % QLatin1Char(')'); + actual = match.captured(QLatin1Literal("actual")) + % QLatin1Literal(" (") % match.captured(QLatin1Literal("actualexpression")) % QLatin1Char(')'); + } + + QTestCharBuffer diagnosticsYamlish; + QTest::qt_asprintf(&diagnosticsYamlish, + YAML_INDENT "type: %s\n" + YAML_INDENT "message: %s\n" + + // Some consumers understand 'wanted/found', while others need + // 'expected/actual', so we do both for maximum compatibility. + YAML_INDENT "wanted: %s\n" + YAML_INDENT "found: %s\n" + YAML_INDENT "expected: %s\n" + YAML_INDENT "actual: %s\n", + + isVerify ? "QVERIFY" : "QCOMPARE", + qPrintable(message), + qPrintable(expected), qPrintable(actual), + qPrintable(expected), qPrintable(actual) + ); + + outputString(diagnosticsYamlish.data()); + } else { + QTestCharBuffer unparsableDescription; + QTest::qt_asprintf(&unparsableDescription, + YAML_INDENT "# %s\n", description); + outputString(unparsableDescription.data()); + } + } + + if (file) { + QTestCharBuffer location; + QTest::qt_asprintf(&location, + // The generic 'at' key is understood by most consumers. + YAML_INDENT "at: %s::%s() (%s:%d)\n" + + // The file and line keys are for consumers that are able + // to read more granular location info. + YAML_INDENT "file: %s\n" + YAML_INDENT "line: %d\n", + + QTestResult::currentTestObjectName(), + QTestResult::currentTestFunction(), + file, line, file, line + ); + outputString(location.data()); + } + + outputString(YAML_INDENT "...\n"); + } + + m_wasExpectedFail = type == XFail; +} + +void QTapTestLogger::addMessage(MessageTypes type, const QString &message, + const char *file, int line) +{ + Q_UNUSED(file); + Q_UNUSED(line); + + if (type == Skip) { + QTestCharBuffer directive; + QTest::qt_asprintf(&directive, " # SKIP %s", message.toUtf8().constData()); + outputTestLine(/* ok = */ true, QTestLog::totalCount(), directive); + return; + } + + QTestCharBuffer diagnostics; + QTest::qt_asprintf(&diagnostics, "# %s\n", qPrintable(message)); + outputString(diagnostics.data()); +} + +QT_END_NAMESPACE + diff --git a/src/testlib/qtaptestlogger_p.h b/src/testlib/qtaptestlogger_p.h new file mode 100644 index 0000000000..b51343e4fe --- /dev/null +++ b/src/testlib/qtaptestlogger_p.h @@ -0,0 +1,85 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QTAPTESTLOGGER_P_H +#define QTAPTESTLOGGER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtTest/private/qabstracttestlogger_p.h> + +QT_BEGIN_NAMESPACE + +class QTapTestLogger : public QAbstractTestLogger +{ +public: + QTapTestLogger(const char *filename); + ~QTapTestLogger(); + + void startLogging() override; + void stopLogging() override; + + void enterTestFunction(const char *) override; + void leaveTestFunction() override {} + + void enterTestData(QTestData *data) override; + + void addIncident(IncidentTypes type, const char *description, + const char *file = 0, int line = 0) override; + void addMessage(MessageTypes type, const QString &message, + const char *file = 0, int line = 0) override; + + void addBenchmarkResult(const QBenchmarkResult &) override {}; +private: + void outputTestLine(bool ok, int testNumber, QTestCharBuffer &directive); + bool m_wasExpectedFail; +}; + +QT_END_NAMESPACE + +#endif // QTAPTESTLOGGER_P_H diff --git a/src/testlib/qtest.h b/src/testlib/qtest.h index 5517b8fd73..2578037946 100644 --- a/src/testlib/qtest.h +++ b/src/testlib/qtest.h @@ -59,6 +59,8 @@ #include <QtCore/qsize.h> #include <QtCore/qrect.h> +#include <memory> + QT_BEGIN_NAMESPACE @@ -215,6 +217,25 @@ inline char *toString(const std::pair<T1, T2> &pair) return toString(QString::asprintf("std::pair(%s,%s)", first.data(), second.data())); } +template <typename Tuple, int... I> +inline char *toString(const Tuple & tuple, QtPrivate::IndexesList<I...>) { + using UP = std::unique_ptr<char[]>; + // Generate a table of N + 1 elements where N is the number of + // elements in the tuple. + // The last element is needed to support the empty tuple use case. + const UP data[] = { + UP(toString(std::get<I>(tuple)))..., UP{} + }; + return formatString("std::tuple(", ")", sizeof...(I), data[I].get()...); +} + +template <class... Types> +inline char *toString(const std::tuple<Types...> &tuple) +{ + static const std::size_t params_count = sizeof...(Types); + return toString(tuple, typename QtPrivate::Indexes<params_count>::Value()); +} + inline char *toString(std::nullptr_t) { return toString(QLatin1String("nullptr")); diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index f5668c274e..32facaf12b 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -60,6 +60,8 @@ #include <QtCore/qwaitcondition.h> #include <QtCore/qmutex.h> +#include <QtCore/qtestsupport_core.h> + #include <QtTest/private/qtestlog_p.h> #include <QtTest/private/qtesttable_p.h> #include <QtTest/qtestdata.h> @@ -85,7 +87,6 @@ #if defined(Q_OS_LINUX) #include <sys/types.h> -#include <unistd.h> #include <fcntl.h> #endif @@ -99,6 +100,7 @@ #include <errno.h> #include <signal.h> #include <time.h> +#include <unistd.h> # if !defined(Q_OS_INTEGRITY) # include <sys/resource.h> # endif @@ -344,7 +346,9 @@ namespace QTest static int keyDelay = -1; static int mouseDelay = -1; static int eventDelay = -1; +#if QT_CONFIG(thread) static int timeout = -1; +#endif static bool noCrashHandler = false; /*! \internal @@ -395,7 +399,7 @@ int Q_TESTLIB_EXPORT defaultKeyDelay() } return keyDelay; } - +#if QT_CONFIG(thread) static int defaultTimeout() { if (timeout == -1) { @@ -407,6 +411,7 @@ static int defaultTimeout() } return timeout; } +#endif Q_TESTLIB_EXPORT bool printAvailableFunctions = false; Q_TESTLIB_EXPORT QStringList testFunctions; @@ -526,6 +531,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) " xml : XML document\n" " lightxml : A stream of XML tags\n" " teamcity : TeamCity format\n" + " tap : Test Anything Protocol\n" "\n" " *** Multiple loggers can be specified, but at most one can log to stdout.\n" "\n" @@ -537,6 +543,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) " -xml : Output results as XML document\n" " -lightxml : Output results as stream of XML tags\n" " -teamcity : Output results in TeamCity format\n" + " -tap : Output results in Test Anything Protocol format\n" "\n" " *** If no output file is specified, stdout is assumed.\n" " *** If no output format is specified, -txt is assumed.\n" @@ -624,6 +631,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) logFormat = QTestLog::LightXML; } else if (strcmp(argv[i], "-teamcity") == 0) { logFormat = QTestLog::TeamCity; + } else if (strcmp(argv[i], "-tap") == 0) { + logFormat = QTestLog::TAP; } else if (strcmp(argv[i], "-silent") == 0) { QTestLog::setVerboseLevel(-1); } else if (strcmp(argv[i], "-v1") == 0) { @@ -658,8 +667,10 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) logFormat = QTestLog::XunitXML; else if (strcmp(format, "teamcity") == 0) logFormat = QTestLog::TeamCity; + else if (strcmp(format, "tap") == 0) + logFormat = QTestLog::TAP; else { - fprintf(stderr, "output format must be one of txt, csv, lightxml, xml, teamcity or xunitxml\n"); + fprintf(stderr, "output format must be one of txt, csv, lightxml, xml, tap, teamcity or xunitxml\n"); exit(1); } if (strcmp(filename, "-") == 0 && QTestLog::loggerUsingStdout()) { @@ -967,6 +978,8 @@ void TestMethods::invokeTestOnData(int index) const } } +#if QT_CONFIG(thread) + class WatchDog : public QThread { public: @@ -1018,6 +1031,17 @@ private: QWaitCondition waitCondition; }; +#else // !QT_CONFIG(thread) + +class WatchDog : public QObject +{ +public: + void beginTest() {}; + void testFinished() {}; +}; + +#endif + /*! \internal @@ -1141,6 +1165,31 @@ void *fetchData(QTestData *data, const char *tagName, int typeId) } /*! + * \internal + */ +char *formatString(const char *prefix, const char *suffix, size_t numArguments, ...) +{ + va_list ap; + va_start(ap, numArguments); + + QByteArray arguments; + arguments += prefix; + + if (numArguments > 0) { + arguments += va_arg(ap, const char *); + + for (size_t i = 1; i < numArguments; ++i) { + arguments += ", "; + arguments += va_arg(ap, const char *); + } + } + + va_end(ap); + arguments += suffix; + return qstrdup(arguments.constData()); +} + +/*! \fn char* QTest::toHexRepresentation(const char *ba, int length) Returns a pointer to a string that is the string \a ba represented @@ -1439,8 +1488,13 @@ void FatalSignalHandler::signal(int signum) { const int msecsFunctionTime = qRound(QTestLog::msecsFunctionTime()); const int msecsTotalTime = qRound(QTestLog::msecsTotalTime()); - if (signum != SIGINT) + if (signum != SIGINT) { stackTrace(); + if (qEnvironmentVariableIsSet("QTEST_PAUSE_ON_CRASH")) { + fprintf(stderr, "Pausing process %d for debugging\n", getpid()); + raise(SIGSTOP); + } + } qFatal("Received signal %d\n" " Function time: %dms Total time: %dms", signum, msecsFunctionTime, msecsTotalTime); @@ -2380,16 +2434,9 @@ bool QTest::currentTestFailed() */ void QTest::qSleep(int ms) { + // ### Qt 6, move to QtCore or remove altogether QTEST_ASSERT(ms > 0); - -#if defined(Q_OS_WINRT) - WaitForSingleObjectEx(GetCurrentThread(), ms, true); -#elif defined(Q_OS_WIN) - Sleep(uint(ms)); -#else - struct timespec ts = { time_t(ms / 1000), (ms % 1000) * 1000 * 1000 }; - nanosleep(&ts, NULL); -#endif + QTestPrivate::qSleep(ms); } /*! \internal diff --git a/src/testlib/qtestcase.h b/src/testlib/qtestcase.h index 54669c11de..f6891dc941 100644 --- a/src/testlib/qtestcase.h +++ b/src/testlib/qtestcase.h @@ -253,6 +253,22 @@ namespace QTest return nullptr; } + template<typename F> // Output QFlags of registered enumerations + inline typename std::enable_if<QtPrivate::IsQEnumHelper<F>::Value, char*>::type toString(QFlags<F> f) + { + const QMetaEnum me = QMetaEnum::fromType<F>(); + return qstrdup(me.valueToKeys(int(f)).constData()); + } + + template <typename F> // Fallback: Output hex value + inline typename std::enable_if<!QtPrivate::IsQEnumHelper<F>::Value, char*>::type toString(QFlags<F> f) + { + const size_t space = 3 + 2 * sizeof(unsigned); // 2 for 0x, two hex digits per byte, 1 for '\0' + char *msg = new char[space]; + qsnprintf(msg, space, "0x%x", unsigned(f)); + return msg; + } + } // namespace Internal template<typename T> @@ -267,6 +283,9 @@ namespace QTest template <typename T1, typename T2> inline char *toString(const std::pair<T1, T2> &pair); + template <class... Types> + inline char *toString(const std::tuple<Types...> &tuple); + Q_TESTLIB_EXPORT char *toHexRepresentation(const char *ba, int length); Q_TESTLIB_EXPORT char *toPrettyCString(const char *unicode, int length); Q_TESTLIB_EXPORT char *toPrettyUnicode(QStringView string); @@ -372,6 +391,8 @@ namespace QTest Q_TESTLIB_EXPORT bool compare_string_helper(const char *t1, const char *t2, const char *actual, const char *expected, const char *file, int line); + Q_TESTLIB_EXPORT char *formatString(const char *prefix, const char *suffix, size_t numArguments, ...); + #ifndef Q_QDOC QTEST_COMPARE_DECL(short) QTEST_COMPARE_DECL(ushort) diff --git a/src/testlib/qtestcase.qdoc b/src/testlib/qtestcase.qdoc index 9a3c770e31..ad9776f7ec 100644 --- a/src/testlib/qtestcase.qdoc +++ b/src/testlib/qtestcase.qdoc @@ -1112,105 +1112,6 @@ Returns a textual representation of size policy \a sp. */ -/*! \fn void QTest::qWait(int ms) - - Waits for \a ms milliseconds. While waiting, events will be processed and - your test will stay responsive to user interface events or network communication. - - Example: - \snippet code/src_qtestlib_qtestcase.cpp 17 - - The code above will wait until the network server is responding for a - maximum of about 12.5 seconds. - - \sa QTest::qSleep(), QSignalSpy::wait() -*/ - -/*! \fn template <typename Functor> bool QTest::qWaitFor(Functor predicate, int timeout) - - Waits for \a timeout milliseconds or until the \a predicate returns true. - - Returns \c true if the \a predicate returned true at any point, otherwise returns \c false. - - Example: - \snippet code/src_qtestlib_qtestcase.cpp 30 - - The code above will wait for the object to become ready, for a - maximum of three seconds. - - \since 5.10 -*/ - -/*! \fn bool QTest::qWaitForWindowExposed(QWindow *window, int timeout) - \since 5.0 - - Waits for \a timeout milliseconds or until the \a window is exposed. - Returns \c true if \c window is exposed within \a timeout milliseconds, otherwise returns \c false. - - This is mainly useful for asynchronous systems like X11, where a window will be mapped to screen some - time after being asked to show itself on the screen. - - Note that a window that is mapped to screen may still not be considered exposed if the window client - area is completely covered by other windows, or if the window is otherwise not visible. This function - will then time out when waiting for such a window. - - \sa QTest::qWaitForWindowActive(), QWindow::isExposed() -*/ - -/*! \fn bool QTest::qWaitForWindowActive(QWindow *window, int timeout) - \since 5.0 - - Waits for \a timeout milliseconds or until the \a window is active. - - Returns \c true if \c window is active within \a timeout milliseconds, otherwise returns \c false. - - \sa QTest::qWaitForWindowExposed(), QWindow::isActive() -*/ - -/*! \fn bool QTest::qWaitForWindowExposed(QWidget *widget, int timeout) - \since 5.0 - - Waits for \a timeout milliseconds or until the \a widget's window is exposed. - Returns \c true if \c widget's window is exposed within \a timeout milliseconds, otherwise returns \c false. - - This is mainly useful for asynchronous systems like X11, where a window will be mapped to screen some - time after being asked to show itself on the screen. - - Note that a window that is mapped to screen may still not be considered exposed if the window client - area is completely covered by other windows, or if the window is otherwise not visible. This function - will then time out when waiting for such a window. - - A specific configuration where this happens is when using QGLWidget as a viewport widget on macOS: - The viewport widget gets the expose event, not the parent widget. - - \sa QTest::qWaitForWindowActive() -*/ - -/*! \fn bool QTest::qWaitForWindowActive(QWidget *widget, int timeout) - \since 5.0 - - Waits for \a timeout milliseconds or until the \a widget's window is active. - - Returns \c true if \c widget's window is active within \a timeout milliseconds, otherwise returns \c false. - - \sa QTest::qWaitForWindowExposed(), QWidget::isActiveWindow() -*/ - -/*! \fn bool QTest::qWaitForWindowShown(QWidget *widget, int timeout) - \since 5.0 - \deprecated - - Waits for \a timeout milliseconds or until the \a widget's window is exposed. - Returns \c true if \c widget's window is exposed within \a timeout milliseconds, otherwise returns \c false. - - This function does the same as qWaitForWindowExposed(). - - Example: - \snippet code/src_qtestlib_qtestcase.cpp 24 - - \sa QTest::qWaitForWindowActive(), QTest::qWaitForWindowExposed() -*/ - /*! \fn QTouchDevice *QTest::createTouchDevice(QTouchDevice::DeviceType devType = QTouchDevice::TouchScreen) \since 5.8 diff --git a/src/testlib/qtesteventloop.h b/src/testlib/qtesteventloop.h index 19ad22565e..a77b47cd7f 100644 --- a/src/testlib/qtesteventloop.h +++ b/src/testlib/qtesteventloop.h @@ -83,7 +83,7 @@ protected: inline void timerEvent(QTimerEvent *e) override; private: - bool inLoop; + Q_DECL_UNUSED_MEMBER bool inLoop; // ### Qt 6: remove bool _timeout; int timerId; @@ -96,7 +96,6 @@ inline void QTestEventLoop::enterLoopMSecs(int ms) QEventLoop l; - inLoop = true; _timeout = false; timerId = startTimer(ms); @@ -120,8 +119,6 @@ inline void QTestEventLoop::exitLoop() if (loop) loop->exit(); - - inLoop = false; } inline void QTestEventLoop::timerEvent(QTimerEvent *e) diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp index 6260b9e3fd..1268730cc6 100644 --- a/src/testlib/qtestlog.cpp +++ b/src/testlib/qtestlog.cpp @@ -47,10 +47,15 @@ #include <QtTest/private/qxunittestlogger_p.h> #include <QtTest/private/qxmltestlogger_p.h> #include <QtTest/private/qteamcitylogger_p.h> +#include <QtTest/private/qtaptestlogger_p.h> #if defined(HAVE_XCTEST) #include <QtTest/private/qxctestlogger_p.h> #endif +#if defined(Q_OS_DARWIN) +#include <QtTest/private/qappletestlogger_p.h> +#endif + #include <QtCore/qatomic.h> #include <QtCore/qbytearray.h> #include <QtCore/QElapsedTimer> @@ -211,6 +216,11 @@ namespace QTest { FOREACH_LOGGER(logger->leaveTestFunction()); } + static void enterTestData(QTestData *data) + { + FOREACH_LOGGER(logger->enterTestData(data)); + } + static void addIncident(QAbstractTestLogger::IncidentTypes type, const char *description, const char *file = 0, int line = 0) { @@ -339,6 +349,12 @@ void QTestLog::enterTestFunction(const char* function) QTest::TestLoggers::enterTestFunction(function); } +void QTestLog::enterTestData(QTestData *data) +{ + QTEST_ASSERT(data); + QTest::TestLoggers::enterTestData(data); +} + int QTestLog::unhandledIgnoreMessages() { int i = 0; @@ -498,12 +514,24 @@ void QTestLog::addLogger(LogMode mode, const char *filename) case QTestLog::TeamCity: logger = new QTeamCityLogger(filename); break; + case QTestLog::TAP: + logger = new QTapTestLogger(filename); + break; #if defined(HAVE_XCTEST) case QTestLog::XCTest: logger = new QXcodeTestLogger; break; #endif } + +#if defined(QT_USE_APPLE_UNIFIED_LOGGING) + // Logger that also feeds messages to AUL. It needs to wrap the existing + // logger, as it needs to be able to short circuit the existing logger + // in case AUL prints to stderr. + if (QAppleTestLogger::debugLoggingEnabled()) + logger = new QAppleTestLogger(logger); +#endif + QTEST_ASSERT(logger); QTest::TestLoggers::addLogger(logger); } @@ -591,6 +619,11 @@ int QTestLog::blacklistCount() return QTest::blacklists; } +int QTestLog::totalCount() +{ + return passCount() + failCount() + skipCount() + blacklistCount(); +} + void QTestLog::resetCounters() { QTest::passes = 0; diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h index 3d28795188..600c078ce2 100644 --- a/src/testlib/qtestlog_p.h +++ b/src/testlib/qtestlog_p.h @@ -57,12 +57,13 @@ QT_BEGIN_NAMESPACE class QBenchmarkResult; class QRegularExpression; +class QTestData; class Q_TESTLIB_EXPORT QTestLog { public: enum LogMode { - Plain = 0, XML, LightXML, XunitXML, CSV, TeamCity, + Plain = 0, XML, LightXML, XunitXML, CSV, TeamCity, TAP, #if defined(HAVE_XCTEST) XCTest #endif @@ -71,6 +72,8 @@ public: static void enterTestFunction(const char* function); static void leaveTestFunction(); + static void enterTestData(QTestData *data); + static void addPass(const char *msg); static void addFail(const char *msg, const char *file, int line); static void addXFail(const char *msg, const char *file, int line); @@ -110,6 +113,7 @@ public: static int failCount(); static int skipCount(); static int blacklistCount(); + static int totalCount(); static void resetCounters(); diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp index 219190d5da..88e3407c90 100644 --- a/src/testlib/qtestresult.cpp +++ b/src/testlib/qtestresult.cpp @@ -110,6 +110,8 @@ void QTestResult::setCurrentTestData(QTestData *data) { QTest::currentTestData = data; QTest::failed = false; + if (data) + QTestLog::enterTestData(data); } void QTestResult::setCurrentTestFunction(const char *func) diff --git a/src/testlib/qtestsystem.h b/src/testlib/qtestsystem.h index daa0d7aea0..7a73bbb5d2 100644 --- a/src/testlib/qtestsystem.h +++ b/src/testlib/qtestsystem.h @@ -41,113 +41,16 @@ #define QTESTSYSTEM_H #include <QtTest/qtestcase.h> -#include <QtCore/qcoreapplication.h> -#include <QtCore/qdeadlinetimer.h> -#ifdef QT_GUI_LIB -# include <QtGui/QWindow> -#endif -#ifdef QT_WIDGETS_LIB -# include <QtWidgets/QWidget> -#endif - -QT_BEGIN_NAMESPACE - -namespace QTest -{ - template <typename Functor> - Q_REQUIRED_RESULT static bool qWaitFor(Functor predicate, int timeout = 5000) - { - // We should not spin the event loop in case the predicate is already true, - // otherwise we might send new events that invalidate the predicate. - if (predicate()) - return true; - - // qWait() is expected to spin the event loop, even when called with a small - // timeout like 1ms, so we we can't use a simple while-loop here based on - // the deadline timer not having timed out. Use do-while instead. - - int remaining = timeout; - QDeadlineTimer deadline(remaining, Qt::PreciseTimer); - - do { - QCoreApplication::processEvents(QEventLoop::AllEvents, remaining); - QCoreApplication::sendPostedEvents(nullptr, QEvent::DeferredDelete); - - remaining = deadline.remainingTime(); - if (remaining > 0) { - QTest::qSleep(qMin(10, remaining)); - remaining = deadline.remainingTime(); - } - - if (predicate()) - return true; - - remaining = deadline.remainingTime(); - } while (remaining > 0); - - return predicate(); // Last chance - } - - Q_DECL_UNUSED inline static void qWait(int ms) - { - // Ideally this method would be implemented in terms of qWaitFor, with - // a predicate that always returns false, but due to a compiler bug in - // GCC 6 we can't do that. - - Q_ASSERT(QCoreApplication::instance()); - - QDeadlineTimer timer(ms, Qt::PreciseTimer); - int remaining = ms; - do { - QCoreApplication::processEvents(QEventLoop::AllEvents, remaining); - QCoreApplication::sendPostedEvents(Q_NULLPTR, QEvent::DeferredDelete); - remaining = timer.remainingTime(); - if (remaining <= 0) - break; - QTest::qSleep(qMin(10, remaining)); - remaining = timer.remainingTime(); - } while (remaining > 0); - } +#include <QtCore/qtestsupport_core.h> #ifdef QT_GUI_LIB - Q_REQUIRED_RESULT inline static bool qWaitForWindowActive(QWindow *window, int timeout = 5000) - { - return qWaitFor([&]() { return window->isActive(); }, timeout); - } - - Q_REQUIRED_RESULT inline static bool qWaitForWindowExposed(QWindow *window, int timeout = 5000) - { - return qWaitFor([&]() { return window->isExposed(); }, timeout); - } +# include <QtGui/qtestsupport_gui.h> #endif - #ifdef QT_WIDGETS_LIB - Q_REQUIRED_RESULT inline static bool qWaitForWindowActive(QWidget *widget, int timeout = 5000) - { - if (QWindow *window = widget->window()->windowHandle()) - return qWaitForWindowActive(window, timeout); - return false; - } - - Q_REQUIRED_RESULT inline static bool qWaitForWindowExposed(QWidget *widget, int timeout = 5000) - { - if (QWindow *window = widget->window()->windowHandle()) - return qWaitForWindowExposed(window, timeout); - return false; - } +# include <QtWidgets/qtestsupport_widgets.h> #endif -#if QT_DEPRECATED_SINCE(5, 0) -# ifdef QT_WIDGETS_LIB - - QT_DEPRECATED Q_REQUIRED_RESULT inline static bool qWaitForWindowShown(QWidget *widget, int timeout = 5000) - { - return qWaitForWindowExposed(widget, timeout); - } -# endif // QT_WIDGETS_LIB -#endif // QT_DEPRECATED_SINCE(5, 0) -} - +QT_BEGIN_NAMESPACE QT_END_NAMESPACE #endif diff --git a/src/testlib/qxctestlogger.mm b/src/testlib/qxctestlogger.mm index 62fd73070f..9fa9da2fdd 100644 --- a/src/testlib/qxctestlogger.mm +++ b/src/testlib/qxctestlogger.mm @@ -200,7 +200,7 @@ private: [autoreleasepool release]; } -+ (id)defaultTestSuite ++ (QTestLibTests *)defaultTestSuite { return [[QtTestLibTests alloc] initWithName:@"QtTestLib"]; } @@ -255,7 +255,7 @@ static XCTestSuiteRun *s_qtTestSuiteRun = 0; @implementation QtTestLibTest -- (id)initWithInvocation:(NSInvocation *)invocation +- (instancetype)initWithInvocation:(NSInvocation *)invocation { if (self = [super initWithInvocation:invocation]) { // The test object name and function name are used by XCTest after QtTestLib has @@ -322,7 +322,7 @@ QXcodeTestLogger *QXcodeTestLogger::s_currentTestLogger = 0; QXcodeTestLogger::QXcodeTestLogger() : QAbstractTestLogger(0) - , m_testRuns([[NSMutableArray arrayWithCapacity:2] retain]) + , m_testRuns([[NSMutableArray<XCTestRun *> arrayWithCapacity:2] retain]) { Q_ASSERT(!s_currentTestLogger); @@ -383,11 +383,11 @@ static bool isTestFunctionInActiveScope(const char *function) Q_ASSERT(activeScope == Selected); - static NSArray *forcedTests = [@[ @"initTestCase", @"initTestCase_data", @"cleanupTestCase" ] retain]; + static NSArray<NSString *> *forcedTests = [@[ @"initTestCase", @"initTestCase_data", @"cleanupTestCase" ] retain]; if ([forcedTests containsObject:[NSString stringWithUTF8String:function]]) return true; - static NSArray *testsInScope = [[testScope componentsSeparatedByString:@","] retain]; + static NSArray<NSString *> *testsInScope = [[testScope componentsSeparatedByString:@","] retain]; bool inScope = [testsInScope containsObject:[NSString stringWithFormat:@"%s/%s", QTestResult::currentTestObjectName(), function]]; diff --git a/src/testlib/qxctestlogger_p.h b/src/testlib/qxctestlogger_p.h index 1b641f18af..8baa5aa27f 100644 --- a/src/testlib/qxctestlogger_p.h +++ b/src/testlib/qxctestlogger_p.h @@ -90,7 +90,7 @@ private: void pushTestRunForTest(XCTest *test, bool start); XCTestRun *popTestRun(); - NSMutableArray *m_testRuns; + NSMutableArray<XCTestRun *> *m_testRuns; static QXcodeTestLogger *s_currentTestLogger; }; diff --git a/src/testlib/testlib.pro b/src/testlib/testlib.pro index 06a6f8b8e5..46b61dac07 100644 --- a/src/testlib/testlib.pro +++ b/src/testlib/testlib.pro @@ -39,7 +39,8 @@ HEADERS = \ qtesttouch.h \ qtestblacklist_p.h \ qtesthelpers_p.h \ - qttestglobal.h + qttestglobal.h \ + qtaptestlogger_p.h SOURCES = \ qtestcase.cpp \ @@ -65,7 +66,8 @@ SOURCES = \ qtestmouse.cpp \ qtestxunitstreamer.cpp \ qxunittestlogger.cpp \ - qtestblacklist.cpp + qtestblacklist.cpp \ + qtaptestlogger.cpp qtConfig(itemmodeltester) { HEADERS += \ @@ -84,6 +86,9 @@ embedded:QMAKE_CXXFLAGS += -fno-rtti mac { LIBS += -framework Security + SOURCES += qappletestlogger.cpp + HEADERS += qappletestlogger_p.h + macos { HEADERS += qtestutil_macos_p.h OBJECTIVE_SOURCES += qtestutil_macos.mm |