From 3b42e098ef711e5c60dd18744f6ff9fa07877424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Thu, 1 Mar 2018 17:06:23 +0100 Subject: testlib: Add Test Anything Protocol (TAP) reporter The Test Anything Protocol (TAP), was originally Perl's simple text-based interface between testing modules and test harnesses, but has since been adopted by a large number of producers and consumers in many different languages, which allows colorizing and summarizing test results. The format is very simple: TAP version 13 ok 1 - test description not ok 2 - test description --- message: 'Failure message' severity: fail expected: 123 actual: 456 ... ok 3 - test description # SKIP 1..3 The specification [1] is very brief, so the implementation has been based on how typical consumers behave, especially when it comes to the undefined diagnostics block. [1] http://testanything.org/tap-version-13-specification.html Change-Id: I616e802ea380165c678510e940ddc6607d39c92d Reviewed-by: Simon Hausmann --- src/testlib/doc/src/qttestlib-manual.qdoc | 8 +- src/testlib/qabstracttestlogger_p.h | 3 + src/testlib/qtaptestlogger.cpp | 254 ++++++++++++++++++++++++++++++ src/testlib/qtaptestlogger_p.h | 85 ++++++++++ src/testlib/qtestcase.cpp | 8 +- src/testlib/qtestlog.cpp | 20 +++ src/testlib/qtestlog_p.h | 6 +- src/testlib/qtestresult.cpp | 2 + src/testlib/testlib.pro | 6 +- 9 files changed, 385 insertions(+), 7 deletions(-) create mode 100644 src/testlib/qtaptestlogger.cpp create mode 100644 src/testlib/qtaptestlogger_p.h (limited to 'src/testlib') diff --git a/src/testlib/doc/src/qttestlib-manual.qdoc b/src/testlib/doc/src/qttestlib-manual.qdoc index 2e2af8d67f..9c6bdf60a4 100644 --- a/src/testlib/doc/src/qttestlib-manual.qdoc +++ b/src/testlib/doc/src/qttestlib-manual.qdoc @@ -175,7 +175,7 @@ \list \li \c -o \e{filename,format} \br Writes output to the specified file, in the specified format (one of - \c txt, \c xml, \c lightxml or \c xunitxml). The special filename \c - + \c txt, \c xml, \c lightxml, \c xunitxml or \c tap). The special filename \c - may be used to log to standard output. \li \c -o \e filename \br Writes output to the specified file. @@ -192,6 +192,8 @@ benchmarks, since it suppresses normal pass/fail messages. \li \c -teamcity \br Outputs results in TeamCity format. + \li \c -tap \br + Outputs results in Test Anything Protocol (TAP) format. \endlist The first version of the \c -o option may be repeated in order to log @@ -199,8 +201,8 @@ option can log test results to standard output. If the first version of the \c -o option is used, neither the second version - of the \c -o option nor the \c -txt, \c -xml, \c -lightxml, \c -teamcity - or \c -xunitxml options should be used. + of the \c -o option nor the \c -txt, \c -xml, \c -lightxml, \c -teamcity, + \c -xunitxml or \c -tap options should be used. If neither version of the \c -o option is used, test results will be logged to standard output. If no format option is used, test results will be logged in diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h index a8796d10f0..018361b81e 100644 --- a/src/testlib/qabstracttestlogger_p.h +++ b/src/testlib/qabstracttestlogger_p.h @@ -58,6 +58,7 @@ QT_BEGIN_NAMESPACE class QBenchmarkResult; +class QTestData; class QAbstractTestLogger { @@ -91,6 +92,8 @@ public: virtual void enterTestFunction(const char *function) = 0; virtual void leaveTestFunction() = 0; + virtual void enterTestData(QTestData *) {} + virtual void addIncident(IncidentTypes type, const char *description, const char *file = 0, int line = 0) = 0; virtual void addBenchmarkResult(const QBenchmarkResult &result) = 0; diff --git a/src/testlib/qtaptestlogger.cpp b/src/testlib/qtaptestlogger.cpp new file mode 100644 index 0000000000..37ab89ac91 --- /dev/null +++ b/src/testlib/qtaptestlogger.cpp @@ -0,0 +1,254 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qtaptestlogger_p.h" + +#include "qtestlog_p.h" +#include "qtestresult_p.h" +#include "qtestassert.h" + +#include + +QT_BEGIN_NAMESPACE + +QTapTestLogger::QTapTestLogger(const char *filename) + : QAbstractTestLogger(filename) + , m_wasExpectedFail(false) +{ +} + +QTapTestLogger::~QTapTestLogger() +{ +} + +void QTapTestLogger::startLogging() +{ + QAbstractTestLogger::startLogging(); + + QTestCharBuffer preamble; + QTest::qt_asprintf(&preamble, "TAP version 13\n" + // By convention, test suite names are output as diagnostics lines + // This is a pretty poor convention, as consumers will then treat + // actual diagnostics, e.g. qDebug, as test suite names o_O + "# %s\n", QTestResult::currentTestObjectName()); + outputString(preamble.data()); +} + +void QTapTestLogger::stopLogging() +{ + const int total = QTestLog::totalCount(); + + QTestCharBuffer testPlanAndStats; + QTest::qt_asprintf(&testPlanAndStats, + "1..%d\n" + "# tests %d\n" + "# pass %d\n" + "# fail %d\n", + total, total, QTestLog::passCount(), QTestLog::failCount()); + outputString(testPlanAndStats.data()); + + QAbstractTestLogger::stopLogging(); +} + +void QTapTestLogger::enterTestFunction(const char *function) +{ + Q_UNUSED(function); + m_wasExpectedFail = false; +} + +void QTapTestLogger::enterTestData(QTestData *data) +{ + Q_UNUSED(data); + m_wasExpectedFail = false; +} + +using namespace QTestPrivate; + +void QTapTestLogger::outputTestLine(bool ok, int testNumber, QTestCharBuffer &directive) +{ + QTestCharBuffer testIdentifier; + QTestPrivate::generateTestIdentifier(&testIdentifier, TestFunction | TestDataTag); + + QTestCharBuffer testLine; + QTest::qt_asprintf(&testLine, "%s %d - %s%s\n", + ok ? "ok" : "not ok", testNumber, testIdentifier.data(), directive.data()); + + outputString(testLine.data()); +} + +void QTapTestLogger::addIncident(IncidentTypes type, const char *description, + const char *file, int line) +{ + if (m_wasExpectedFail && type == Pass) { + // XFail comes with a corresponding Pass incident, but we only want + // to emit a single test point for it, so skip the this pass. + return; + } + + bool ok = type == Pass || type == XPass || type == BlacklistedPass; + + QTestCharBuffer directive; + if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass) + // We treat expected or blacklisted failures/passes as TODO-failures/passes, + // which should be treated as soft issues by consumers. Not all do though :/ + QTest::qt_asprintf(&directive, " # TODO %s", description); + + int testNumber = QTestLog::totalCount(); + if (type == XFail) { + // The global test counter hasn't been updated yet for XFail + testNumber += 1; + } + + outputTestLine(ok, testNumber, directive); + + if (!ok) { + // All failures need a diagnostics sections to not confuse consumers + + // The indent needs to be two spaces for maximum compatibility + #define YAML_INDENT " " + + outputString(YAML_INDENT "---\n"); + + if (type != XFail) { + // This is fragile, but unfortunately testlib doesn't plumb + // the expected and actual values to the loggers (yet). + static QRegularExpression verifyRegex( + QLatin1Literal("^'(?.*)' returned (?\\w+).+\\((?.*)\\)$")); + + static QRegularExpression comparRegex( + QLatin1Literal("^(?.*)\n" + "\\s*Actual\\s+\\((?.*)\\)\\s*: (?.*)\n" + "\\s*Expected\\s+\\((?.*)\\)\\s*: (?.*)$")); + + QString descriptionString = QString::fromUtf8(description); + QRegularExpressionMatch match = verifyRegex.match(descriptionString); + if (!match.hasMatch()) + match = comparRegex.match(descriptionString); + + if (match.hasMatch()) { + bool isVerify = match.regularExpression() == verifyRegex; + QString message = match.captured(QLatin1Literal("message")); + QString expected; + QString actual; + + if (isVerify) { + QString expression = QLatin1Literal(" (") + % match.captured(QLatin1Literal("actualexpression")) % QLatin1Char(')') ; + actual = match.captured(QLatin1Literal("actual")).toLower() % expression; + expected = (actual.startsWith(QLatin1Literal("true")) ? QLatin1Literal("false") : QLatin1Literal("true")) % expression; + if (message.isEmpty()) + message = QLatin1Literal("Verification failed"); + } else { + expected = match.captured(QLatin1Literal("expected")) + % QLatin1Literal(" (") % match.captured(QLatin1Literal("expectedexpresssion")) % QLatin1Char(')'); + actual = match.captured(QLatin1Literal("actual")) + % QLatin1Literal(" (") % match.captured(QLatin1Literal("actualexpression")) % QLatin1Char(')'); + } + + QTestCharBuffer diagnosticsYamlish; + QTest::qt_asprintf(&diagnosticsYamlish, + YAML_INDENT "type: %s\n" + YAML_INDENT "message: %s\n" + + // Some consumers understand 'wanted/found', while others need + // 'expected/actual', so we do both for maximum compatibility. + YAML_INDENT "wanted: %s\n" + YAML_INDENT "found: %s\n" + YAML_INDENT "expected: %s\n" + YAML_INDENT "actual: %s\n", + + isVerify ? "QVERIFY" : "QCOMPARE", + qPrintable(message), + qPrintable(expected), qPrintable(actual), + qPrintable(expected), qPrintable(actual) + ); + + outputString(diagnosticsYamlish.data()); + } else { + QTestCharBuffer unparsableDescription; + QTest::qt_asprintf(&unparsableDescription, + YAML_INDENT "# %s\n", description); + outputString(unparsableDescription.data()); + } + } + + if (file) { + QTestCharBuffer location; + QTest::qt_asprintf(&location, + // The generic 'at' key is understood by most consumers. + YAML_INDENT "at: %s::%s() (%s:%d)\n" + + // The file and line keys are for consumers that are able + // to read more granular location info. + YAML_INDENT "file: %s\n" + YAML_INDENT "line: %d\n", + + QTestResult::currentTestObjectName(), + QTestResult::currentTestFunction(), + file, line, file, line + ); + outputString(location.data()); + } + + outputString(YAML_INDENT "...\n"); + } + + m_wasExpectedFail = type == XFail; +} + +void QTapTestLogger::addMessage(MessageTypes type, const QString &message, + const char *file, int line) +{ + Q_UNUSED(file); + Q_UNUSED(line); + + if (type == Skip) { + QTestCharBuffer directive; + QTest::qt_asprintf(&directive, " # SKIP %s", message.toUtf8().constData()); + outputTestLine(/* ok = */ true, QTestLog::totalCount(), directive); + return; + } + + QTestCharBuffer diagnostics; + QTest::qt_asprintf(&diagnostics, "# %s\n", qPrintable(message)); + outputString(diagnostics.data()); +} + +QT_END_NAMESPACE + diff --git a/src/testlib/qtaptestlogger_p.h b/src/testlib/qtaptestlogger_p.h new file mode 100644 index 0000000000..b51343e4fe --- /dev/null +++ b/src/testlib/qtaptestlogger_p.h @@ -0,0 +1,85 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QTAPTESTLOGGER_P_H +#define QTAPTESTLOGGER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include + +QT_BEGIN_NAMESPACE + +class QTapTestLogger : public QAbstractTestLogger +{ +public: + QTapTestLogger(const char *filename); + ~QTapTestLogger(); + + void startLogging() override; + void stopLogging() override; + + void enterTestFunction(const char *) override; + void leaveTestFunction() override {} + + void enterTestData(QTestData *data) override; + + void addIncident(IncidentTypes type, const char *description, + const char *file = 0, int line = 0) override; + void addMessage(MessageTypes type, const QString &message, + const char *file = 0, int line = 0) override; + + void addBenchmarkResult(const QBenchmarkResult &) override {}; +private: + void outputTestLine(bool ok, int testNumber, QTestCharBuffer &directive); + bool m_wasExpectedFail; +}; + +QT_END_NAMESPACE + +#endif // QTAPTESTLOGGER_P_H diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index 7e9c03dbd4..d780dfc6c9 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -521,6 +521,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) " xml : XML document\n" " lightxml : A stream of XML tags\n" " teamcity : TeamCity format\n" + " tap : Test Anything Protocol\n" "\n" " *** Multiple loggers can be specified, but at most one can log to stdout.\n" "\n" @@ -532,6 +533,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) " -xml : Output results as XML document\n" " -lightxml : Output results as stream of XML tags\n" " -teamcity : Output results in TeamCity format\n" + " -tap : Output results in Test Anything Protocol format\n" "\n" " *** If no output file is specified, stdout is assumed.\n" " *** If no output format is specified, -txt is assumed.\n" @@ -619,6 +621,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) logFormat = QTestLog::LightXML; } else if (strcmp(argv[i], "-teamcity") == 0) { logFormat = QTestLog::TeamCity; + } else if (strcmp(argv[i], "-tap") == 0) { + logFormat = QTestLog::TAP; } else if (strcmp(argv[i], "-silent") == 0) { QTestLog::setVerboseLevel(-1); } else if (strcmp(argv[i], "-v1") == 0) { @@ -653,8 +657,10 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) logFormat = QTestLog::XunitXML; else if (strcmp(format, "teamcity") == 0) logFormat = QTestLog::TeamCity; + else if (strcmp(format, "tap") == 0) + logFormat = QTestLog::TAP; else { - fprintf(stderr, "output format must be one of txt, csv, lightxml, xml, teamcity or xunitxml\n"); + fprintf(stderr, "output format must be one of txt, csv, lightxml, xml, tap, teamcity or xunitxml\n"); exit(1); } if (strcmp(filename, "-") == 0 && QTestLog::loggerUsingStdout()) { diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp index 1c864db71a..3fec395fbe 100644 --- a/src/testlib/qtestlog.cpp +++ b/src/testlib/qtestlog.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #if defined(HAVE_XCTEST) #include #endif @@ -213,6 +214,11 @@ namespace QTest { FOREACH_LOGGER(logger->leaveTestFunction()); } + static void enterTestData(QTestData *data) + { + FOREACH_LOGGER(logger->enterTestData(data)); + } + static void addIncident(QAbstractTestLogger::IncidentTypes type, const char *description, const char *file = 0, int line = 0) { @@ -341,6 +347,12 @@ void QTestLog::enterTestFunction(const char* function) QTest::TestLoggers::enterTestFunction(function); } +void QTestLog::enterTestData(QTestData *data) +{ + QTEST_ASSERT(data); + QTest::TestLoggers::enterTestData(data); +} + int QTestLog::unhandledIgnoreMessages() { int i = 0; @@ -500,6 +512,9 @@ void QTestLog::addLogger(LogMode mode, const char *filename) case QTestLog::TeamCity: logger = new QTeamCityLogger(filename); break; + case QTestLog::TAP: + logger = new QTapTestLogger(filename); + break; #if defined(HAVE_XCTEST) case QTestLog::XCTest: logger = new QXcodeTestLogger; @@ -602,6 +617,11 @@ int QTestLog::blacklistCount() return QTest::blacklists; } +int QTestLog::totalCount() +{ + return passCount() + failCount() + skipCount() + blacklistCount(); +} + void QTestLog::resetCounters() { QTest::passes = 0; diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h index f72c047c35..da16a3d539 100644 --- a/src/testlib/qtestlog_p.h +++ b/src/testlib/qtestlog_p.h @@ -57,12 +57,13 @@ QT_BEGIN_NAMESPACE class QBenchmarkResult; class QRegularExpression; +class QTestData; class Q_TESTLIB_EXPORT QTestLog { public: enum LogMode { - Plain = 0, XML, LightXML, XunitXML, CSV, TeamCity, + Plain = 0, XML, LightXML, XunitXML, CSV, TeamCity, TAP, #if defined(HAVE_XCTEST) XCTest #endif @@ -71,6 +72,8 @@ public: static void enterTestFunction(const char* function); static void leaveTestFunction(); + static void enterTestData(QTestData *data); + static void addPass(const char *msg); static void addFail(const char *msg, const char *file, int line); static void addXFail(const char *msg, const char *file, int line); @@ -110,6 +113,7 @@ public: static int failCount(); static int skipCount(); static int blacklistCount(); + static int totalCount(); static void resetCounters(); diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp index 219190d5da..88e3407c90 100644 --- a/src/testlib/qtestresult.cpp +++ b/src/testlib/qtestresult.cpp @@ -110,6 +110,8 @@ void QTestResult::setCurrentTestData(QTestData *data) { QTest::currentTestData = data; QTest::failed = false; + if (data) + QTestLog::enterTestData(data); } void QTestResult::setCurrentTestFunction(const char *func) diff --git a/src/testlib/testlib.pro b/src/testlib/testlib.pro index 46ccf72957..c13b851e03 100644 --- a/src/testlib/testlib.pro +++ b/src/testlib/testlib.pro @@ -40,7 +40,8 @@ HEADERS = \ qtestsystem.h \ qtesttouch.h \ qtestblacklist_p.h \ - qtesthelpers_p.h + qtesthelpers_p.h \ + qtaptestlogger_p.h SOURCES = \ qabstractitemmodeltester.cpp \ @@ -67,7 +68,8 @@ SOURCES = \ qtestmouse.cpp \ qtestxunitstreamer.cpp \ qxunittestlogger.cpp \ - qtestblacklist.cpp + qtestblacklist.cpp \ + qtaptestlogger.cpp DEFINES *= QT_NO_CAST_TO_ASCII \ QT_NO_CAST_FROM_ASCII \ -- cgit v1.2.3