9 files changed, 385 insertions, 7 deletions
diff --git a/src/testlib/doc/src/qttestlib-manual.qdoc b/src/testlib/doc/src/qttestlib-manual.qdoc
index 2e2af8d67f..9c6bdf60a4 100644
--- a/src/testlib/doc/src/qttestlib-manual.qdoc
+++ b/src/testlib/doc/src/qttestlib-manual.qdoc
@@ -175,7 +175,7 @@
     \list
     \li \c -o \e{filename,format} \br
     Writes output to the specified file, in the specified format (one of
-    \c txt, \c xml, \c lightxml or \c xunitxml).  The special filename \c -
+    \c txt, \c xml, \c lightxml, \c xunitxml or \c tap).  The special filename \c -
     may be used to log to standard output.
     \li \c -o \e filename \br
     Writes output to the specified file.
@@ -192,6 +192,8 @@
     benchmarks, since it suppresses normal pass/fail messages.
     \li \c -teamcity \br
     Outputs results in TeamCity format.
+    \li \c -tap \br
+    Outputs results in Test Anything Protocol (TAP) format.
     \endlist
 
     The first version of the \c -o option may be repeated in order to log
@@ -199,8 +201,8 @@
     option can log test results to standard output.
 
     If the first version of the \c -o option is used, neither the second version
-    of the \c -o option nor the \c -txt, \c -xml, \c -lightxml, \c -teamcity
-    or \c -xunitxml options should be used.
+    of the \c -o option nor the \c -txt, \c -xml, \c -lightxml, \c -teamcity,
+    \c -xunitxml or \c -tap options should be used.
 
     If neither version of the \c -o option is used, test results will be logged to
     standard output.  If no format option is used, test results will be logged in
diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h
index a8796d10f0..018361b81e 100644
--- a/src/testlib/qabstracttestlogger_p.h
+++ b/src/testlib/qabstracttestlogger_p.h
@@ -58,6 +58,7 @@
 QT_BEGIN_NAMESPACE
 
 class QBenchmarkResult;
+class QTestData;
 
 class QAbstractTestLogger
 {
@@ -91,6 +92,8 @@ public:
     virtual void enterTestFunction(const char *function) = 0;
     virtual void leaveTestFunction() = 0;
 
+    virtual void enterTestData(QTestData *) {}
+
     virtual void addIncident(IncidentTypes type, const char *description,
                              const char *file = 0, int line = 0) = 0;
     virtual void addBenchmarkResult(const QBenchmarkResult &result) = 0;
diff --git a/src/testlib/qtaptestlogger.cpp b/src/testlib/qtaptestlogger.cpp
new file mode 100644
index 0000000000..37ab89ac91
--- /dev/null
+++ b/src/testlib/qtaptestlogger.cpp
@@ -0,0 +1,254 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtTest module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qtaptestlogger_p.h"
+
+#include "qtestlog_p.h"
+#include "qtestresult_p.h"
+#include "qtestassert.h"
+
+#include <QtCore/qregularexpression.h>
+
+QT_BEGIN_NAMESPACE
+
+QTapTestLogger::QTapTestLogger(const char *filename)
+    : QAbstractTestLogger(filename)
+    , m_wasExpectedFail(false)
+{
+}
+
+QTapTestLogger::~QTapTestLogger()
+{
+}
+
+void QTapTestLogger::startLogging()
+{
+    QAbstractTestLogger::startLogging();
+
+    QTestCharBuffer preamble;
+    QTest::qt_asprintf(&preamble, "TAP version 13\n"
+        // By convention, test suite names are output as diagnostics lines
+        // This is a pretty poor convention, as consumers will then treat
+        // actual diagnostics, e.g. qDebug, as test suite names o_O
+        "# %s\n", QTestResult::currentTestObjectName());
+    outputString(preamble.data());
+}
+
+void QTapTestLogger::stopLogging()
+{
+    const int total = QTestLog::totalCount();
+
+    QTestCharBuffer testPlanAndStats;
+    QTest::qt_asprintf(&testPlanAndStats,
+        "1..%d\n"
+        "# tests %d\n"
+        "# pass %d\n"
+        "# fail %d\n",
+    total, total, QTestLog::passCount(), QTestLog::failCount());
+    outputString(testPlanAndStats.data());
+
+    QAbstractTestLogger::stopLogging();
+}
+
+void QTapTestLogger::enterTestFunction(const char *function)
+{
+    Q_UNUSED(function);
+    m_wasExpectedFail = false;
+}
+
+void QTapTestLogger::enterTestData(QTestData *data)
+{
+    Q_UNUSED(data);
+    m_wasExpectedFail = false;
+}
+
+using namespace QTestPrivate;
+
+void QTapTestLogger::outputTestLine(bool ok, int testNumber, QTestCharBuffer &directive)
+{
+    QTestCharBuffer testIdentifier;
+    QTestPrivate::generateTestIdentifier(&testIdentifier, TestFunction | TestDataTag);
+
+    QTestCharBuffer testLine;
+    QTest::qt_asprintf(&testLine, "%s %d - %s%s\n",
+        ok ? "ok" : "not ok", testNumber, testIdentifier.data(), directive.data());
+
+    outputString(testLine.data());
+}
+
+void QTapTestLogger::addIncident(IncidentTypes type, const char *description,
+                                   const char *file, int line)
+{
+    if (m_wasExpectedFail && type == Pass) {
+        // XFail comes with a corresponding Pass incident, but we only want
+        // to emit a single test point for it, so skip the this pass.
+        return;
+    }
+
+    bool ok = type == Pass || type == XPass || type == BlacklistedPass;
+
+    QTestCharBuffer directive;
+    if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass)
+        // We treat expected or blacklisted failures/passes as TODO-failures/passes,
+        // which should be treated as soft issues by consumers. Not all do though :/
+        QTest::qt_asprintf(&directive, " # TODO %s", description);
+
+    int testNumber = QTestLog::totalCount();
+    if (type == XFail) {
+        // The global test counter hasn't been updated yet for XFail
+        testNumber += 1;
+    }
+
+    outputTestLine(ok, testNumber, directive);
+
+    if (!ok) {
+        // All failures need a diagnostics sections to not confuse consumers
+
+        // The indent needs to be two spaces for maximum compatibility
+        #define YAML_INDENT "  "
+
+        outputString(YAML_INDENT "---\n");
+
+        if (type != XFail) {
+            // This is fragile, but unfortunately testlib doesn't plumb
+            // the expected and actual values to the loggers (yet).
+            static QRegularExpression verifyRegex(
+                QLatin1Literal("^'(?<actualexpression>.*)' returned (?<actual>\\w+).+\\((?<message>.*)\\)$"));
+
+            static QRegularExpression comparRegex(
+                QLatin1Literal("^(?<message>.*)\n"
+                    "\\s*Actual\\s+\\((?<actualexpression>.*)\\)\\s*: (?<actual>.*)\n"
+                    "\\s*Expected\\s+\\((?<expectedexpresssion>.*)\\)\\s*: (?<expected>.*)$"));
+
+            QString descriptionString = QString::fromUtf8(description);
+            QRegularExpressionMatch match = verifyRegex.match(descriptionString);
+            if (!match.hasMatch())
+                match = comparRegex.match(descriptionString);
+
+            if (match.hasMatch()) {
+                bool isVerify = match.regularExpression() == verifyRegex;
+                QString message = match.captured(QLatin1Literal("message"));
+                QString expected;
+                QString actual;
+
+                if (isVerify) {
+                    QString expression = QLatin1Literal(" (")
+                        % match.captured(QLatin1Literal("actualexpression")) % QLatin1Char(')') ;
+                    actual = match.captured(QLatin1Literal("actual")).toLower() % expression;
+                    expected = (actual.startsWith(QLatin1Literal("true")) ? QLatin1Literal("false") : QLatin1Literal("true")) % expression;
+                    if (message.isEmpty())
+                        message = QLatin1Literal("Verification failed");
+                } else {
+                    expected = match.captured(QLatin1Literal("expected"))
+                        % QLatin1Literal(" (") % match.captured(QLatin1Literal("expectedexpresssion")) % QLatin1Char(')');
+                    actual = match.captured(QLatin1Literal("actual"))
+                        % QLatin1Literal(" (") % match.captured(QLatin1Literal("actualexpression")) % QLatin1Char(')');
+                }
+
+                QTestCharBuffer diagnosticsYamlish;
+                QTest::qt_asprintf(&diagnosticsYamlish,
+                    YAML_INDENT "type: %s\n"
+                    YAML_INDENT "message: %s\n"
+
+                    // Some consumers understand 'wanted/found', while others need
+                    // 'expected/actual', so we do both for maximum compatibility.
+                    YAML_INDENT "wanted: %s\n"
+                    YAML_INDENT "found: %s\n"
+                    YAML_INDENT "expected: %s\n"
+                    YAML_INDENT "actual: %s\n",
+
+                    isVerify ? "QVERIFY" : "QCOMPARE",
+                    qPrintable(message),
+                    qPrintable(expected), qPrintable(actual),
+                    qPrintable(expected), qPrintable(actual)
+                );
+
+                outputString(diagnosticsYamlish.data());
+            } else {
+                QTestCharBuffer unparsableDescription;
+                QTest::qt_asprintf(&unparsableDescription,
+                    YAML_INDENT "# %s\n", description);
+                outputString(unparsableDescription.data());
+            }
+        }
+
+        if (file) {
+            QTestCharBuffer location;
+            QTest::qt_asprintf(&location,
+                // The generic 'at' key is understood by most consumers.
+                YAML_INDENT "at: %s::%s() (%s:%d)\n"
+
+                // The file and line keys are for consumers that are able
+                // to read more granular location info.
+                YAML_INDENT "file: %s\n"
+                YAML_INDENT "line: %d\n",
+
+                QTestResult::currentTestObjectName(),
+                QTestResult::currentTestFunction(),
+                file, line, file, line
+            );
+            outputString(location.data());
+        }
+
+        outputString(YAML_INDENT "...\n");
+    }
+
+    m_wasExpectedFail = type == XFail;
+}
+
+void QTapTestLogger::addMessage(MessageTypes type, const QString &message,
+                    const char *file, int line)
+{
+    Q_UNUSED(file);
+    Q_UNUSED(line);
+
+    if (type == Skip) {
+        QTestCharBuffer directive;
+        QTest::qt_asprintf(&directive, " # SKIP %s", message.toUtf8().constData());
+        outputTestLine(/* ok  = */ true, QTestLog::totalCount(), directive);
+        return;
+    }
+
+    QTestCharBuffer diagnostics;
+    QTest::qt_asprintf(&diagnostics, "# %s\n", qPrintable(message));
+    outputString(diagnostics.data());
+}
+
+QT_END_NAMESPACE
+
diff --git a/src/testlib/qtaptestlogger_p.h b/src/testlib/qtaptestlogger_p.h
new file mode 100644
index 0000000000..b51343e4fe
--- /dev/null
+++ b/src/testlib/qtaptestlogger_p.h
@@ -0,0 +1,85 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtTest module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QTAPTESTLOGGER_P_H
+#define QTAPTESTLOGGER_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include <QtTest/private/qabstracttestlogger_p.h>
+
+QT_BEGIN_NAMESPACE
+
+class QTapTestLogger : public QAbstractTestLogger
+{
+public:
+    QTapTestLogger(const char *filename);
+    ~QTapTestLogger();
+
+    void startLogging() override;
+    void stopLogging() override;
+
+    void enterTestFunction(const char *) override;
+    void leaveTestFunction() override {}
+
+    void enterTestData(QTestData *data) override;
+
+    void addIncident(IncidentTypes type, const char *description,
+                 const char *file = 0, int line = 0) override;
+    void addMessage(MessageTypes type, const QString &message,
+                const char *file = 0, int line = 0) override;
+
+    void addBenchmarkResult(const QBenchmarkResult &) override {};
+private:
+    void outputTestLine(bool ok, int testNumber, QTestCharBuffer &directive);
+    bool m_wasExpectedFail;
+};
+
+QT_END_NAMESPACE
+
+#endif // QTAPTESTLOGGER_P_H
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index 7e9c03dbd4..d780dfc6c9 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -521,6 +521,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
          "                         xml      : XML document\n"
          "                         lightxml : A stream of XML tags\n"
          "                         teamcity : TeamCity format\n"
+         "                         tap      : Test Anything Protocol\n"
          "\n"
          "     *** Multiple loggers can be specified, but at most one can log to stdout.\n"
          "\n"
@@ -532,6 +533,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
          " -xml                : Output results as XML document\n"
          " -lightxml           : Output results as stream of XML tags\n"
          " -teamcity           : Output results in TeamCity format\n"
+         " -tap                : Output results in Test Anything Protocol format\n"
          "\n"
          "     *** If no output file is specified, stdout is assumed.\n"
          "     *** If no output format is specified, -txt is assumed.\n"
@@ -619,6 +621,8 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
             logFormat = QTestLog::LightXML;
         } else if (strcmp(argv[i], "-teamcity") == 0) {
             logFormat = QTestLog::TeamCity;
+        } else if (strcmp(argv[i], "-tap") == 0) {
+            logFormat = QTestLog::TAP;
         } else if (strcmp(argv[i], "-silent") == 0) {
             QTestLog::setVerboseLevel(-1);
         } else if (strcmp(argv[i], "-v1") == 0) {
@@ -653,8 +657,10 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
                     logFormat = QTestLog::XunitXML;
                 else if (strcmp(format, "teamcity") == 0)
                     logFormat = QTestLog::TeamCity;
+                else if (strcmp(format, "tap") == 0)
+                    logFormat = QTestLog::TAP;
                 else {
-                    fprintf(stderr, "output format must be one of txt, csv, lightxml, xml, teamcity or xunitxml\n");
+                    fprintf(stderr, "output format must be one of txt, csv, lightxml, xml, tap, teamcity or xunitxml\n");
                     exit(1);
                 }
                 if (strcmp(filename, "-") == 0 && QTestLog::loggerUsingStdout()) {
diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp
index 1c864db71a..3fec395fbe 100644
--- a/src/testlib/qtestlog.cpp
+++ b/src/testlib/qtestlog.cpp
@@ -47,6 +47,7 @@
 #include <QtTest/private/qxunittestlogger_p.h>
 #include <QtTest/private/qxmltestlogger_p.h>
 #include <QtTest/private/qteamcitylogger_p.h>
+#include <QtTest/private/qtaptestlogger_p.h>
 #if defined(HAVE_XCTEST)
 #include <QtTest/private/qxctestlogger_p.h>
 #endif
@@ -213,6 +214,11 @@ namespace QTest {
             FOREACH_LOGGER(logger->leaveTestFunction());
         }
 
+        static void enterTestData(QTestData *data)
+        {
+            FOREACH_LOGGER(logger->enterTestData(data));
+        }
+
         static void addIncident(QAbstractTestLogger::IncidentTypes type, const char *description,
                                 const char *file = 0, int line = 0)
         {
@@ -341,6 +347,12 @@ void QTestLog::enterTestFunction(const char* function)
     QTest::TestLoggers::enterTestFunction(function);
 }
 
+void QTestLog::enterTestData(QTestData *data)
+{
+    QTEST_ASSERT(data);
+    QTest::TestLoggers::enterTestData(data);
+}
+
 int QTestLog::unhandledIgnoreMessages()
 {
     int i = 0;
@@ -500,6 +512,9 @@ void QTestLog::addLogger(LogMode mode, const char *filename)
     case QTestLog::TeamCity:
         logger = new QTeamCityLogger(filename);
         break;
+    case QTestLog::TAP:
+        logger = new QTapTestLogger(filename);
+        break;
 #if defined(HAVE_XCTEST)
     case QTestLog::XCTest:
         logger = new QXcodeTestLogger;
@@ -602,6 +617,11 @@ int QTestLog::blacklistCount()
     return QTest::blacklists;
 }
 
+int QTestLog::totalCount()
+{
+    return passCount() + failCount() + skipCount() + blacklistCount();
+}
+
 void QTestLog::resetCounters()
 {
     QTest::passes = 0;
diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h
index f72c047c35..da16a3d539 100644
--- a/src/testlib/qtestlog_p.h
+++ b/src/testlib/qtestlog_p.h
@@ -57,12 +57,13 @@ QT_BEGIN_NAMESPACE
 
 class QBenchmarkResult;
 class QRegularExpression;
+class QTestData;
 
 class Q_TESTLIB_EXPORT QTestLog
 {
 public:
     enum LogMode {
-        Plain = 0, XML, LightXML, XunitXML, CSV, TeamCity,
+        Plain = 0, XML, LightXML, XunitXML, CSV, TeamCity, TAP,
 #if defined(HAVE_XCTEST)
         XCTest
 #endif
@@ -71,6 +72,8 @@ public:
     static void enterTestFunction(const char* function);
     static void leaveTestFunction();
 
+    static void enterTestData(QTestData *data);
+
     static void addPass(const char *msg);
     static void addFail(const char *msg, const char *file, int line);
     static void addXFail(const char *msg, const char *file, int line);
@@ -110,6 +113,7 @@ public:
     static int failCount();
     static int skipCount();
     static int blacklistCount();
+    static int totalCount();
 
     static void resetCounters();
 
diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp
index 219190d5da..88e3407c90 100644
--- a/src/testlib/qtestresult.cpp
+++ b/src/testlib/qtestresult.cpp
@@ -110,6 +110,8 @@ void QTestResult::setCurrentTestData(QTestData *data)
 {
     QTest::currentTestData = data;
     QTest::failed = false;
+    if (data)
+        QTestLog::enterTestData(data);
 }
 
 void QTestResult::setCurrentTestFunction(const char *func)
diff --git a/src/testlib/testlib.pro b/src/testlib/testlib.pro
index 46ccf72957..c13b851e03 100644
--- a/src/testlib/testlib.pro
+++ b/src/testlib/testlib.pro
@@ -40,7 +40,8 @@ HEADERS = \
     qtestsystem.h \
     qtesttouch.h \
     qtestblacklist_p.h \
-    qtesthelpers_p.h
+    qtesthelpers_p.h \
+    qtaptestlogger_p.h
 
 SOURCES = \
     qabstractitemmodeltester.cpp \
@@ -67,7 +68,8 @@ SOURCES = \
     qtestmouse.cpp \
     qtestxunitstreamer.cpp \
     qxunittestlogger.cpp \
-    qtestblacklist.cpp
+    qtestblacklist.cpp \
+    qtaptestlogger.cpp
 
 DEFINES *= QT_NO_CAST_TO_ASCII \
     QT_NO_CAST_FROM_ASCII \