testlib: Add Test Anything Protocol (TAP) reporter

The Test Anything Protocol (TAP), was originally Perl's simple text-based
interface between testing modules and test harnesses, but has since been
adopted by a large number of producers and consumers in many different
languages, which allows colorizing and summarizing test results.

The format is very simple:

TAP version 13
ok 1 - test description
not ok 2 - test description
  ---
  message: 'Failure message'
  severity: fail
  expected: 123
  actual: 456
  ...
ok 3 - test description # SKIP
1..3

The specification [1] is very brief, so the implementation has been
based on how typical consumers behave, especially when it comes to
the undefined diagnostics block.

[1] http://testanything.org/tap-version-13-specification.html

Change-Id: I616e802ea380165c678510e940ddc6607d39c92d
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

1 files changed, 23 insertions, 1 deletions

diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp
index 9c23574456..8ac2ca7b90 100644
--- a/tests/auto/testlib/selftests/tst_selftests.cpp
+++ b/tests/auto/testlib/selftests/tst_selftests.cpp
@@ -310,6 +310,14 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
                      QStringList() << "teamcity",
                      QStringList() << "-teamcity" << "-o" << logName("teamcity")
                     )
+        << LoggerSet("old stdout tap",
+                     QStringList() << "stdout tap",
+                     QStringList() << "-tap"
+                    )
+        << LoggerSet("old tap",
+                     QStringList() << "tap",
+                     QStringList() << "-tap" << "-o" << logName("tap")
+                    )
         // Test with new-style options for a single logger
         << LoggerSet("new stdout txt",
                      QStringList() << "stdout txt",
@@ -357,6 +365,14 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
                      QStringList() << "teamcity",
                      QStringList() << "-o" << logName("teamcity")+",teamcity"
                     )
+        << LoggerSet("new stdout tap",
+                     QStringList() << "stdout tap",
+                     QStringList() << "-o" << "-,tap"
+                    )
+        << LoggerSet("new tap",
+                     QStringList() << "tap",
+                     QStringList() << "-o" << logName("tap")+",tap"
+                    )
         // Test with two loggers (don't test all 32 combinations, just a sample)
         << LoggerSet("stdout txt + txt",
                      QStringList() << "stdout txt" << "txt",
@@ -380,13 +396,14 @@ QList<LoggerSet> tst_Selftests::allLoggerSets() const
                     )
         // All loggers at the same time (except csv)
         << LoggerSet("all loggers",
-                     QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml",
+                     QStringList() << "txt" << "xml" << "lightxml" << "stdout txt" << "xunitxml" << "tap",
                      QStringList() << "-o" << logName("txt")+",txt"
                                    << "-o" << logName("xml")+",xml"
                                    << "-o" << logName("lightxml")+",lightxml"
                                    << "-o" << "-,txt"
                                    << "-o" << logName("xunitxml")+",xunitxml"
                                    << "-o" << logName("teamcity")+",teamcity"
+                                   << "-o" << logName("tap")+",tap"
                     )
     ;
 }
@@ -851,6 +868,11 @@ bool tst_Selftests::compareOutput(const QString &logger, const QString &subdir,
             expectedLine.replace(teamcityLocRegExp, teamCityLocation());
         }
 
+        if (logger.endsWith(QLatin1String("tap"))) {
+            if (expectedLine.contains(QLatin1String("at:")) || expectedLine.contains(QLatin1String("file:")))
+                actualLine = expectedLine;
+        }
+
         if (!compareLine(logger, subdir, benchmark, actualLine,
                          expectedLine, errorMessage)) {
             errorMessage->prepend(QLatin1String("Line ") + QString::number(i + 1)