testlib: Improve JUnit XML conformance

The JUnit test framework did not initially have any XML reporting
facilities built in. Instead, the XML report was generated by the
Apache Ant JUnit task:

 https://github.com/apache/ant/search?q=filename%3AXMLJUnitResultFormatter.java

Many users interacted with these reports via the Jenkins JUnit plugin,
which provided graphical visualization of the test results:

 https://plugins.jenkins.io/junit/

Due to the lack of an official XML schema for the Apache Ant JUnit
report there was some confusion about what the actual format was.
People started documenting the de-facto format, both as produced
by Ant, and as consumed by Jenkins:

 https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd
 https://github.com/junit-team/junit5/search?q=filename%3Ajenkins-junit.xsd

The XML produced by the Qt Test JUnit reporter was far from these
schemas, causing issues when importing results into tools such
as Jenkins, Allure2, or Test Center.

The following changes have been made to improve conformance:

  - The 'timestamp' attribute on <testsuite> is is now in ISO
    8601 local time, without any time zone specified
  - The 'hostname' attribute on <testsuite> is now included
  - The 'classname' attribute on <testcase> is now included
  - The non-standard 'result' attribute on <testcase> has
    been removed
  - The non-standard 'result' attribute on <failure> has
    been renamed to 'type'
  - The <system-out> element on <testsuite> is always included,
    even when empty
  - The non-standard 'tag' attribute on <failure> has been
    removed. Data-driven tests are now represented as individual
    <testcase> elements, e.g.:

      <testcase name="someTest(someData X)" ...>
      <testcase name="someTest(someData Y)" ...>
      <testcase name="someTest(someData Z)" ...>

The resulting XML validates against both the de-facto Apache Ant
'JUnit 4' schema and the Jenkins JUnit plugin schema.

Task-number: QTBUG-95424
Change-Id: I6fc9abedbfb319f2545b99b37d059b18c16776ff
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>

4 files changed, 63 insertions, 68 deletions

diff --git a/src/testlib/qjunittestlogger.cpp b/src/testlib/qjunittestlogger.cpp
index 6374eb0f9d..a34b431060 100644
--- a/src/testlib/qjunittestlogger.cpp
+++ b/src/testlib/qjunittestlogger.cpp
@@ -69,6 +69,14 @@ QJUnitTestLogger::~QJUnitTestLogger()
     delete logFormatter;
 }
 
+// We track test timing per test case, so we
+// need to maintain our own elapsed timer.
+static QElapsedTimer elapsedTestcaseTime;
+static qreal elapsedTestCaseSeconds()
+{
+    return elapsedTestcaseTime.nsecsElapsed() / 1e9;
+}
+
 void QJUnitTestLogger::startLogging()
 {
     QAbstractTestLogger::startLogging();
@@ -84,9 +92,11 @@ void QJUnitTestLogger::startLogging()
     currentTestSuite->addAttribute(QTest::AI_Name, QTestResult::currentTestObjectName());
 
     auto localTime = QDateTime::currentDateTime();
-    auto localTimeWithUtcOffset = localTime.toOffsetFromUtc(localTime.offsetFromUtc());
     currentTestSuite->addAttribute(QTest::AI_Timestamp,
-        localTimeWithUtcOffset.toString(Qt::ISODate).toUtf8().constData());
+        localTime.toString(Qt::ISODate).toUtf8().constData());
+
+    currentTestSuite->addAttribute(QTest::AI_Hostname,
+        QSysInfo::machineHostName().toUtf8().constData());
 
     QTestElement *property;
     QTestElement *properties = new QTestElement(QTest::LET_Properties);
@@ -107,6 +117,8 @@ void QJUnitTestLogger::startLogging()
     properties->addLogElement(property);
 
     currentTestSuite->addLogElement(properties);
+
+    elapsedTestcaseTime.start();
 }
 
 void QJUnitTestLogger::stopLogging()
@@ -134,8 +146,7 @@ void QJUnitTestLogger::stopLogging()
         testcase = testcase->nextElement();
     }
 
-    if (systemOutputElement->childElements())
-        currentTestSuite->addLogElement(systemOutputElement);
+    currentTestSuite->addLogElement(systemOutputElement);
     currentTestSuite->addLogElement(systemErrorElement);
 
     logFormatter->output(currentTestSuite);
@@ -148,19 +159,53 @@ void QJUnitTestLogger::stopLogging()
 
 void QJUnitTestLogger::enterTestFunction(const char *function)
 {
+    enterTestCase(function);
+}
+
+void QJUnitTestLogger::enterTestCase(const char *name)
+{
     currentLogElement = new QTestElement(QTest::LET_TestCase);
-    currentLogElement->addAttribute(QTest::AI_Name, function);
+    currentLogElement->addAttribute(QTest::AI_Name, name);
+    currentLogElement->addAttribute(QTest::AI_Classname, QTestResult::currentTestObjectName());
     currentLogElement->addToList(&listOfTestcases);
 
     // The element will be deleted when the suite is deleted
 
     ++testCounter;
+
+    elapsedTestcaseTime.restart();
+}
+
+void QJUnitTestLogger::enterTestData(QTestData *)
+{
+    QTestCharBuffer testIdentifier;
+    QTestPrivate::generateTestIdentifier(&testIdentifier,
+        QTestPrivate::TestFunction | QTestPrivate::TestDataTag);
+
+    static const char *lastTestFunction = nullptr;
+    if (QTestResult::currentTestFunction() != lastTestFunction) {
+        // Adopt existing testcase for the initial test data
+        auto *name = const_cast<QTestElementAttribute*>(
+            currentLogElement->attribute(QTest::AI_Name));
+        name->setPair(QTest::AI_Name, testIdentifier.data());
+        lastTestFunction = QTestResult::currentTestFunction();
+        elapsedTestcaseTime.restart();
+    } else {
+        // Create new test cases for remaining test data
+        leaveTestCase();
+        enterTestCase(testIdentifier.data());
+    }
 }
 
 void QJUnitTestLogger::leaveTestFunction()
 {
+    leaveTestCase();
+}
+
+void QJUnitTestLogger::leaveTestCase()
+{
     currentLogElement->addAttribute(QTest::AI_Time,
-        QByteArray::number(QTestLog::msecsFunctionTime() / 1000, 'f').constData());
+        QByteArray::number(elapsedTestCaseSeconds(), 'f').constData());
 }
 
 void QJUnitTestLogger::addIncident(IncidentTypes type, const char *description,
@@ -204,47 +249,12 @@ void QJUnitTestLogger::addIncident(IncidentTypes type, const char *description,
 
     if (type == QAbstractTestLogger::Fail || type == QAbstractTestLogger::XPass) {
         QTestElement *failureElement = new QTestElement(QTest::LET_Failure);
-        failureElement->addAttribute(QTest::AI_Result, typeBuf);
+        failureElement->addAttribute(QTest::AI_Type, typeBuf);
         failureElement->addAttribute(QTest::AI_Message, description);
-        addTag(failureElement);
         currentLogElement->addLogElement(failureElement);
     }
 
     /*
-        Only one result can be shown for the whole testfunction.
-        Check if we currently have a result, and if so, overwrite it
-        iff the new result is worse.
-    */
-    QTestElementAttribute* resultAttr =
-        const_cast<QTestElementAttribute*>(currentLogElement->attribute(QTest::AI_Result));
-    if (resultAttr) {
-        const char* oldResult = resultAttr->value();
-        bool overwrite = false;
-        if (!strcmp(oldResult, "pass")) {
-            overwrite = true;
-        }
-        else if (!strcmp(oldResult, "bpass") || !strcmp(oldResult, "bxfail")) {
-            overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail)
-                    || (type == QAbstractTestLogger::BlacklistedFail) || (type == QAbstractTestLogger::BlacklistedXPass);
-        }
-        else if (!strcmp(oldResult, "bfail") || !strcmp(oldResult, "bxpass")) {
-            overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail);
-        }
-        else if (!strcmp(oldResult, "xfail")) {
-            overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail);
-        }
-        else if (!strcmp(oldResult, "xpass")) {
-            overwrite = (type == QAbstractTestLogger::Fail);
-        }
-        if (overwrite) {
-            resultAttr->setPair(QTest::AI_Result, typeBuf);
-        }
-    }
-    else {
-        currentLogElement->addAttribute(QTest::AI_Result, typeBuf);
-    }
-
-    /*
         Since XFAIL does not add a failure to the testlog in junitxml, add a message, so we still
         have some information about the expected failure.
     */
@@ -253,27 +263,6 @@ void QJUnitTestLogger::addIncident(IncidentTypes type, const char *description,
     }
 }
 
-void QJUnitTestLogger::addTag(QTestElement* element)
-{
-    const char *tag = QTestResult::currentDataTag();
-    const char *gtag = QTestResult::currentGlobalDataTag();
-    const char *filler = (tag && gtag) ? ":" : "";
-    if ((!tag || !tag[0]) && (!gtag || !gtag[0])) {
-        return;
-    }
-
-    if (!tag) {
-        tag = "";
-    }
-    if (!gtag) {
-        gtag = "";
-    }
-
-    QTestCharBuffer buf;
-    QTest::qt_asprintf(&buf, "%s%s%s", gtag, filler, tag);
-    element->addAttribute(QTest::AI_Tag, buf.constData());
-}
-
 void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line)
 {
     Q_UNUSED(file);
@@ -318,7 +307,6 @@ void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, con
 
     messageElement->addAttribute(QTest::AI_Type, typeBuf);
     messageElement->addAttribute(QTest::AI_Message, message.toUtf8().constData());
-    addTag(messageElement);
 
     currentLogElement->addLogElement(messageElement);
     ++errorCounter;
diff --git a/src/testlib/qjunittestlogger_p.h b/src/testlib/qjunittestlogger_p.h
index 30de0b2eb0..4c5f05e885 100644
--- a/src/testlib/qjunittestlogger_p.h
+++ b/src/testlib/qjunittestlogger_p.h
@@ -70,16 +70,19 @@ class QJUnitTestLogger : public QAbstractTestLogger
         void enterTestFunction(const char *function) override;
         void leaveTestFunction() override;
 
+        void enterTestData(QTestData *) override;
+
         void addIncident(IncidentTypes type, const char *description,
                      const char *file = nullptr, int line = 0) override;
-        void addTag(QTestElement* element);
-
         void addMessage(MessageTypes type, const QString &message,
                     const char *file = nullptr, int line = 0) override;
 
         void addBenchmarkResult(const QBenchmarkResult &) override {}
 
     private:
+        void enterTestCase(const char *name);
+        void leaveTestCase();
+
         QTestElement *currentTestSuite = nullptr;
         QTestElement *listOfTestcases = nullptr;
         QTestElement *currentLogElement = nullptr;
diff --git a/src/testlib/qtestelementattribute.cpp b/src/testlib/qtestelementattribute.cpp
index 265f126e5a..d091804f8c 100644
--- a/src/testlib/qtestelementattribute.cpp
+++ b/src/testlib/qtestelementattribute.cpp
@@ -143,7 +143,9 @@ const char *QTestElementAttribute::name() const
         "value",
         "iterations",
         "time",
-        "timestamp"
+        "timestamp",
+        "hostname",
+        "classname"
     };
 
     if (attributeIndex != QTest::AI_Undefined)
diff --git a/src/testlib/qtestelementattribute_p.h b/src/testlib/qtestelementattribute_p.h
index a09e19dccb..973d43ace8 100644
--- a/src/testlib/qtestelementattribute_p.h
+++ b/src/testlib/qtestelementattribute_p.h
@@ -78,7 +78,9 @@ namespace QTest {
         AI_Value = 14,
         AI_Iterations = 15,
         AI_Time = 16,
-        AI_Timestamp = 17
+        AI_Timestamp = 17,
+        AI_Hostname = 18,
+        AI_Classname = 19,
     };
 
     enum LogElementType