1 files changed, 165 insertions, 253 deletions

diff --git a/src/testlib/qjunittestlogger.cpp b/src/testlib/qjunittestlogger.cpp
index 147777f77c..4ee5788bee 100644
--- a/src/testlib/qjunittestlogger.cpp
+++ b/src/testlib/qjunittestlogger.cpp
@@ -1,41 +1,5 @@
-/****************************************************************************
-**
-** Copyright (C) 2016 The Qt Company Ltd.
-** Contact: https://www.qt.io/licensing/
-**
-** This file is part of the QtTest module of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:LGPL$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and The Qt Company. For licensing terms
-** and conditions see https://www.qt.io/terms-conditions. For further
-** information use the contact form at https://www.qt.io/contact-us.
-**
-** GNU Lesser General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU Lesser
-** General Public License version 3 as published by the Free Software
-** Foundation and appearing in the file LICENSE.LGPL3 included in the
-** packaging of this file. Please review the following information to
-** ensure the GNU Lesser General Public License version 3 requirements
-** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 2.0 or (at your option) the GNU General
-** Public license version 3 or any later version approved by the KDE Free
-** Qt Foundation. The licenses are as published by the Free Software
-** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
-** included in the packaging of this file. Please review the following
-** information to ensure the GNU General Public License requirements will
-** be met: https://www.gnu.org/licenses/gpl-2.0.html and
-** https://www.gnu.org/licenses/gpl-3.0.html.
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 #include <QtTest/private/qjunittestlogger_p.h>
 #include <QtTest/private/qtestelement_p.h>
@@ -45,18 +9,21 @@
 #include <QtTest/private/qbenchmark_p.h>
 #include <QtTest/private/qtestlog_p.h>
 
-#ifdef min // windows.h without NOMINMAX is included by the benchmark headers.
-#  undef min
-#endif
-#ifdef max
-#  undef max
-#endif
-
 #include <QtCore/qlibraryinfo.h>
 
 #include <string.h>
 
 QT_BEGIN_NAMESPACE
+/*! \internal
+    \class QJUnitTestLogger
+    \inmodule QtTest
+
+    QJUnitTestLogger implements logging in a JUnit-compatible XML format.
+
+    The \l{JUnit XML} format was originally developed for Java testing.
+    It is supported by \l{Test Center}.
+*/
+// QTBUG-95424 links to further useful documentation.
 
 QJUnitTestLogger::QJUnitTestLogger(const char *filename)
     : QAbstractTestLogger(filename)
@@ -69,24 +36,35 @@ QJUnitTestLogger::~QJUnitTestLogger()
     delete logFormatter;
 }
 
+// We track test timing per test case, so we
+// need to maintain our own elapsed timer.
+Q_CONSTINIT static QElapsedTimer elapsedTestcaseTime;
+static qreal elapsedTestCaseSeconds()
+{
+    return elapsedTestcaseTime.nsecsElapsed() / 1e9;
+}
+
+static QByteArray toSecondsFormat(qreal ms)
+{
+    return QByteArray::number(ms / 1000, 'f', 3);
+}
+
 void QJUnitTestLogger::startLogging()
 {
     QAbstractTestLogger::startLogging();
 
     logFormatter = new QTestJUnitStreamer(this);
-    delete systemOutputElement;
-    systemOutputElement = new QTestElement(QTest::LET_SystemOutput);
-    delete systemErrorElement;
-    systemErrorElement = new QTestElement(QTest::LET_SystemError);
 
     Q_ASSERT(!currentTestSuite);
     currentTestSuite = new QTestElement(QTest::LET_TestSuite);
     currentTestSuite->addAttribute(QTest::AI_Name, QTestResult::currentTestObjectName());
 
     auto localTime = QDateTime::currentDateTime();
-    auto localTimeWithUtcOffset = localTime.toOffsetFromUtc(localTime.offsetFromUtc());
     currentTestSuite->addAttribute(QTest::AI_Timestamp,
-        localTimeWithUtcOffset.toString(Qt::ISODate).toUtf8().constData());
+        localTime.toString(Qt::ISODate).toUtf8().constData());
+
+    currentTestSuite->addAttribute(QTest::AI_Hostname,
+        QSysInfo::machineHostName().toUtf8().constData());
 
     QTestElement *property;
     QTestElement *properties = new QTestElement(QTest::LET_Properties);
@@ -94,19 +72,21 @@ void QJUnitTestLogger::startLogging()
     property = new QTestElement(QTest::LET_Property);
     property->addAttribute(QTest::AI_Name, "QTestVersion");
     property->addAttribute(QTest::AI_PropertyValue, QTEST_VERSION_STR);
-    properties->addLogElement(property);
+    properties->addChild(property);
 
     property = new QTestElement(QTest::LET_Property);
     property->addAttribute(QTest::AI_Name, "QtVersion");
     property->addAttribute(QTest::AI_PropertyValue, qVersion());
-    properties->addLogElement(property);
+    properties->addChild(property);
 
     property = new QTestElement(QTest::LET_Property);
     property->addAttribute(QTest::AI_Name, "QtBuild");
     property->addAttribute(QTest::AI_PropertyValue, QLibraryInfo::build());
-    properties->addLogElement(property);
+    properties->addChild(property);
 
-    currentTestSuite->addLogElement(properties);
+    currentTestSuite->addChild(properties);
+
+    elapsedTestcaseTime.start();
 }
 
 void QJUnitTestLogger::stopLogging()
@@ -122,21 +102,15 @@ void QJUnitTestLogger::stopLogging()
     qsnprintf(buf, sizeof(buf), "%i", errorCounter);
     currentTestSuite->addAttribute(QTest::AI_Errors, buf);
 
-    currentTestSuite->addAttribute(QTest::AI_Time,
-        QByteArray::number(QTestLog::msecsTotalTime() / 1000, 'f').constData());
-
-    currentTestSuite->addLogElement(listOfTestcases);
+    qsnprintf(buf, sizeof(buf), "%i", QTestLog::skipCount());
+    currentTestSuite->addAttribute(QTest::AI_Skipped, buf);
 
-    // For correct indenting, make sure every testcase knows its parent
-    QTestElement *testcase = listOfTestcases;
-    while (testcase) {
-        testcase->setParent(currentTestSuite);
-        testcase = testcase->nextElement();
-    }
+    currentTestSuite->addAttribute(QTest::AI_Time,
+        toSecondsFormat(QTestLog::msecsTotalTime()).constData());
 
-    if (systemOutputElement->childElements())
-        currentTestSuite->addLogElement(systemOutputElement);
-    currentTestSuite->addLogElement(systemErrorElement);
+    for (auto *testCase : listOfTestcases)
+        currentTestSuite->addChild(testCase);
+    listOfTestcases.clear();
 
     logFormatter->output(currentTestSuite);
 
@@ -148,226 +122,164 @@ void QJUnitTestLogger::stopLogging()
 
 void QJUnitTestLogger::enterTestFunction(const char *function)
 {
-    currentLogElement = new QTestElement(QTest::LET_TestCase);
-    currentLogElement->addAttribute(QTest::AI_Name, function);
-    currentLogElement->addToList(&listOfTestcases);
+    enterTestCase(function);
+}
+
+void QJUnitTestLogger::enterTestCase(const char *name)
+{
+    currentTestCase = new QTestElement(QTest::LET_TestCase);
+    currentTestCase->addAttribute(QTest::AI_Name, name);
+    currentTestCase->addAttribute(QTest::AI_Classname, QTestResult::currentTestObjectName());
+    listOfTestcases.push_back(currentTestCase);
+
+    Q_ASSERT(!systemOutputElement && !systemErrorElement);
+    systemOutputElement = new QTestElement(QTest::LET_SystemOutput);
+    systemErrorElement = new QTestElement(QTest::LET_SystemError);
 
     // The element will be deleted when the suite is deleted
 
     ++testCounter;
+
+    elapsedTestcaseTime.restart();
 }
 
-void QJUnitTestLogger::leaveTestFunction()
+void QJUnitTestLogger::enterTestData(QTestData *)
 {
-    currentLogElement->addAttribute(QTest::AI_Time,
-        QByteArray::number(QTestLog::msecsFunctionTime() / 1000, 'f').constData());
+    QTestCharBuffer testIdentifier;
+    QTestPrivate::generateTestIdentifier(&testIdentifier,
+        QTestPrivate::TestFunction | QTestPrivate::TestDataTag);
+
+    static const char *lastTestFunction = nullptr;
+    if (QTestResult::currentTestFunction() != lastTestFunction) {
+        // Adopt existing testcase for the initial test data
+        auto *name = const_cast<QTestElementAttribute*>(
+            currentTestCase->attribute(QTest::AI_Name));
+        name->setPair(QTest::AI_Name, testIdentifier.data());
+        lastTestFunction = QTestResult::currentTestFunction();
+        elapsedTestcaseTime.restart();
+    } else {
+        // Create new test cases for remaining test data
+        leaveTestCase();
+        enterTestCase(testIdentifier.data());
+    }
 }
 
-void QJUnitTestLogger::addIncident(IncidentTypes type, const char *description,
-                                   const char *file, int line)
+void QJUnitTestLogger::leaveTestFunction()
 {
-    const char *typeBuf = nullptr;
-    char buf[100];
-
-    switch (type) {
-    case QAbstractTestLogger::XPass:
-        ++failureCounter;
-        typeBuf = "xpass";
-        break;
-    case QAbstractTestLogger::Pass:
-        typeBuf = "pass";
-        break;
-    case QAbstractTestLogger::XFail:
-        typeBuf = "xfail";
-        break;
-    case QAbstractTestLogger::Fail:
-        ++failureCounter;
-        typeBuf = "fail";
-        break;
-    case QAbstractTestLogger::BlacklistedPass:
-        typeBuf = "bpass";
-        break;
-    case QAbstractTestLogger::BlacklistedFail:
-        ++failureCounter;
-        typeBuf = "bfail";
-        break;
-    case QAbstractTestLogger::BlacklistedXPass:
-        typeBuf = "bxpass";
-        break;
-    case QAbstractTestLogger::BlacklistedXFail:
-        ++failureCounter;
-        typeBuf = "bxfail";
-        break;
-    default:
-        typeBuf = "??????";
-        break;
-    }
+    leaveTestCase();
+}
 
-    if (type == QAbstractTestLogger::Fail || type == QAbstractTestLogger::XPass) {
-        QTestElement *failureElement = new QTestElement(QTest::LET_Failure);
-        failureElement->addAttribute(QTest::AI_Result, typeBuf);
-        if (file)
-            failureElement->addAttribute(QTest::AI_File, file);
-        else
-            failureElement->addAttribute(QTest::AI_File, "");
-        qsnprintf(buf, sizeof(buf), "%i", line);
-        failureElement->addAttribute(QTest::AI_Line, buf);
-        failureElement->addAttribute(QTest::AI_Description, description);
-        addTag(failureElement);
-        currentLogElement->addLogElement(failureElement);
-    }
+void QJUnitTestLogger::leaveTestCase()
+{
+    currentTestCase->addAttribute(QTest::AI_Time,
+        toSecondsFormat(elapsedTestCaseSeconds() * 1000).constData());
 
-    /*
-        Only one result can be shown for the whole testfunction.
-        Check if we currently have a result, and if so, overwrite it
-        iff the new result is worse.
-    */
-    QTestElementAttribute* resultAttr =
-        const_cast<QTestElementAttribute*>(currentLogElement->attribute(QTest::AI_Result));
-    if (resultAttr) {
-        const char* oldResult = resultAttr->value();
-        bool overwrite = false;
-        if (!strcmp(oldResult, "pass")) {
-            overwrite = true;
-        }
-        else if (!strcmp(oldResult, "bpass") || !strcmp(oldResult, "bxfail")) {
-            overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail)
-                    || (type == QAbstractTestLogger::BlacklistedFail) || (type == QAbstractTestLogger::BlacklistedXPass);
-        }
-        else if (!strcmp(oldResult, "bfail") || !strcmp(oldResult, "bxpass")) {
-            overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail);
-        }
-        else if (!strcmp(oldResult, "xfail")) {
-            overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail);
-        }
-        else if (!strcmp(oldResult, "xpass")) {
-            overwrite = (type == QAbstractTestLogger::Fail);
-        }
-        if (overwrite) {
-            resultAttr->setPair(QTest::AI_Result, typeBuf);
-        }
-    }
-    else {
-        currentLogElement->addAttribute(QTest::AI_Result, typeBuf);
-    }
+    if (!systemOutputElement->childElements().empty())
+        currentTestCase->addChild(systemOutputElement);
+    else
+        delete systemOutputElement;
 
-    if (file)
-        currentLogElement->addAttribute(QTest::AI_File, file);
+    if (!systemErrorElement->childElements().empty())
+        currentTestCase->addChild(systemErrorElement);
     else
-        currentLogElement->addAttribute(QTest::AI_File, "");
+        delete systemErrorElement;
 
-    qsnprintf(buf, sizeof(buf), "%i", line);
-    currentLogElement->addAttribute(QTest::AI_Line, buf);
+    systemOutputElement = nullptr;
+    systemErrorElement = nullptr;
+}
 
-    /*
-        Since XFAIL does not add a failure to the testlog in junitxml, add a message, so we still
-        have some information about the expected failure.
-    */
-    if (type == QAbstractTestLogger::XFail) {
-        QJUnitTestLogger::addMessage(QAbstractTestLogger::Info, QString::fromUtf8(description), file, line);
+void QJUnitTestLogger::addIncident(IncidentTypes type, const char *description,
+                                   const char *file, int line)
+{
+    if (type == Fail || type == XPass) {
+        auto failureType = [&]() {
+            switch (type) {
+            case QAbstractTestLogger::Fail: return "fail";
+            case QAbstractTestLogger::XPass: return "xpass";
+            default: Q_UNREACHABLE();
+            }
+        }();
+
+        addFailure(QTest::LET_Failure, failureType, QString::fromUtf8(description));
+    } else if (type == XFail) {
+        // Since XFAIL does not add a failure to the testlog in JUnit XML we add a
+        // message, so we still have some information about the expected failure.
+        addMessage(Info, QString::fromUtf8(description), file, line);
+    } else if (type == Skip) {
+        auto skippedElement = new QTestElement(QTest::LET_Skipped);
+        skippedElement->addAttribute(QTest::AI_Message, description);
+        currentTestCase->addChild(skippedElement);
     }
 }
 
-void QJUnitTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
+void QJUnitTestLogger::addFailure(QTest::LogElementType elementType,
+    const char *failureType, const QString &failureDescription)
 {
-    QTestElement *benchmarkElement = new QTestElement(QTest::LET_Benchmark);
+    if (elementType == QTest::LET_Failure) {
+        // Make sure we're not adding failure when we already have error,
+        // or adding additional failures when we already have a failure.
+        for (auto *childElement : currentTestCase->childElements()) {
+            if (childElement->elementType() == QTest::LET_Error ||
+                childElement->elementType() == QTest::LET_Failure)
+                return;
+        }
+    }
 
-    benchmarkElement->addAttribute(
-        QTest::AI_Metric,
-        QTest::benchmarkMetricName(result.metric));
-    benchmarkElement->addAttribute(QTest::AI_Tag, result.context.tag.toUtf8().data());
+    QTestElement *failureElement = new QTestElement(elementType);
+    failureElement->addAttribute(QTest::AI_Type, failureType);
 
-    const qreal valuePerIteration = qreal(result.value) / qreal(result.iterations);
-    benchmarkElement->addAttribute(QTest::AI_Value, QByteArray::number(valuePerIteration).constData());
+    // Assume the first line is the message, and the remainder are details
+    QString message = failureDescription.section(u'\n', 0, 0);
+    QString details = failureDescription.section(u'\n', 1);
 
-    char buf[100];
-    qsnprintf(buf, sizeof(buf), "%i", result.iterations);
-    benchmarkElement->addAttribute(QTest::AI_Iterations, buf);
-    currentLogElement->addLogElement(benchmarkElement);
-}
+    failureElement->addAttribute(QTest::AI_Message, message.toUtf8().constData());
 
-void QJUnitTestLogger::addTag(QTestElement* element)
-{
-    const char *tag = QTestResult::currentDataTag();
-    const char *gtag = QTestResult::currentGlobalDataTag();
-    const char *filler = (tag && gtag) ? ":" : "";
-    if ((!tag || !tag[0]) && (!gtag || !gtag[0])) {
-        return;
+    if (!details.isEmpty()) {
+        auto textNode = new QTestElement(QTest::LET_Text);
+        textNode->addAttribute(QTest::AI_Value, details.toUtf8().constData());
+        failureElement->addChild(textNode);
     }
 
-    if (!tag) {
-        tag = "";
-    }
-    if (!gtag) {
-        gtag = "";
-    }
+    currentTestCase->addChild(failureElement);
 
-    QTestCharBuffer buf;
-    QTest::qt_asprintf(&buf, "%s%s%s", gtag, filler, tag);
-    element->addAttribute(QTest::AI_Tag, buf.constData());
+    switch (elementType) {
+    case QTest::LET_Failure: ++failureCounter; break;
+    case QTest::LET_Error: ++errorCounter; break;
+    default: Q_UNREACHABLE();
+    }
 }
 
 void QJUnitTestLogger::addMessage(MessageTypes type, const QString &message, const char *file, int line)
 {
-    auto messageElement = new QTestElement(QTest::LET_Message);
-    auto systemLogElement = systemOutputElement;
-    const char *typeBuf = nullptr;
-
-    switch (type) {
-    case QAbstractTestLogger::Warn:
-        systemLogElement = systemErrorElement;
-        typeBuf = "warn";
-        break;
-    case QAbstractTestLogger::QSystem:
-        typeBuf = "system";
-        break;
-    case QAbstractTestLogger::QDebug:
-        typeBuf = "qdebug";
-        break;
-    case QAbstractTestLogger::QInfo:
-        typeBuf = "qinfo";
-        break;
-    case QAbstractTestLogger::QWarning:
-        systemLogElement = systemErrorElement;
-        typeBuf = "qwarn";
-        break;
-    case QAbstractTestLogger::QFatal:
-        systemLogElement = systemErrorElement;
-        typeBuf = "qfatal";
-        break;
-    case QAbstractTestLogger::Skip:
-        typeBuf = "skip";
-        break;
-    case QAbstractTestLogger::Info:
-        typeBuf = "info";
-        break;
-    default:
-        typeBuf = "??????";
-        break;
-    }
-
-    messageElement->addAttribute(QTest::AI_Type, typeBuf);
-    messageElement->addAttribute(QTest::AI_Description, message.toUtf8().constData());
-    addTag(messageElement);
+    Q_UNUSED(file);
+    Q_UNUSED(line);
 
-    if (file)
-        messageElement->addAttribute(QTest::AI_File, file);
-    else
-        messageElement->addAttribute(QTest::AI_File, "");
+    if (type == QFatal) {
+        addFailure(QTest::LET_Error, "qfatal", message);
+        return;
+    }
 
-    char buf[100];
-    qsnprintf(buf, sizeof(buf), "%i", line);
-    messageElement->addAttribute(QTest::AI_Line, buf);
+    auto systemLogElement = [&]() {
+        switch (type) {
+        case QAbstractTestLogger::QDebug:
+        case QAbstractTestLogger::Info:
+        case QAbstractTestLogger::QInfo:
+            return systemOutputElement;
+        case QAbstractTestLogger::Warn:
+        case QAbstractTestLogger::QWarning:
+        case QAbstractTestLogger::QCritical:
+            return systemErrorElement;
+        default:
+            Q_UNREACHABLE();
+        }
+    }();
 
-    currentLogElement->addLogElement(messageElement);
-    ++errorCounter;
+    if (!systemLogElement)
+        return; // FIXME: Handle messages outside of test functions
 
-    // Also add the message to the system log (stdout/stderr), if one exists
-    if (systemLogElement) {
-        auto messageElement = new QTestElement(QTest::LET_Message);
-        messageElement->addAttribute(QTest::AI_Description, message.toUtf8().constData());
-        systemLogElement->addLogElement(messageElement);
-    }
+    auto textNode = new QTestElement(QTest::LET_Text);
+    textNode->addAttribute(QTest::AI_Value, message.toUtf8().constData());
+    systemLogElement->addChild(textNode);
 }
 
 QT_END_NAMESPACE