From 3b42e098ef711e5c60dd18744f6ff9fa07877424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tor=20Arne=20Vestb=C3=B8?= Date: Thu, 1 Mar 2018 17:06:23 +0100 Subject: testlib: Add Test Anything Protocol (TAP) reporter The Test Anything Protocol (TAP), was originally Perl's simple text-based interface between testing modules and test harnesses, but has since been adopted by a large number of producers and consumers in many different languages, which allows colorizing and summarizing test results. The format is very simple: TAP version 13 ok 1 - test description not ok 2 - test description --- message: 'Failure message' severity: fail expected: 123 actual: 456 ... ok 3 - test description # SKIP 1..3 The specification [1] is very brief, so the implementation has been based on how typical consumers behave, especially when it comes to the undefined diagnostics block. [1] http://testanything.org/tap-version-13-specification.html Change-Id: I616e802ea380165c678510e940ddc6607d39c92d Reviewed-by: Simon Hausmann --- src/testlib/qtaptestlogger.cpp | 254 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 src/testlib/qtaptestlogger.cpp (limited to 'src/testlib/qtaptestlogger.cpp') diff --git a/src/testlib/qtaptestlogger.cpp b/src/testlib/qtaptestlogger.cpp new file mode 100644 index 0000000000..37ab89ac91 --- /dev/null +++ b/src/testlib/qtaptestlogger.cpp @@ -0,0 +1,254 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qtaptestlogger_p.h" + +#include "qtestlog_p.h" +#include "qtestresult_p.h" +#include "qtestassert.h" + +#include + +QT_BEGIN_NAMESPACE + +QTapTestLogger::QTapTestLogger(const char *filename) + : QAbstractTestLogger(filename) + , m_wasExpectedFail(false) +{ +} + +QTapTestLogger::~QTapTestLogger() +{ +} + +void QTapTestLogger::startLogging() +{ + QAbstractTestLogger::startLogging(); + + QTestCharBuffer preamble; + QTest::qt_asprintf(&preamble, "TAP version 13\n" + // By convention, test suite names are output as diagnostics lines + // This is a pretty poor convention, as consumers will then treat + // actual diagnostics, e.g. qDebug, as test suite names o_O + "# %s\n", QTestResult::currentTestObjectName()); + outputString(preamble.data()); +} + +void QTapTestLogger::stopLogging() +{ + const int total = QTestLog::totalCount(); + + QTestCharBuffer testPlanAndStats; + QTest::qt_asprintf(&testPlanAndStats, + "1..%d\n" + "# tests %d\n" + "# pass %d\n" + "# fail %d\n", + total, total, QTestLog::passCount(), QTestLog::failCount()); + outputString(testPlanAndStats.data()); + + QAbstractTestLogger::stopLogging(); +} + +void QTapTestLogger::enterTestFunction(const char *function) +{ + Q_UNUSED(function); + m_wasExpectedFail = false; +} + +void QTapTestLogger::enterTestData(QTestData *data) +{ + Q_UNUSED(data); + m_wasExpectedFail = false; +} + +using namespace QTestPrivate; + +void QTapTestLogger::outputTestLine(bool ok, int testNumber, QTestCharBuffer &directive) +{ + QTestCharBuffer testIdentifier; + QTestPrivate::generateTestIdentifier(&testIdentifier, TestFunction | TestDataTag); + + QTestCharBuffer testLine; + QTest::qt_asprintf(&testLine, "%s %d - %s%s\n", + ok ? "ok" : "not ok", testNumber, testIdentifier.data(), directive.data()); + + outputString(testLine.data()); +} + +void QTapTestLogger::addIncident(IncidentTypes type, const char *description, + const char *file, int line) +{ + if (m_wasExpectedFail && type == Pass) { + // XFail comes with a corresponding Pass incident, but we only want + // to emit a single test point for it, so skip the this pass. + return; + } + + bool ok = type == Pass || type == XPass || type == BlacklistedPass; + + QTestCharBuffer directive; + if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass) + // We treat expected or blacklisted failures/passes as TODO-failures/passes, + // which should be treated as soft issues by consumers. Not all do though :/ + QTest::qt_asprintf(&directive, " # TODO %s", description); + + int testNumber = QTestLog::totalCount(); + if (type == XFail) { + // The global test counter hasn't been updated yet for XFail + testNumber += 1; + } + + outputTestLine(ok, testNumber, directive); + + if (!ok) { + // All failures need a diagnostics sections to not confuse consumers + + // The indent needs to be two spaces for maximum compatibility + #define YAML_INDENT " " + + outputString(YAML_INDENT "---\n"); + + if (type != XFail) { + // This is fragile, but unfortunately testlib doesn't plumb + // the expected and actual values to the loggers (yet). + static QRegularExpression verifyRegex( + QLatin1Literal("^'(?.*)' returned (?\\w+).+\\((?.*)\\)$")); + + static QRegularExpression comparRegex( + QLatin1Literal("^(?.*)\n" + "\\s*Actual\\s+\\((?.*)\\)\\s*: (?.*)\n" + "\\s*Expected\\s+\\((?.*)\\)\\s*: (?.*)$")); + + QString descriptionString = QString::fromUtf8(description); + QRegularExpressionMatch match = verifyRegex.match(descriptionString); + if (!match.hasMatch()) + match = comparRegex.match(descriptionString); + + if (match.hasMatch()) { + bool isVerify = match.regularExpression() == verifyRegex; + QString message = match.captured(QLatin1Literal("message")); + QString expected; + QString actual; + + if (isVerify) { + QString expression = QLatin1Literal(" (") + % match.captured(QLatin1Literal("actualexpression")) % QLatin1Char(')') ; + actual = match.captured(QLatin1Literal("actual")).toLower() % expression; + expected = (actual.startsWith(QLatin1Literal("true")) ? QLatin1Literal("false") : QLatin1Literal("true")) % expression; + if (message.isEmpty()) + message = QLatin1Literal("Verification failed"); + } else { + expected = match.captured(QLatin1Literal("expected")) + % QLatin1Literal(" (") % match.captured(QLatin1Literal("expectedexpresssion")) % QLatin1Char(')'); + actual = match.captured(QLatin1Literal("actual")) + % QLatin1Literal(" (") % match.captured(QLatin1Literal("actualexpression")) % QLatin1Char(')'); + } + + QTestCharBuffer diagnosticsYamlish; + QTest::qt_asprintf(&diagnosticsYamlish, + YAML_INDENT "type: %s\n" + YAML_INDENT "message: %s\n" + + // Some consumers understand 'wanted/found', while others need + // 'expected/actual', so we do both for maximum compatibility. + YAML_INDENT "wanted: %s\n" + YAML_INDENT "found: %s\n" + YAML_INDENT "expected: %s\n" + YAML_INDENT "actual: %s\n", + + isVerify ? "QVERIFY" : "QCOMPARE", + qPrintable(message), + qPrintable(expected), qPrintable(actual), + qPrintable(expected), qPrintable(actual) + ); + + outputString(diagnosticsYamlish.data()); + } else { + QTestCharBuffer unparsableDescription; + QTest::qt_asprintf(&unparsableDescription, + YAML_INDENT "# %s\n", description); + outputString(unparsableDescription.data()); + } + } + + if (file) { + QTestCharBuffer location; + QTest::qt_asprintf(&location, + // The generic 'at' key is understood by most consumers. + YAML_INDENT "at: %s::%s() (%s:%d)\n" + + // The file and line keys are for consumers that are able + // to read more granular location info. + YAML_INDENT "file: %s\n" + YAML_INDENT "line: %d\n", + + QTestResult::currentTestObjectName(), + QTestResult::currentTestFunction(), + file, line, file, line + ); + outputString(location.data()); + } + + outputString(YAML_INDENT "...\n"); + } + + m_wasExpectedFail = type == XFail; +} + +void QTapTestLogger::addMessage(MessageTypes type, const QString &message, + const char *file, int line) +{ + Q_UNUSED(file); + Q_UNUSED(line); + + if (type == Skip) { + QTestCharBuffer directive; + QTest::qt_asprintf(&directive, " # SKIP %s", message.toUtf8().constData()); + outputTestLine(/* ok = */ true, QTestLog::totalCount(), directive); + return; + } + + QTestCharBuffer diagnostics; + QTest::qt_asprintf(&diagnostics, "# %s\n", qPrintable(message)); + outputString(diagnostics.data()); +} + +QT_END_NAMESPACE + -- cgit v1.2.3