From 9906cc57ed3eed64d534f43c677bb16e08561bb6 Mon Sep 17 00:00:00 2001 From: Oliver Wolff Date: Tue, 15 Jan 2019 09:58:34 +0100 Subject: testlib: Add BXPASS and BXFAIL Prioritize blacklisting over QEXPECT_FAIL so that a test that is blacklisted no longer fails if QEXPECT_FAIL returns true unexpectedly. To reflect this state properly, the two values of BXPASS and BXFAIL were added to testlib's output. [ChangeLog][Important Behavior Changes][QtTestLib] Blacklisting of tests will be taken into account for XPASS and XFAIL. A blacklisted test that causes an XPASS will no longer be a fail. Task-number: QTBUG-72928 Change-Id: Ia2232fdc714d405fa3fd9aea6c89eb2836bc5950 Reviewed-by: Edward Welbourne --- src/testlib/qabstracttestlogger_p.h | 4 +++- src/testlib/qappletestlogger.cpp | 4 ++++ src/testlib/qplaintestlogger.cpp | 4 ++++ src/testlib/qtaptestlogger.cpp | 6 ++++-- src/testlib/qteamcitylogger.cpp | 4 ++++ src/testlib/qtestlog.cpp | 18 ++++++++++++++++++ src/testlib/qtestlog_p.h | 2 ++ src/testlib/qtestresult.cpp | 13 ++++++++++--- src/testlib/qxmltestlogger.cpp | 4 ++++ src/testlib/qxunittestlogger.cpp | 13 ++++++++++--- 10 files changed, 63 insertions(+), 9 deletions(-) (limited to 'src/testlib') diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h index 018361b81e..a64e7ea96f 100644 --- a/src/testlib/qabstracttestlogger_p.h +++ b/src/testlib/qabstracttestlogger_p.h @@ -69,7 +69,9 @@ public: Fail, XPass, BlacklistedPass, - BlacklistedFail + BlacklistedFail, + BlacklistedXPass, + BlacklistedXFail }; enum MessageTypes { diff --git a/src/testlib/qappletestlogger.cpp b/src/testlib/qappletestlogger.cpp index 2c1005ad80..8e75da88f8 100644 --- a/src/testlib/qappletestlogger.cpp +++ b/src/testlib/qappletestlogger.cpp @@ -101,6 +101,10 @@ static IncidentClassification incidentTypeToClassification(QAbstractTestLogger:: return IncidentClassification(QtWarningMsg, "bpass"); case QAbstractTestLogger::BlacklistedFail: return IncidentClassification(QtInfoMsg, "bfail"); + case QAbstractTestLogger::BlacklistedXPass: + return IncidentClassification(QtWarningMsg, "bxpass"); + case QAbstractTestLogger::BlacklistedXFail: + return IncidentClassification(QtInfoMsg, "bxfail"); } return IncidentClassification(QtFatalMsg, nullptr); } diff --git a/src/testlib/qplaintestlogger.cpp b/src/testlib/qplaintestlogger.cpp index 853515f2d9..ed53dcdde8 100644 --- a/src/testlib/qplaintestlogger.cpp +++ b/src/testlib/qplaintestlogger.cpp @@ -89,6 +89,10 @@ namespace QTest { return "BPASS "; case QAbstractTestLogger::BlacklistedFail: return "BFAIL "; + case QAbstractTestLogger::BlacklistedXPass: + return "BXPASS "; + case QAbstractTestLogger::BlacklistedXFail: + return "BXFAIL "; } return "??????"; } diff --git a/src/testlib/qtaptestlogger.cpp b/src/testlib/qtaptestlogger.cpp index a81d9d5c8b..540b36e273 100644 --- a/src/testlib/qtaptestlogger.cpp +++ b/src/testlib/qtaptestlogger.cpp @@ -123,13 +123,15 @@ void QTapTestLogger::addIncident(IncidentTypes type, const char *description, return; } - bool ok = type == Pass || type == XPass || type == BlacklistedPass; + bool ok = type == Pass || type == XPass || type == BlacklistedPass || type == BlacklistedXPass; QTestCharBuffer directive; - if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass) + if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass + || type == BlacklistedXFail || type == BlacklistedXPass) { // We treat expected or blacklisted failures/passes as TODO-failures/passes, // which should be treated as soft issues by consumers. Not all do though :/ QTest::qt_asprintf(&directive, " # TODO %s", description); + } int testNumber = QTestLog::totalCount(); if (type == XFail) { diff --git a/src/testlib/qteamcitylogger.cpp b/src/testlib/qteamcitylogger.cpp index 9cfbe92b7d..577c8e70cd 100644 --- a/src/testlib/qteamcitylogger.cpp +++ b/src/testlib/qteamcitylogger.cpp @@ -66,6 +66,10 @@ namespace QTest { return "BPASS"; case QAbstractTestLogger::BlacklistedFail: return "BFAIL"; + case QAbstractTestLogger::BlacklistedXPass: + return "BXPASS"; + case QAbstractTestLogger::BlacklistedXFail: + return "BXFAIL"; } return "??????"; } diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp index 1268730cc6..3285a6d8a7 100644 --- a/src/testlib/qtestlog.cpp +++ b/src/testlib/qtestlog.cpp @@ -455,6 +455,24 @@ void QTestLog::addBFail(const char *msg, const char *file, int line) QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line); } +void QTestLog::addBXPass(const char *msg, const char *file, int line) +{ + QTEST_ASSERT(msg); + QTEST_ASSERT(file); + + ++QTest::blacklists; + + QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedXPass, msg, file, line); +} + +void QTestLog::addBXFail(const char *msg, const char *file, int line) +{ + QTEST_ASSERT(msg); + QTEST_ASSERT(file); + + QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedXFail, msg, file, line); +} + void QTestLog::addSkip(const char *msg, const char *file, int line) { QTEST_ASSERT(msg); diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h index 600c078ce2..0bdd6290e1 100644 --- a/src/testlib/qtestlog_p.h +++ b/src/testlib/qtestlog_p.h @@ -80,6 +80,8 @@ public: static void addXPass(const char *msg, const char *file, int line); static void addBPass(const char *msg); static void addBFail(const char *msg, const char *file, int line); + static void addBXPass(const char *msg, const char *file, int line); + static void addBXFail(const char *msg, const char *file, int line); static void addSkip(const char *msg, const char *file, int line); static void addBenchmarkResult(const QBenchmarkResult &result); diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp index 88e3407c90..a7a4807e06 100644 --- a/src/testlib/qtestresult.cpp +++ b/src/testlib/qtestresult.cpp @@ -218,17 +218,24 @@ static bool checkStatement(bool statement, const char *msg, const char *file, in { if (statement) { if (QTest::expectFailMode) { - QTestLog::addXPass(msg, file, line); + if (QTest::blacklistCurrentTest) + QTestLog::addBXPass(msg, file, line); + else + QTestLog::addXPass(msg, file, line); + + QTest::failed = true; bool doContinue = (QTest::expectFailMode == QTest::Continue); clearExpectFail(); - QTest::failed = true; return doContinue; } return true; } if (QTest::expectFailMode) { - QTestLog::addXFail(QTest::expectFailComment, file, line); + if (QTest::blacklistCurrentTest) + QTestLog::addBXFail(QTest::expectFailComment, file, line); + else + QTestLog::addXFail(QTest::expectFailComment, file, line); bool doContinue = (QTest::expectFailMode == QTest::Continue); clearExpectFail(); return doContinue; diff --git a/src/testlib/qxmltestlogger.cpp b/src/testlib/qxmltestlogger.cpp index 7153c016c5..c47042c3a0 100644 --- a/src/testlib/qxmltestlogger.cpp +++ b/src/testlib/qxmltestlogger.cpp @@ -91,6 +91,10 @@ namespace QTest { return "bpass"; case QAbstractTestLogger::BlacklistedFail: return "bfail"; + case QAbstractTestLogger::BlacklistedXPass: + return "bxpass"; + case QAbstractTestLogger::BlacklistedXFail: + return "bxfail"; } return "??????"; } diff --git a/src/testlib/qxunittestlogger.cpp b/src/testlib/qxunittestlogger.cpp index ec33c29ae5..336edb5994 100644 --- a/src/testlib/qxunittestlogger.cpp +++ b/src/testlib/qxunittestlogger.cpp @@ -180,6 +180,13 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description, ++failureCounter; typeBuf = "bfail"; break; + case QAbstractTestLogger::BlacklistedXPass: + typeBuf = "bxpass"; + break; + case QAbstractTestLogger::BlacklistedXFail: + ++failureCounter; + typeBuf = "bxfail"; + break; default: typeBuf = "??????"; break; @@ -212,11 +219,11 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description, if (!strcmp(oldResult, "pass")) { overwrite = true; } - else if (!strcmp(oldResult, "bpass")) { + else if (!strcmp(oldResult, "bpass") || !strcmp(oldResult, "bxfail")) { overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail) - || (type == QAbstractTestLogger::BlacklistedFail); + || (type == QAbstractTestLogger::BlacklistedFail) || (type == QAbstractTestLogger::BlacklistedXPass); } - else if (!strcmp(oldResult, "bfail")) { + else if (!strcmp(oldResult, "bfail") || !strcmp(oldResult, "bxpass")) { overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail); } else if (!strcmp(oldResult, "xfail")) { -- cgit v1.2.3