testlib: Add BXPASS and BXFAIL

Prioritize blacklisting over QEXPECT_FAIL so that a test that is
blacklisted no longer fails if QEXPECT_FAIL returns true unexpectedly. To
reflect this state properly, the two values of BXPASS and BXFAIL were
added to testlib's output.

[ChangeLog][Important Behavior Changes][QtTestLib] Blacklisting of tests
will be taken into account for XPASS and XFAIL. A blacklisted test that
causes an XPASS will no longer be a fail.

Task-number: QTBUG-72928
Change-Id: Ia2232fdc714d405fa3fd9aea6c89eb2836bc5950
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

10 files changed, 63 insertions, 9 deletions

diff --git a/src/testlib/qabstracttestlogger_p.h b/src/testlib/qabstracttestlogger_p.h
index 018361b81e..a64e7ea96f 100644
--- a/src/testlib/qabstracttestlogger_p.h
+++ b/src/testlib/qabstracttestlogger_p.h
@@ -69,7 +69,9 @@ public:
         Fail,
         XPass,
         BlacklistedPass,
-        BlacklistedFail
+        BlacklistedFail,
+        BlacklistedXPass,
+        BlacklistedXFail
     };
 
     enum MessageTypes {
diff --git a/src/testlib/qappletestlogger.cpp b/src/testlib/qappletestlogger.cpp
index 2c1005ad80..8e75da88f8 100644
--- a/src/testlib/qappletestlogger.cpp
+++ b/src/testlib/qappletestlogger.cpp
@@ -101,6 +101,10 @@ static IncidentClassification incidentTypeToClassification(QAbstractTestLogger::
         return IncidentClassification(QtWarningMsg, "bpass");
     case QAbstractTestLogger::BlacklistedFail:
         return IncidentClassification(QtInfoMsg, "bfail");
+    case QAbstractTestLogger::BlacklistedXPass:
+        return IncidentClassification(QtWarningMsg, "bxpass");
+    case QAbstractTestLogger::BlacklistedXFail:
+        return IncidentClassification(QtInfoMsg, "bxfail");
     }
     return IncidentClassification(QtFatalMsg, nullptr);
 }
diff --git a/src/testlib/qplaintestlogger.cpp b/src/testlib/qplaintestlogger.cpp
index 853515f2d9..ed53dcdde8 100644
--- a/src/testlib/qplaintestlogger.cpp
+++ b/src/testlib/qplaintestlogger.cpp
@@ -89,6 +89,10 @@ namespace QTest {
             return "BPASS  ";
         case QAbstractTestLogger::BlacklistedFail:
             return "BFAIL  ";
+        case QAbstractTestLogger::BlacklistedXPass:
+            return "BXPASS ";
+        case QAbstractTestLogger::BlacklistedXFail:
+            return "BXFAIL ";
         }
         return "??????";
     }
diff --git a/src/testlib/qtaptestlogger.cpp b/src/testlib/qtaptestlogger.cpp
index a81d9d5c8b..540b36e273 100644
--- a/src/testlib/qtaptestlogger.cpp
+++ b/src/testlib/qtaptestlogger.cpp
@@ -123,13 +123,15 @@ void QTapTestLogger::addIncident(IncidentTypes type, const char *description,
         return;
     }
 
-    bool ok = type == Pass || type == XPass || type == BlacklistedPass;
+    bool ok = type == Pass || type == XPass || type == BlacklistedPass || type == BlacklistedXPass;
 
     QTestCharBuffer directive;
-    if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass)
+    if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass
+            || type == BlacklistedXFail || type == BlacklistedXPass) {
         // We treat expected or blacklisted failures/passes as TODO-failures/passes,
         // which should be treated as soft issues by consumers. Not all do though :/
         QTest::qt_asprintf(&directive, " # TODO %s", description);
+    }
 
     int testNumber = QTestLog::totalCount();
     if (type == XFail) {
diff --git a/src/testlib/qteamcitylogger.cpp b/src/testlib/qteamcitylogger.cpp
index 9cfbe92b7d..577c8e70cd 100644
--- a/src/testlib/qteamcitylogger.cpp
+++ b/src/testlib/qteamcitylogger.cpp
@@ -66,6 +66,10 @@ namespace QTest {
             return "BPASS";
         case QAbstractTestLogger::BlacklistedFail:
             return "BFAIL";
+        case QAbstractTestLogger::BlacklistedXPass:
+            return "BXPASS";
+        case QAbstractTestLogger::BlacklistedXFail:
+            return "BXFAIL";
         }
         return "??????";
     }
diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp
index 1268730cc6..3285a6d8a7 100644
--- a/src/testlib/qtestlog.cpp
+++ b/src/testlib/qtestlog.cpp
@@ -455,6 +455,24 @@ void QTestLog::addBFail(const char *msg, const char *file, int line)
     QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line);
 }
 
+void QTestLog::addBXPass(const char *msg, const char *file, int line)
+{
+    QTEST_ASSERT(msg);
+    QTEST_ASSERT(file);
+
+    ++QTest::blacklists;
+
+    QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedXPass, msg, file, line);
+}
+
+void QTestLog::addBXFail(const char *msg, const char *file, int line)
+{
+    QTEST_ASSERT(msg);
+    QTEST_ASSERT(file);
+
+    QTest::TestLoggers::addIncident(QAbstractTestLogger::BlacklistedXFail, msg, file, line);
+}
+
 void QTestLog::addSkip(const char *msg, const char *file, int line)
 {
     QTEST_ASSERT(msg);
diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h
index 600c078ce2..0bdd6290e1 100644
--- a/src/testlib/qtestlog_p.h
+++ b/src/testlib/qtestlog_p.h
@@ -80,6 +80,8 @@ public:
     static void addXPass(const char *msg, const char *file, int line);
     static void addBPass(const char *msg);
     static void addBFail(const char *msg, const char *file, int line);
+    static void addBXPass(const char *msg, const char *file, int line);
+    static void addBXFail(const char *msg, const char *file, int line);
     static void addSkip(const char *msg, const char *file, int line);
     static void addBenchmarkResult(const QBenchmarkResult &result);
 
diff --git a/src/testlib/qtestresult.cpp b/src/testlib/qtestresult.cpp
index 88e3407c90..a7a4807e06 100644
--- a/src/testlib/qtestresult.cpp
+++ b/src/testlib/qtestresult.cpp
@@ -218,17 +218,24 @@ static bool checkStatement(bool statement, const char *msg, const char *file, in
 {
     if (statement) {
         if (QTest::expectFailMode) {
-            QTestLog::addXPass(msg, file, line);
+            if (QTest::blacklistCurrentTest)
+                QTestLog::addBXPass(msg, file, line);
+            else
+                QTestLog::addXPass(msg, file, line);
+
+            QTest::failed = true;
             bool doContinue = (QTest::expectFailMode == QTest::Continue);
             clearExpectFail();
-            QTest::failed = true;
             return doContinue;
         }
         return true;
     }
 
     if (QTest::expectFailMode) {
-        QTestLog::addXFail(QTest::expectFailComment, file, line);
+        if (QTest::blacklistCurrentTest)
+            QTestLog::addBXFail(QTest::expectFailComment, file, line);
+        else
+            QTestLog::addXFail(QTest::expectFailComment, file, line);
         bool doContinue = (QTest::expectFailMode == QTest::Continue);
         clearExpectFail();
         return doContinue;
diff --git a/src/testlib/qxmltestlogger.cpp b/src/testlib/qxmltestlogger.cpp
index 7153c016c5..c47042c3a0 100644
--- a/src/testlib/qxmltestlogger.cpp
+++ b/src/testlib/qxmltestlogger.cpp
@@ -91,6 +91,10 @@ namespace QTest {
             return "bpass";
         case QAbstractTestLogger::BlacklistedFail:
             return "bfail";
+        case QAbstractTestLogger::BlacklistedXPass:
+            return "bxpass";
+        case QAbstractTestLogger::BlacklistedXFail:
+            return "bxfail";
         }
         return "??????";
     }
diff --git a/src/testlib/qxunittestlogger.cpp b/src/testlib/qxunittestlogger.cpp
index ec33c29ae5..336edb5994 100644
--- a/src/testlib/qxunittestlogger.cpp
+++ b/src/testlib/qxunittestlogger.cpp
@@ -180,6 +180,13 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
         ++failureCounter;
         typeBuf = "bfail";
         break;
+    case QAbstractTestLogger::BlacklistedXPass:
+        typeBuf = "bxpass";
+        break;
+    case QAbstractTestLogger::BlacklistedXFail:
+        ++failureCounter;
+        typeBuf = "bxfail";
+        break;
     default:
         typeBuf = "??????";
         break;
@@ -212,11 +219,11 @@ void QXunitTestLogger::addIncident(IncidentTypes type, const char *description,
         if (!strcmp(oldResult, "pass")) {
             overwrite = true;
         }
-        else if (!strcmp(oldResult, "bpass")) {
+        else if (!strcmp(oldResult, "bpass") || !strcmp(oldResult, "bxfail")) {
             overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail)
-                    || (type == QAbstractTestLogger::BlacklistedFail);
+                    || (type == QAbstractTestLogger::BlacklistedFail) || (type == QAbstractTestLogger::BlacklistedXPass);
         }
-        else if (!strcmp(oldResult, "bfail")) {
+        else if (!strcmp(oldResult, "bfail") || !strcmp(oldResult, "bxpass")) {
             overwrite = (type == QAbstractTestLogger::XPass || type == QAbstractTestLogger::Fail) || (type == QAbstractTestLogger::XFail);
         }
         else if (!strcmp(oldResult, "xfail")) {