testlib: Add BXPASS and BXFAIL

Prioritize blacklisting over QEXPECT_FAIL so that a test that is
blacklisted no longer fails if QEXPECT_FAIL returns true unexpectedly. To
reflect this state properly, the two values of BXPASS and BXFAIL were
added to testlib's output.

[ChangeLog][Important Behavior Changes][QtTestLib] Blacklisting of tests
will be taken into account for XPASS and XFAIL. A blacklisted test that
causes an XPASS will no longer be a fail.

Task-number: QTBUG-72928
Change-Id: Ia2232fdc714d405fa3fd9aea6c89eb2836bc5950
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

6 files changed, 18 insertions, 24 deletions

diff --git a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
index 8578752e22..b25489ca00 100644
--- a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+++ b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
@@ -64,14 +64,14 @@ void tst_Blacklisted::fail()
 
 void tst_Blacklisted::xfail()
 {
-    QEXPECT_FAIL("", "This test should XFAIL then BFAIL", Abort);
+    QEXPECT_FAIL("", "This test should BXFAIL then BPASS", Abort);
     QVERIFY(false);
 }
 
 void tst_Blacklisted::xpass()
 {
-    QEXPECT_FAIL("", "This test should XPASS", Abort);
-    QVERIFY2(true, "This test should XPASS, blacklist ignored for XPASS");
+    QEXPECT_FAIL("", "This test should BXPASS", Abort);
+    QVERIFY2(true, "This test should BXPASS");
 }
 
 void tst_Blacklisted::messages()
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.lightxml b/tests/auto/testlib/selftests/expected_blacklisted.lightxml
index 4193628e7c..5cf62ed104 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.lightxml
+++ b/tests/auto/testlib/selftests/expected_blacklisted.lightxml
@@ -24,15 +24,15 @@
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="xfail">
-<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
-    <Description><![CDATA[This test should XFAIL then BFAIL]]></Description>
+<Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This test should BXFAIL then BPASS]]></Description>
 </Incident>
 <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="xpass">
-<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
-    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)]]></Description>
+<Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS)]]></Description>
 </Incident>
     <Duration msecs="0"/>
 </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.teamcity b/tests/auto/testlib/selftests/expected_blacklisted.teamcity
index 8180a7ce76..42b75b752b 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.teamcity
+++ b/tests/auto/testlib/selftests/expected_blacklisted.teamcity
@@ -7,10 +7,9 @@
 ##teamcity[testStarted name='fail()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='fail()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted']
-##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BFAIL' flowId='tst_Blacklisted']
+##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='xpass()' flowId='tst_Blacklisted']
-##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xpass()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='messages()' flowId='tst_Blacklisted']
 ##teamcity[testStdOut name='messages()' out='QWARN: This is a warning that should not appear in silent test output|nWARNING |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This is an internal testlib warning that should not appear in silent test output|nQDEBUG: This is a debug message that should not appear in silent test output|nQSYSTEM: This is a critical message that should not appear in silent test output|nQINFO: This is an info message that should not appear in silent test output|nINFO |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This is an internal testlib info message that should not appear in silent test output|nQFATAL: This is a fatal error message that should still appear in silent test output' flowId='tst_Blacklisted']
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.txt b/tests/auto/testlib/selftests/expected_blacklisted.txt
index 6fa2403b59..fccaa7d8c3 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.txt
+++ b/tests/auto/testlib/selftests/expected_blacklisted.txt
@@ -6,10 +6,10 @@ SKIP   : tst_Blacklisted::skip() This test should SKIP
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 BFAIL  : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL)
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
-XFAIL  : tst_Blacklisted::xfail() This test should XFAIL then BFAIL
+BXFAIL : tst_Blacklisted::xfail() This test should BXFAIL then BPASS
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 BPASS  : tst_Blacklisted::xfail()
-XPASS  : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)
+BXPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should BXPASS)
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 QWARN  : tst_Blacklisted::messages() This is a warning that should not appear in silent test output
 WARNING: tst_Blacklisted::messages() This is an internal testlib warning that should not appear in silent test output
@@ -22,5 +22,5 @@ INFO   : tst_Blacklisted::messages() This is an internal testlib info message th
 QFATAL : tst_Blacklisted::messages() This is a fatal error message that should still appear in silent test output
 BFAIL  : tst_Blacklisted::messages() Received a fatal error.
    Loc: [Unknown file(0)]
-Totals: 1 passed, 1 failed, 1 skipped, 4 blacklisted, 0ms
+Totals: 1 passed, 0 failed, 1 skipped, 5 blacklisted, 0ms
 ********* Finished testing of tst_Blacklisted *********
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xml b/tests/auto/testlib/selftests/expected_blacklisted.xml
index 443bc6b199..04d7e6c828 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.xml
+++ b/tests/auto/testlib/selftests/expected_blacklisted.xml
@@ -26,15 +26,15 @@
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="xfail">
-<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
-    <Description><![CDATA[This test should XFAIL then BFAIL]]></Description>
+<Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA[This test should BXFAIL then BPASS]]></Description>
 </Incident>
 <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
 </TestFunction>
 <TestFunction name="xpass">
-<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
-    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)]]></Description>
+<Incident type="bxpass" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
+    <Description><![CDATA['true' returned TRUE unexpectedly. (This test should BXPASS)]]></Description>
 </Incident>
     <Duration msecs="0"/>
 </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml
index 2752bc18b4..6e192687fb 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml
+++ b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite errors="9" failures="3" tests="7" name="tst_Blacklisted">
+<testsuite errors="8" failures="3" tests="7" name="tst_Blacklisted">
   <properties>
     <property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
     <property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
@@ -11,12 +11,8 @@
     <!-- message="This test should SKIP" type="skip" -->
   </testcase>
   <testcase result="bfail" name="fail"/>
-  <testcase result="xfail" name="xfail">
-    <!-- message="This test should XFAIL then BFAIL" type="info" -->
-  </testcase>
-  <testcase result="xpass" name="xpass">
-    <failure message="&apos;true&apos; returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)" result="xpass"/>
-  </testcase>
+  <testcase result="bxfail" name="xfail"/>
+  <testcase result="bxpass" name="xpass"/>
   <testcase result="bfail" name="messages">
     <!-- message="This is a warning that should not appear in silent test output" type="qwarn" -->
     <!-- message="This is an internal testlib warning that should not appear in silent test output" type="warn" -->
@@ -28,7 +24,6 @@
   </testcase>
   <system-err>
 <![CDATA[This test should SKIP]]>
-<![CDATA[This test should XFAIL then BFAIL]]>
 <![CDATA[This is a warning that should not appear in silent test output]]>
 <![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
 <![CDATA[This is a debug message that should not appear in silent test output]]>