Tweak a selftest blacklist to exercise more of the blacklisting code

In the process, corrected an inaccurate XFAIL message (an XPASS is
normally an error, unless blacklisting ignores it so turns it into a
BPASS). Added the missing .tap file to its expected output.
Documented the similarity to the silent/ selftest.

Task-number: QTPM-1385
Change-Id: Id74a1353d54af2f3bfe2c764e33c1f051958ab21
Reviewed-by: Jędrzej Nowacki <jedrzej.nowacki@qt.io>

8 files changed, 63 insertions, 11 deletions

diff --git a/tests/auto/testlib/selftests/blacklisted/BLACKLIST b/tests/auto/testlib/selftests/blacklisted/BLACKLIST
index 36b7699cbd..a923c11416 100644
--- a/tests/auto/testlib/selftests/blacklisted/BLACKLIST
+++ b/tests/auto/testlib/selftests/blacklisted/BLACKLIST
@@ -1,12 +1,20 @@
-[pass]
+obscure # no such platform; is ignored
 *
+
+[pass]
+!*
+
 [skip]
 *
+
 [fail]
 *
-[xpass]
-*
+
 [xfail]
 *
+
+[xpass]
+*
+
 [messages]
 *
diff --git a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
index 8578752e22..90520385ec 100644
--- a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+++ b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
@@ -45,7 +45,8 @@ private slots:
     void messages();
 };
 
-// All the tests below have been blacklisted in blacklisted/BLACKLIST
+// All the tests below except pass() have been blacklisted in blacklisted/BLACKLIST
+// Contrast with ../silent/, for the same tests without blacklisting but with -silent
 
 void tst_Blacklisted::pass()
 {
@@ -64,7 +65,7 @@ void tst_Blacklisted::fail()
 
 void tst_Blacklisted::xfail()
 {
-    QEXPECT_FAIL("", "This test should XFAIL then BFAIL", Abort);
+    QEXPECT_FAIL("", "This test should XFAIL then BPASS", Abort);
     QVERIFY(false);
 }
 
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.lightxml b/tests/auto/testlib/selftests/expected_blacklisted.lightxml
index 4193628e7c..98d7a38ca9 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.lightxml
+++ b/tests/auto/testlib/selftests/expected_blacklisted.lightxml
@@ -25,7 +25,7 @@
 </TestFunction>
 <TestFunction name="xfail">
 <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
-    <Description><![CDATA[This test should XFAIL then BFAIL]]></Description>
+    <Description><![CDATA[This test should XFAIL then BPASS]]></Description>
 </Incident>
 <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.tap b/tests/auto/testlib/selftests/expected_blacklisted.tap
new file mode 100644
index 0000000000..f26155ded0
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_blacklisted.tap
@@ -0,0 +1,43 @@
+TAP version 13
+# tst_Blacklisted
+ok 1 - initTestCase()
+ok 2 - pass() # TODO 
+ok 3 - skip() # SKIP This test should SKIP
+not ok 4 - fail() # TODO 'false' returned FALSE. (This test should BFAIL)
+  ---
+  type: QVERIFY
+  message: This test should BFAIL
+  wanted: true (false)
+  found: false (false)
+  expected: true (false)
+  actual: false (false)
+  at: tst_Blacklisted::fail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:63)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 63
+  ...
+not ok 5 - xfail() # TODO This test should XFAIL then BPASS
+  ---
+  at: tst_Blacklisted::xfail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:69)
+  file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+  line: 69
+  ...
+ok 5 - xfail() # TODO 
+ok 6 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)
+# This is a warning that should not appear in silent test output
+# This is an internal testlib warning that should not appear in silent test output
+# This is a debug message that should not appear in silent test output
+# This is a critical message that should not appear in silent test output
+# This is an info message that should not appear in silent test output
+# This is an internal testlib info message that should not appear in silent test output
+# This is a fatal error message that should still appear in silent test output
+not ok 7 - messages() # TODO Received a fatal error.
+  ---
+  # Received a fatal error.
+  at: tst_Blacklisted::messages() (Unknown file:0)
+  file: Unknown file
+  line: 0
+  ...
+1..7
+# tests 7
+# pass 1
+# fail 1
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.teamcity b/tests/auto/testlib/selftests/expected_blacklisted.teamcity
index 8180a7ce76..df58208fb3 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.teamcity
+++ b/tests/auto/testlib/selftests/expected_blacklisted.teamcity
@@ -7,7 +7,7 @@
 ##teamcity[testStarted name='fail()' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='fail()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted']
-##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BFAIL' flowId='tst_Blacklisted']
+##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BPASS' flowId='tst_Blacklisted']
 ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted']
 ##teamcity[testStarted name='xpass()' flowId='tst_Blacklisted']
 ##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)' flowId='tst_Blacklisted']
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.txt b/tests/auto/testlib/selftests/expected_blacklisted.txt
index 6fa2403b59..3cfe40eb12 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.txt
+++ b/tests/auto/testlib/selftests/expected_blacklisted.txt
@@ -6,7 +6,7 @@ SKIP   : tst_Blacklisted::skip() This test should SKIP
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 BFAIL  : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL)
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
-XFAIL  : tst_Blacklisted::xfail() This test should XFAIL then BFAIL
+XFAIL  : tst_Blacklisted::xfail() This test should XFAIL then BPASS
    Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)]
 BPASS  : tst_Blacklisted::xfail()
 XPASS  : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xml b/tests/auto/testlib/selftests/expected_blacklisted.xml
index 443bc6b199..f0387bae00 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.xml
+++ b/tests/auto/testlib/selftests/expected_blacklisted.xml
@@ -27,7 +27,7 @@
 </TestFunction>
 <TestFunction name="xfail">
 <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0">
-    <Description><![CDATA[This test should XFAIL then BFAIL]]></Description>
+    <Description><![CDATA[This test should XFAIL then BPASS]]></Description>
 </Incident>
 <Incident type="bpass" file="" line="0" />
     <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml
index 2752bc18b4..e2d0cd009b 100644
--- a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml
+++ b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml
@@ -12,7 +12,7 @@
   </testcase>
   <testcase result="bfail" name="fail"/>
   <testcase result="xfail" name="xfail">
-    <!-- message="This test should XFAIL then BFAIL" type="info" -->
+    <!-- message="This test should XFAIL then BPASS" type="info" -->
   </testcase>
   <testcase result="xpass" name="xpass">
     <failure message="&apos;true&apos; returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)" result="xpass"/>
@@ -28,7 +28,7 @@
   </testcase>
   <system-err>
 <![CDATA[This test should SKIP]]>
-<![CDATA[This test should XFAIL then BFAIL]]>
+<![CDATA[This test should XFAIL then BPASS]]>
 <![CDATA[This is a warning that should not appear in silent test output]]>
 <![CDATA[This is an internal testlib warning that should not appear in silent test output]]>
 <![CDATA[This is a debug message that should not appear in silent test output]]>