diff options
Diffstat (limited to 'tests/auto')
8 files changed, 63 insertions, 11 deletions
diff --git a/tests/auto/testlib/selftests/blacklisted/BLACKLIST b/tests/auto/testlib/selftests/blacklisted/BLACKLIST index 36b7699cbd..a923c11416 100644 --- a/tests/auto/testlib/selftests/blacklisted/BLACKLIST +++ b/tests/auto/testlib/selftests/blacklisted/BLACKLIST @@ -1,12 +1,20 @@ -[pass] +obscure # no such platform; is ignored * + +[pass] +!* + [skip] * + [fail] * -[xpass] -* + [xfail] * + +[xpass] +* + [messages] * diff --git a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp index 8578752e22..90520385ec 100644 --- a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp +++ b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp @@ -45,7 +45,8 @@ private slots: void messages(); }; -// All the tests below have been blacklisted in blacklisted/BLACKLIST +// All the tests below except pass() have been blacklisted in blacklisted/BLACKLIST +// Contrast with ../silent/, for the same tests without blacklisting but with -silent void tst_Blacklisted::pass() { @@ -64,7 +65,7 @@ void tst_Blacklisted::fail() void tst_Blacklisted::xfail() { - QEXPECT_FAIL("", "This test should XFAIL then BFAIL", Abort); + QEXPECT_FAIL("", "This test should XFAIL then BPASS", Abort); QVERIFY(false); } diff --git a/tests/auto/testlib/selftests/expected_blacklisted.lightxml b/tests/auto/testlib/selftests/expected_blacklisted.lightxml index 4193628e7c..98d7a38ca9 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.lightxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.lightxml @@ -25,7 +25,7 @@ </TestFunction> <TestFunction name="xfail"> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> - <Description><![CDATA[This test should XFAIL then BFAIL]]></Description> + <Description><![CDATA[This test should XFAIL then BPASS]]></Description> </Incident> <Incident type="bpass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_blacklisted.tap b/tests/auto/testlib/selftests/expected_blacklisted.tap new file mode 100644 index 0000000000..f26155ded0 --- /dev/null +++ b/tests/auto/testlib/selftests/expected_blacklisted.tap @@ -0,0 +1,43 @@ +TAP version 13 +# tst_Blacklisted +ok 1 - initTestCase() +ok 2 - pass() # TODO +ok 3 - skip() # SKIP This test should SKIP +not ok 4 - fail() # TODO 'false' returned FALSE. (This test should BFAIL) + --- + type: QVERIFY + message: This test should BFAIL + wanted: true (false) + found: false (false) + expected: true (false) + actual: false (false) + at: tst_Blacklisted::fail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:63) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 63 + ... +not ok 5 - xfail() # TODO This test should XFAIL then BPASS + --- + at: tst_Blacklisted::xfail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:69) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 69 + ... +ok 5 - xfail() # TODO +ok 6 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS) +# This is a warning that should not appear in silent test output +# This is an internal testlib warning that should not appear in silent test output +# This is a debug message that should not appear in silent test output +# This is a critical message that should not appear in silent test output +# This is an info message that should not appear in silent test output +# This is an internal testlib info message that should not appear in silent test output +# This is a fatal error message that should still appear in silent test output +not ok 7 - messages() # TODO Received a fatal error. + --- + # Received a fatal error. + at: tst_Blacklisted::messages() (Unknown file:0) + file: Unknown file + line: 0 + ... +1..7 +# tests 7 +# pass 1 +# fail 1 diff --git a/tests/auto/testlib/selftests/expected_blacklisted.teamcity b/tests/auto/testlib/selftests/expected_blacklisted.teamcity index 8180a7ce76..df58208fb3 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.teamcity +++ b/tests/auto/testlib/selftests/expected_blacklisted.teamcity @@ -7,7 +7,7 @@ ##teamcity[testStarted name='fail()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='fail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted'] -##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BFAIL' flowId='tst_Blacklisted'] +##teamcity[testStdOut name='xfail()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]: This test should XFAIL then BPASS' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='xpass()' flowId='tst_Blacklisted'] ##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)' flowId='tst_Blacklisted'] diff --git a/tests/auto/testlib/selftests/expected_blacklisted.txt b/tests/auto/testlib/selftests/expected_blacklisted.txt index 6fa2403b59..3cfe40eb12 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.txt +++ b/tests/auto/testlib/selftests/expected_blacklisted.txt @@ -6,7 +6,7 @@ SKIP : tst_Blacklisted::skip() This test should SKIP Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BFAIL : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL) Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] -XFAIL : tst_Blacklisted::xfail() This test should XFAIL then BFAIL +XFAIL : tst_Blacklisted::xfail() This test should XFAIL then BPASS Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BPASS : tst_Blacklisted::xfail() XPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS) diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xml b/tests/auto/testlib/selftests/expected_blacklisted.xml index 443bc6b199..f0387bae00 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.xml +++ b/tests/auto/testlib/selftests/expected_blacklisted.xml @@ -27,7 +27,7 @@ </TestFunction> <TestFunction name="xfail"> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> - <Description><![CDATA[This test should XFAIL then BFAIL]]></Description> + <Description><![CDATA[This test should XFAIL then BPASS]]></Description> </Incident> <Incident type="bpass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml index 2752bc18b4..e2d0cd009b 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.xunitxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.xunitxml @@ -12,7 +12,7 @@ </testcase> <testcase result="bfail" name="fail"/> <testcase result="xfail" name="xfail"> - <!-- message="This test should XFAIL then BFAIL" type="info" --> + <!-- message="This test should XFAIL then BPASS" type="info" --> </testcase> <testcase result="xpass" name="xpass"> <failure message="'true' returned TRUE unexpectedly. (This test should XPASS, blacklist ignored for XPASS)" result="xpass"/> @@ -28,7 +28,7 @@ </testcase> <system-err> <![CDATA[This test should SKIP]]> -<![CDATA[This test should XFAIL then BFAIL]]> +<![CDATA[This test should XFAIL then BPASS]]> <![CDATA[This is a warning that should not appear in silent test output]]> <![CDATA[This is an internal testlib warning that should not appear in silent test output]]> <![CDATA[This is a debug message that should not appear in silent test output]]> |