From 1bdc35bfe05ac5385278d95a052ac51c1bc7bba6 Mon Sep 17 00:00:00 2001 From: Edward Welbourne Date: Mon, 16 Aug 2021 16:56:04 +0200 Subject: Add more tests of blacklisting, combined with XPASS and XFAIL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Include counting of test types, to catch two more cases where totals don't add up. Task-number: QTBUG-95661 Change-Id: I9fe5424bc6652c61a065bf2889333e2ed9437c81 Reviewed-by: Mårten Nordheim Reviewed-by: Tor Arne Vestbø --- tests/auto/testlib/selftests/blacklisted/BLACKLIST | 12 ++++ .../selftests/blacklisted/tst_blacklisted.cpp | 65 +++++++++++++++++++++- .../selftests/expected_blacklisted.junitxml | 22 +++++++- .../selftests/expected_blacklisted.lightxml | 42 ++++++++++++++ .../testlib/selftests/expected_blacklisted.tap | 42 ++++++++++++-- .../selftests/expected_blacklisted.teamcity | 14 +++++ .../testlib/selftests/expected_blacklisted.txt | 20 ++++++- .../testlib/selftests/expected_blacklisted.xml | 42 ++++++++++++++ 8 files changed, 250 insertions(+), 9 deletions(-) (limited to 'tests/auto/testlib') diff --git a/tests/auto/testlib/selftests/blacklisted/BLACKLIST b/tests/auto/testlib/selftests/blacklisted/BLACKLIST index d652129db3..edda648921 100644 --- a/tests/auto/testlib/selftests/blacklisted/BLACKLIST +++ b/tests/auto/testlib/selftests/blacklisted/BLACKLIST @@ -12,5 +12,17 @@ obscure # no such platform; is ignored [xfail] * +[xfailContinueSkip] +* + +[xfailContinueFail] +* + [xpass] * + +[xpassContinueSkip] +* + +[xpassContinueFail] +* diff --git a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp index 0ac6ae19f4..cd7e529f92 100644 --- a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp +++ b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2021 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the test suite of the Qt Toolkit. @@ -35,41 +35,104 @@ class tst_Blacklisted : public QObject Q_OBJECT private slots: + void init(); + void cleanupTestCase(); void pass(); void skip(); void fail(); void xfail(); + void xfailContinueSkip(); + void xfailContinueFail(); void xpass(); + void xpassContinueSkip(); + void xpassContinueFail(); + +private: + int casesTested = 2; + // What the totals line's numbers *should* be: + int passed = 2, skipped = 0, blacklisted = 0; + // Total and passed get {init,cleanup}TestCase() in addition to the actual tests. }; +void tst_Blacklisted::init() +{ + ++casesTested; +} + +void tst_Blacklisted::cleanupTestCase() +{ + qDebug("Totals should add up to %d: %d passed, 0 failed, %d skipped, %d blacklisted", + casesTested, passed, skipped, blacklisted); +} + // All the tests below have been blacklisted in blacklisted/BLACKLIST void tst_Blacklisted::pass() { + ++blacklisted; + qDebug("This test should BPASS"); QVERIFY(true); } void tst_Blacklisted::skip() { + ++skipped; QSKIP("This test should SKIP"); } void tst_Blacklisted::fail() { + ++blacklisted; QVERIFY2(false, "This test should BFAIL"); } void tst_Blacklisted::xfail() { + ++blacklisted; QEXPECT_FAIL("", "This test should BXFAIL then BPASS", Abort); QVERIFY(false); } +void tst_Blacklisted::xfailContinueSkip() +{ + ++skipped; + QEXPECT_FAIL("", "This test should BXFAIL then SKIP", Continue); + QVERIFY(false); + QSKIP("This skip should be seen and counted"); +} + +void tst_Blacklisted::xfailContinueFail() +{ + ++blacklisted; + QEXPECT_FAIL("", "This test should BXFAIL then BFAIL", Continue); + QVERIFY(false); + QFAIL("This fail should be seen and counted as blacklisted"); +} + void tst_Blacklisted::xpass() { + ++blacklisted; QEXPECT_FAIL("", "This test should BXPASS", Abort); QVERIFY2(true, "This test should BXPASS"); } +void tst_Blacklisted::xpassContinueSkip() +{ + ++blacklisted; + QEXPECT_FAIL("", "This test should BXPASS then SKIP", Continue); + QVERIFY2(true, "This test should BXPASS then SKIP"); + // FIXME QTBUG-95661: skip gets counted + QSKIP("This skip should be seen but not counted"); +} + +void tst_Blacklisted::xpassContinueFail() +{ + ++blacklisted; + QEXPECT_FAIL("", "This test should BXPASS then BFAIL", Continue); + QVERIFY2(true, "This test should BXPASS then BFAIL"); + // FIXME QTBUG-95661: gets double-counted + QFAIL("This fail should be seen and not counted (due to prior XPASS)"); +} + QTEST_MAIN(tst_Blacklisted) #include "tst_blacklisted.moc" diff --git a/tests/auto/testlib/selftests/expected_blacklisted.junitxml b/tests/auto/testlib/selftests/expected_blacklisted.junitxml index bc268a8041..ce176d7301 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.junitxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.junitxml @@ -1,17 +1,33 @@ - + - + + + + + + + + + - + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_blacklisted.lightxml b/tests/auto/testlib/selftests/expected_blacklisted.lightxml index c05977661e..6d3931e47e 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.lightxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.lightxml @@ -8,6 +8,9 @@ + + + @@ -30,13 +33,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_blacklisted.tap b/tests/auto/testlib/selftests/expected_blacklisted.tap index f75fabbdd2..3ea8db06e1 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.tap +++ b/tests/auto/testlib/selftests/expected_blacklisted.tap @@ -1,6 +1,7 @@ TAP version 13 # tst_Blacklisted ok 1 - initTestCase() +# This test should BPASS ok 2 - pass() # TODO ok 3 - skip() # SKIP This test should SKIP not ok 4 - fail() # TODO 'false' returned FALSE. (This test should BFAIL) @@ -22,9 +23,42 @@ not ok 5 - xfail() # TODO This test should BXFAIL then BPASS file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -ok 6 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS) -ok 7 - cleanupTestCase() -1..7 -# tests 7 +not ok 6 - xfailContinueSkip() # TODO This test should BXFAIL then SKIP + --- + # This test should BXFAIL then SKIP + at: tst_Blacklisted::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +ok 6 - xfailContinueSkip() # SKIP This skip should be seen and counted +not ok 7 - xfailContinueFail() # TODO This test should BXFAIL then BFAIL + --- + # This test should BXFAIL then BFAIL + at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - xfailContinueFail() # TODO This fail should be seen and counted as blacklisted + --- + # This fail should be seen and counted as blacklisted + at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +ok 8 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS) +ok 9 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP) +ok 10 - xpassContinueSkip() # SKIP This skip should be seen but not counted +ok 11 - xpassContinueFail() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL) +not ok 12 - xpassContinueFail() # TODO This fail should be seen and not counted (due to prior XPASS) + --- + # This fail should be seen and not counted (due to prior XPASS) + at: tst_Blacklisted::xpassContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +# Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted +ok 13 - cleanupTestCase() +1..13 +# tests 13 # pass 2 # fail 0 diff --git a/tests/auto/testlib/selftests/expected_blacklisted.teamcity b/tests/auto/testlib/selftests/expected_blacklisted.teamcity index ffdd02c9c2..401b7153a3 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.teamcity +++ b/tests/auto/testlib/selftests/expected_blacklisted.teamcity @@ -2,6 +2,7 @@ ##teamcity[testStarted name='initTestCase()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='initTestCase()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='pass()' flowId='tst_Blacklisted'] +##teamcity[testStdOut name='pass()' out='QDEBUG: This test should BPASS' flowId='tst_Blacklisted'] ##teamcity[testFinished name='pass()' flowId='tst_Blacklisted'] ##teamcity[testIgnored name='skip()' message='This test should SKIP |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] ##teamcity[testStarted name='fail()' flowId='tst_Blacklisted'] @@ -9,8 +10,21 @@ ##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted'] +##teamcity[testStarted name='xfailContinueSkip()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='xfailContinueSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='xfailContinueSkip()' message='This skip should be seen and counted |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testStarted name='xfailContinueFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='xfailContinueFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='xfailContinueFail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='xpass()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xpass()' flowId='tst_Blacklisted'] +##teamcity[testStarted name='xpassContinueSkip()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='xpassContinueSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='xpassContinueSkip()' message='This skip should be seen but not counted |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testStarted name='xpassContinueFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='xpassContinueFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='xpassContinueFail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Blacklisted'] +##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted' flowId='tst_Blacklisted'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Blacklisted'] ##teamcity[testSuiteFinished name='tst_Blacklisted' flowId='tst_Blacklisted'] diff --git a/tests/auto/testlib/selftests/expected_blacklisted.txt b/tests/auto/testlib/selftests/expected_blacklisted.txt index b633578029..07c0150ee9 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.txt +++ b/tests/auto/testlib/selftests/expected_blacklisted.txt @@ -1,6 +1,7 @@ ********* Start testing of tst_Blacklisted ********* Config: Using QtTest library PASS : tst_Blacklisted::initTestCase() +QDEBUG : tst_Blacklisted::pass() This test should BPASS BPASS : tst_Blacklisted::pass() SKIP : tst_Blacklisted::skip() This test should SKIP Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] @@ -9,8 +10,25 @@ BFAIL : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL BXFAIL : tst_Blacklisted::xfail() This test should BXFAIL then BPASS Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BPASS : tst_Blacklisted::xfail() +BXFAIL : tst_Blacklisted::xfailContinueSkip() This test should BXFAIL then SKIP + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::xfailContinueSkip() This skip should be seen and counted + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BXFAIL : tst_Blacklisted::xfailContinueFail() This test should BXFAIL then BFAIL + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::xfailContinueFail() This fail should be seen and counted as blacklisted + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BXPASS : tst_Blacklisted::xpass() 'true' returned TRUE unexpectedly. (This test should BXPASS) Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BXPASS : tst_Blacklisted::xpassContinueSkip() 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP) + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::xpassContinueSkip() This skip should be seen but not counted + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BXPASS : tst_Blacklisted::xpassContinueFail() 'true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL) + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::xpassContinueFail() This fail should be seen and not counted (due to prior XPASS) + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +QDEBUG : tst_Blacklisted::cleanupTestCase() Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted PASS : tst_Blacklisted::cleanupTestCase() -Totals: 2 passed, 0 failed, 1 skipped, 4 blacklisted, 0ms +Totals: 2 passed, 0 failed, 3 skipped, 8 blacklisted, 0ms ********* Finished testing of tst_Blacklisted ********* diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xml b/tests/auto/testlib/selftests/expected_blacklisted.xml index c23ab177f6..959a87de68 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.xml +++ b/tests/auto/testlib/selftests/expected_blacklisted.xml @@ -10,6 +10,9 @@ + + + @@ -32,13 +35,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -- cgit v1.2.3