diff options
| author | Edward Welbourne <edward.welbourne@qt.io> | 2021-09-07 15:30:34 +0200 |
|---|---|---|
| committer | Edward Welbourne <edward.welbourne@qt.io> | 2022-01-13 19:47:22 +0100 |
| commit | 13d2e13290d61fbe85c0744b6dc4115c07d4a1fa (patch) | |
| tree | def4ec6f2090046b78eb76b2fc34ea87242b97e6 /tests/auto | |
| parent | 95e49966889069ffafca0167d7766cbc40fda8eb (diff) | |
Make counts of various types of test result add up correctly
Added tests for repeated skips and failures (from within void lambdas,
to simulate skips and failures from within event handlers). These
exhibit yet more ways to count more than one outcome for a test. The
new QTest::failOnWarning() can also provoke more than one failure from
a single test, and several existing selftests exhibited various ways
for the Totals line's counts to add up to more than the number of
actual tests run.
Fixed counting so that only the first decisive incident is counted.
Tests can still report later failure or skipping, but only the first
is counted.
Added a currentTestState in qtestlog.cpp, by which it keeps track of
whether the test has resolved to a result, and clearCurrentTestState()
by which other code can reset that at the end of each test. This
brought to light various places where test-end clean-up was not being
handled - due to failure or skipping in a *_data() method or init, or
a skip in cleanup.
Fixes: QTBUG-95661
Change-Id: I5d24a37a53d3db225fa602649d8aad8f5ed6c1ad
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
Diffstat (limited to 'tests/auto')
28 files changed, 593 insertions, 82 deletions
diff --git a/tests/auto/testlib/selftests/blacklisted/BLACKLIST b/tests/auto/testlib/selftests/blacklisted/BLACKLIST index edda648921..e8c9b5a6d1 100644 --- a/tests/auto/testlib/selftests/blacklisted/BLACKLIST +++ b/tests/auto/testlib/selftests/blacklisted/BLACKLIST @@ -9,6 +9,12 @@ obscure # no such platform; is ignored [fail] * +[multiSkip] +* + +[multiFail] +* + [xfail] * diff --git a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp index cd7e529f92..90e7876f97 100644 --- a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp +++ b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp @@ -41,6 +41,8 @@ private slots: void skip(); void fail(); void xfail(); + void multiSkip(); + void multiFail(); void xfailContinueSkip(); void xfailContinueFail(); void xpass(); @@ -86,6 +88,23 @@ void tst_Blacklisted::fail() QVERIFY2(false, "This test should BFAIL"); } +void tst_Blacklisted::multiFail() // cf. ../subtest/'s similar tests +{ + ++blacklisted; + for (int i = 0; i < 10; ++i) + []() { QFAIL("This failure message should be repeated ten times"); }(); + QFAIL("But this test should only contribute one to the blacklisted count"); +} + +void tst_Blacklisted::multiSkip() +{ + // Similar to multiFail() + ++skipped; + for (int i = 0; i < 10; ++i) + []() { QSKIP("This skip should be repeated ten times"); }(); + QSKIP("But this test should only contribute one to the skip count"); +} + void tst_Blacklisted::xfail() { ++blacklisted; @@ -121,7 +140,6 @@ void tst_Blacklisted::xpassContinueSkip() ++blacklisted; QEXPECT_FAIL("", "This test should BXPASS then SKIP", Continue); QVERIFY2(true, "This test should BXPASS then SKIP"); - // FIXME QTBUG-95661: skip gets counted QSKIP("This skip should be seen but not counted"); } @@ -130,7 +148,6 @@ void tst_Blacklisted::xpassContinueFail() ++blacklisted; QEXPECT_FAIL("", "This test should BXPASS then BFAIL", Continue); QVERIFY2(true, "This test should BXPASS then BFAIL"); - // FIXME QTBUG-95661: gets double-counted QFAIL("This fail should be seen and not counted (due to prior XPASS)"); } diff --git a/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv b/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv index caa54d0905..f682c48e0f 100644 --- a/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv +++ b/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv @@ -1 +1 @@ -"threeBillionTicks","","CPUTicks",3000023453,3000023453,1 +"threeBillionTicks","","CPUTicks",3000012216,3000012216,1 diff --git a/tests/auto/testlib/selftests/expected_blacklisted.junitxml b/tests/auto/testlib/selftests/expected_blacklisted.junitxml index ce176d7301..5f7eb749cc 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.junitxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.junitxml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<testsuite name="tst_Blacklisted" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="11" failures="0" errors="0" skipped="3" time="@TEST_DURATION@"> +<testsuite name="tst_Blacklisted" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="13" failures="0" errors="0" skipped="3" time="@TEST_DURATION@"> <properties> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> @@ -16,6 +16,20 @@ </testcase> <testcase name="fail" classname="tst_Blacklisted" time="@TEST_DURATION@"/> <testcase name="xfail" classname="tst_Blacklisted" time="@TEST_DURATION@"/> + <testcase name="multiSkip" classname="tst_Blacklisted" time="@TEST_DURATION@"> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="But this test should only contribute one to the skip count"/> + </testcase> + <testcase name="multiFail" classname="tst_Blacklisted" time="@TEST_DURATION@"/> <testcase name="xfailContinueSkip" classname="tst_Blacklisted" time="@TEST_DURATION@"> <skipped message="This skip should be seen and counted"/> </testcase> @@ -27,7 +41,7 @@ <testcase name="xpassContinueFail" classname="tst_Blacklisted" time="@TEST_DURATION@"/> <testcase name="cleanupTestCase" classname="tst_Blacklisted" time="@TEST_DURATION@"> <system-out> - <![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]> + <![CDATA[Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted]]> </system-out> </testcase> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_blacklisted.lightxml b/tests/auto/testlib/selftests/expected_blacklisted.lightxml index ba5f043c32..cc22d4b089 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.lightxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.lightxml @@ -33,6 +33,78 @@ <Incident type="bpass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> + <TestFunction name="multiSkip"> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the skip count]]></Description> + </Incident> + <Duration msecs="0"/> + </TestFunction> + <TestFunction name="multiFail"> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the blacklisted count]]></Description> + </Incident> + <Duration msecs="0"/> + </TestFunction> <TestFunction name="xfailContinueSkip"> <Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> <Description><![CDATA[This test should BXFAIL then SKIP]]></Description> @@ -77,7 +149,7 @@ </TestFunction> <TestFunction name="cleanupTestCase"> <Message type="qdebug" file="" line="0"> - <Description><![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]></Description> + <Description><![CDATA[Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted]]></Description> </Message> <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_blacklisted.tap b/tests/auto/testlib/selftests/expected_blacklisted.tap index af0dab417f..63a0e90e21 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.tap +++ b/tests/auto/testlib/selftests/expected_blacklisted.tap @@ -23,32 +23,120 @@ not ok 5 - xfail() # TODO This test should BXFAIL then BPASS file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -not ok 6 - xfailContinueSkip() # TODO This test should BXFAIL then SKIP +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP But this test should only contribute one to the skip count +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO But this test should only contribute one to the blacklisted count + --- + # But this test should only contribute one to the blacklisted count + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 8 - xfailContinueSkip() # TODO This test should BXFAIL then SKIP --- # This test should BXFAIL then SKIP at: tst_Blacklisted::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -ok 6 - xfailContinueSkip() # SKIP This skip should be seen and counted -not ok 7 - xfailContinueFail() # TODO This test should BXFAIL then BFAIL +ok 8 - xfailContinueSkip() # SKIP This skip should be seen and counted +not ok 9 - xfailContinueFail() # TODO This test should BXFAIL then BFAIL --- # This test should BXFAIL then BFAIL at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -not ok 7 - xfailContinueFail() # TODO This fail should be seen and counted as blacklisted +not ok 9 - xfailContinueFail() # TODO This fail should be seen and counted as blacklisted --- # This fail should be seen and counted as blacklisted at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -ok 8 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS) -ok 9 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP) -ok 10 - xpassContinueSkip() # SKIP This skip should be seen but not counted -ok 11 - xpassContinueFail() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL) +ok 10 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS) +ok 11 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP) +ok 11 - xpassContinueSkip() # SKIP This skip should be seen but not counted +ok 12 - xpassContinueFail() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL) not ok 12 - xpassContinueFail() # TODO This fail should be seen and not counted (due to prior XPASS) --- # This fail should be seen and not counted (due to prior XPASS) @@ -56,7 +144,7 @@ not ok 12 - xpassContinueFail() # TODO This fail should be seen and not counted file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -# Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted +# Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted ok 13 - cleanupTestCase() 1..13 # tests 13 diff --git a/tests/auto/testlib/selftests/expected_blacklisted.teamcity b/tests/auto/testlib/selftests/expected_blacklisted.teamcity index a0f1833405..31790bd6b4 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.teamcity +++ b/tests/auto/testlib/selftests/expected_blacklisted.teamcity @@ -12,6 +12,41 @@ ##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted'] +##teamcity[testStarted name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='But this test should only contribute one to the skip count |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testStarted name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='xfailContinueSkip()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xfailContinueSkip()' flowId='tst_Blacklisted'] ##teamcity[testIgnored name='xfailContinueSkip()' message='This skip should be seen and counted |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] @@ -29,6 +64,6 @@ ##teamcity[testFinished name='xpassContinueFail()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xpassContinueFail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Blacklisted'] -##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted' flowId='tst_Blacklisted'] +##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted' flowId='tst_Blacklisted'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Blacklisted'] ##teamcity[testSuiteFinished name='tst_Blacklisted' flowId='tst_Blacklisted'] diff --git a/tests/auto/testlib/selftests/expected_blacklisted.txt b/tests/auto/testlib/selftests/expected_blacklisted.txt index 07c0150ee9..772ef33b73 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.txt +++ b/tests/auto/testlib/selftests/expected_blacklisted.txt @@ -10,6 +10,50 @@ BFAIL : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL BXFAIL : tst_Blacklisted::xfail() This test should BXFAIL then BPASS Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BPASS : tst_Blacklisted::xfail() +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() But this test should only contribute one to the skip count + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() But this test should only contribute one to the blacklisted count + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BXFAIL : tst_Blacklisted::xfailContinueSkip() This test should BXFAIL then SKIP Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] SKIP : tst_Blacklisted::xfailContinueSkip() This skip should be seen and counted @@ -28,7 +72,7 @@ BXPASS : tst_Blacklisted::xpassContinueFail() 'true' returned TRUE unexpectedly. Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BFAIL : tst_Blacklisted::xpassContinueFail() This fail should be seen and not counted (due to prior XPASS) Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] -QDEBUG : tst_Blacklisted::cleanupTestCase() Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted +QDEBUG : tst_Blacklisted::cleanupTestCase() Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted PASS : tst_Blacklisted::cleanupTestCase() Totals: 2 passed, 0 failed, 3 skipped, 8 blacklisted, 0ms ********* Finished testing of tst_Blacklisted ********* diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xml b/tests/auto/testlib/selftests/expected_blacklisted.xml index 319bddee6d..2ef3513786 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.xml +++ b/tests/auto/testlib/selftests/expected_blacklisted.xml @@ -35,6 +35,78 @@ <Incident type="bpass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> + <TestFunction name="multiSkip"> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the skip count]]></Description> + </Incident> + <Duration msecs="0"/> + </TestFunction> + <TestFunction name="multiFail"> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the blacklisted count]]></Description> + </Incident> + <Duration msecs="0"/> + </TestFunction> <TestFunction name="xfailContinueSkip"> <Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> <Description><![CDATA[This test should BXFAIL then SKIP]]></Description> @@ -79,7 +151,7 @@ </TestFunction> <TestFunction name="cleanupTestCase"> <Message type="qdebug" file="" line="0"> - <Description><![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]></Description> + <Description><![CDATA[Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted]]></Description> </Message> <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_expectfail.junitxml b/tests/auto/testlib/selftests/expected_expectfail.junitxml index 54f0416016..75da637ab9 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.junitxml +++ b/tests/auto/testlib/selftests/expected_expectfail.junitxml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<testsuite name="tst_ExpectFail" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="44" failures="17" errors="0" skipped="5" time="@TEST_DURATION@"> +<testsuite name="tst_ExpectFail" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="44" failures="17" errors="0" skipped="4" time="@TEST_DURATION@"> <properties> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> diff --git a/tests/auto/testlib/selftests/expected_expectfail.tap b/tests/auto/testlib/selftests/expected_expectfail.tap index c520bc25c0..8527acdaf1 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.tap +++ b/tests/auto/testlib/selftests/expected_expectfail.tap @@ -145,29 +145,29 @@ ok 32 - xpassAbortXfailContinue() # TODO 'true' returned TRUE unexpectedly. () ok 33 - xpassContinue() # TODO 'true' returned TRUE unexpectedly. () # This should be reached ok 34 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. () -ok 35 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count -ok 36 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. () -not ok 37 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals +ok 34 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count +ok 35 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. () +not ok 36 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals --- at: tst_ExpectFail::xpassContinueXfailAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -ok 37 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () -ok 38 - xpassAbortDataDrivenWithQVerify(Pass) -ok 39 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () +ok 36 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () +ok 37 - xpassAbortDataDrivenWithQVerify(Pass) +ok 38 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () # Test should Continue past XPASS # Test should simply PASS -ok 40 - xpassContinueDataDrivenWithQVerify(Pass) -ok 41 - xpassAbortDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. -ok 42 - xpassAbortDataDrivenWithQCompare(Pass) -ok 43 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. +ok 39 - xpassContinueDataDrivenWithQVerify(Pass) +ok 40 - xpassAbortDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. +ok 41 - xpassAbortDataDrivenWithQCompare(Pass) +ok 42 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. # Test should Continue past XPASS # Test should simply PASS -ok 44 - xpassContinueDataDrivenWithQCompare(Pass) +ok 43 - xpassContinueDataDrivenWithQCompare(Pass) # Totals should add up to 44: 23 passed, 17 failed, 4 skipped -ok 45 - cleanupTestCase() -1..45 -# tests 45 +ok 44 - cleanupTestCase() +1..44 +# tests 44 # pass 23 # fail 17 diff --git a/tests/auto/testlib/selftests/expected_expectfail.txt b/tests/auto/testlib/selftests/expected_expectfail.txt index 2dff4ad55f..fee6f6a30a 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.txt +++ b/tests/auto/testlib/selftests/expected_expectfail.txt @@ -110,5 +110,5 @@ QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) Test should s PASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) QDEBUG : tst_ExpectFail::cleanupTestCase() Totals should add up to 44: 23 passed, 17 failed, 4 skipped PASS : tst_ExpectFail::cleanupTestCase() -Totals: 23 passed, 17 failed, 5 skipped, 0 blacklisted, 0ms +Totals: 23 passed, 17 failed, 4 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_ExpectFail ********* diff --git a/tests/auto/testlib/selftests/expected_skipinit.lightxml b/tests/auto/testlib/selftests/expected_skipinit.lightxml index 7dd35a4ed3..c2517b07c2 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.lightxml +++ b/tests/auto/testlib/selftests/expected_skipinit.lightxml @@ -10,7 +10,6 @@ <Duration msecs="0"/> </TestFunction> <TestFunction name="cleanupTestCase"> - <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_skipinit.tap b/tests/auto/testlib/selftests/expected_skipinit.tap index 1fc38e79aa..880b4471e1 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.tap +++ b/tests/auto/testlib/selftests/expected_skipinit.tap @@ -1,8 +1,7 @@ TAP version 13 # tst_SkipInit ok 1 - initTestCase() # SKIP Skip inside initTestCase. This should skip all tests in the class. -ok 2 - cleanupTestCase() -1..2 -# tests 2 -# pass 1 +1..1 +# tests 1 +# pass 0 # fail 0 diff --git a/tests/auto/testlib/selftests/expected_skipinit.teamcity b/tests/auto/testlib/selftests/expected_skipinit.teamcity index 189953f532..4538d67f7e 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.teamcity +++ b/tests/auto/testlib/selftests/expected_skipinit.teamcity @@ -2,6 +2,4 @@ ##teamcity[testStarted name='initTestCase()' flowId='tst_SkipInit'] ##teamcity[testIgnored name='initTestCase()' message='Skip inside initTestCase. This should skip all tests in the class. |[Loc: qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(0)|]' flowId='tst_SkipInit'] ##teamcity[testFinished name='initTestCase()' flowId='tst_SkipInit'] -##teamcity[testStarted name='cleanupTestCase()' flowId='tst_SkipInit'] -##teamcity[testFinished name='cleanupTestCase()' flowId='tst_SkipInit'] ##teamcity[testSuiteFinished name='tst_SkipInit' flowId='tst_SkipInit'] diff --git a/tests/auto/testlib/selftests/expected_skipinit.txt b/tests/auto/testlib/selftests/expected_skipinit.txt index 9fc2d98006..ef5e68ac3b 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.txt +++ b/tests/auto/testlib/selftests/expected_skipinit.txt @@ -2,6 +2,5 @@ Config: Using QtTest library SKIP : tst_SkipInit::initTestCase() Skip inside initTestCase. This should skip all tests in the class. Loc: [qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(0)] -PASS : tst_SkipInit::cleanupTestCase() -Totals: 1 passed, 0 failed, 1 skipped, 0 blacklisted, 0ms +Totals: 0 passed, 0 failed, 1 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_SkipInit ********* diff --git a/tests/auto/testlib/selftests/expected_skipinit.xml b/tests/auto/testlib/selftests/expected_skipinit.xml index 7633cb9152..eb2d24213b 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.xml +++ b/tests/auto/testlib/selftests/expected_skipinit.xml @@ -12,7 +12,6 @@ <Duration msecs="0"/> </TestFunction> <TestFunction name="cleanupTestCase"> - <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_subtest.junitxml b/tests/auto/testlib/selftests/expected_subtest.junitxml index 353f739985..289053e3f9 100644 --- a/tests/auto/testlib/selftests/expected_subtest.junitxml +++ b/tests/auto/testlib/selftests/expected_subtest.junitxml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<testsuite name="tst_Subtest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="10" failures="3" errors="0" skipped="0" time="@TEST_DURATION@"> +<testsuite name="tst_Subtest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="11" failures="3" errors="0" skipped="1" time="@TEST_DURATION@"> <properties> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> @@ -82,6 +82,23 @@ <![CDATA[cleanup multiFail (null)]]> </system-out> </testcase> + <testcase name="multiSkip" classname="tst_Subtest" time="@TEST_DURATION@"> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="But this test should only contribute one to the skip count"/> + <system-out> + <![CDATA[init multiSkip (null)]]> + <![CDATA[cleanup multiSkip (null)]]> + </system-out> + </testcase> <testcase name="cleanupTestCase" classname="tst_Subtest" time="@TEST_DURATION@"> <system-out> <![CDATA[cleanupTestCase cleanupTestCase (null)]]> diff --git a/tests/auto/testlib/selftests/expected_subtest.lightxml b/tests/auto/testlib/selftests/expected_subtest.lightxml index 36c502b2f7..f666ba5b12 100644 --- a/tests/auto/testlib/selftests/expected_subtest.lightxml +++ b/tests/auto/testlib/selftests/expected_subtest.lightxml @@ -195,6 +195,48 @@ </Message> <Duration msecs="0"/> </TestFunction> + <TestFunction name="multiSkip"> + <Message type="qdebug" file="" line="0"> + <Description><![CDATA[init multiSkip (null)]]></Description> + </Message> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the skip count]]></Description> + </Incident> + <Message type="qdebug" file="" line="0"> + <Description><![CDATA[cleanup multiSkip (null)]]></Description> + </Message> + <Duration msecs="0"/> + </TestFunction> <TestFunction name="cleanupTestCase"> <Message type="qdebug" file="" line="0"> <Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description> diff --git a/tests/auto/testlib/selftests/expected_subtest.tap b/tests/auto/testlib/selftests/expected_subtest.tap index ebbc1d1067..ee69487233 100644 --- a/tests/auto/testlib/selftests/expected_subtest.tap +++ b/tests/auto/testlib/selftests/expected_subtest.tap @@ -68,70 +68,70 @@ not ok 9 - multiFail() file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 10 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 11 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 12 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 13 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 14 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 15 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 16 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 17 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 18 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 19 - multiFail() +not ok 9 - multiFail() --- # But this test should only contribute one to the failure count at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) @@ -139,9 +139,22 @@ not ok 19 - multiFail() line: 0 ... # cleanup multiFail (null) +# init multiSkip (null) +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP But this test should only contribute one to the skip count +# cleanup multiSkip (null) # cleanupTestCase cleanupTestCase (null) -ok 20 - cleanupTestCase() -1..20 -# tests 20 +ok 11 - cleanupTestCase() +1..11 +# tests 11 # pass 7 -# fail 13 +# fail 3 diff --git a/tests/auto/testlib/selftests/expected_subtest.teamcity b/tests/auto/testlib/selftests/expected_subtest.teamcity index 580e5695dd..e58444ecf1 100644 --- a/tests/auto/testlib/selftests/expected_subtest.teamcity +++ b/tests/auto/testlib/selftests/expected_subtest.teamcity @@ -49,7 +49,31 @@ ##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] ##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='But this test should only contribute one to the failure count' flowId='tst_Subtest'] ##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testStarted name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testStdOut name='multiSkip()' out='QDEBUG: cleanup multiFail (null)|nQDEBUG: init multiSkip (null)' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='But this test should only contribute one to the skip count |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Subtest'] -##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup multiFail (null)|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest'] +##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup multiSkip (null)|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Subtest'] ##teamcity[testSuiteFinished name='tst_Subtest' flowId='tst_Subtest'] diff --git a/tests/auto/testlib/selftests/expected_subtest.txt b/tests/auto/testlib/selftests/expected_subtest.txt index d35d7b4d24..365bbf087a 100644 --- a/tests/auto/testlib/selftests/expected_subtest.txt +++ b/tests/auto/testlib/selftests/expected_subtest.txt @@ -68,7 +68,31 @@ FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten ti FAIL! : tst_Subtest::multiFail() But this test should only contribute one to the failure count Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] QDEBUG : tst_Subtest::multiFail() cleanup multiFail (null) +QDEBUG : tst_Subtest::multiSkip() init multiSkip (null) +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() But this test should only contribute one to the skip count + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +QDEBUG : tst_Subtest::multiSkip() cleanup multiSkip (null) QDEBUG : tst_Subtest::cleanupTestCase() cleanupTestCase cleanupTestCase (null) PASS : tst_Subtest::cleanupTestCase() -Totals: 7 passed, 13 failed, 0 skipped, 0 blacklisted, 0ms +Totals: 7 passed, 3 failed, 1 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_Subtest ********* diff --git a/tests/auto/testlib/selftests/expected_subtest.xml b/tests/auto/testlib/selftests/expected_subtest.xml index 975ba60af7..8e4f9ae563 100644 --- a/tests/auto/testlib/selftests/expected_subtest.xml +++ b/tests/auto/testlib/selftests/expected_subtest.xml @@ -197,6 +197,48 @@ </Message> <Duration msecs="0"/> </TestFunction> + <TestFunction name="multiSkip"> + <Message type="qdebug" file="" line="0"> + <Description><![CDATA[init multiSkip (null)]]></Description> + </Message> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the skip count]]></Description> + </Incident> + <Message type="qdebug" file="" line="0"> + <Description><![CDATA[cleanup multiSkip (null)]]></Description> + </Message> + <Duration msecs="0"/> + </TestFunction> <TestFunction name="cleanupTestCase"> <Message type="qdebug" file="" line="0"> <Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description> diff --git a/tests/auto/testlib/selftests/expected_warnings.junitxml b/tests/auto/testlib/selftests/expected_warnings.junitxml index 92285a4937..1799be8279 100644 --- a/tests/auto/testlib/selftests/expected_warnings.junitxml +++ b/tests/auto/testlib/selftests/expected_warnings.junitxml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="15" failures="10" errors="0" skipped="1" time="@TEST_DURATION@"> +<testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="15" failures="10" errors="0" skipped="0" time="@TEST_DURATION@"> <properties> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> diff --git a/tests/auto/testlib/selftests/expected_warnings.tap b/tests/auto/testlib/selftests/expected_warnings.tap index 4dbebb45b3..161aa6db2b 100644 --- a/tests/auto/testlib/selftests/expected_warnings.tap +++ b/tests/auto/testlib/selftests/expected_warnings.tap @@ -47,7 +47,7 @@ Ran out of cabbage! # Ran out of tortillas! # Ran out of oil! # nope -not ok 8 - testFailOnWarnings() +not ok 7 - testFailOnWarnings() --- # Received a warning that resulted in a failure: Ran out of biscuits! @@ -55,7 +55,7 @@ Ran out of biscuits! file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -not ok 9 - testFailOnWarnings() +not ok 7 - testFailOnWarnings() --- # Received a warning that resulted in a failure: Running low on toothpaste! @@ -67,8 +67,8 @@ Running low on toothpaste! # Running low on toothpaste! # Running low on toothpaste! # Ran out of muffins! -ok 10 - testFailOnWarningsCleared() -not ok 11 - testFailOnWarningsWithData(warning1) +ok 8 - testFailOnWarningsCleared() +not ok 9 - testFailOnWarningsWithData(warning1) --- # Received a warning that resulted in a failure: warning1 @@ -79,7 +79,7 @@ warning1 # warning2 # warning3 # warning1 -not ok 12 - testFailOnWarningsWithData(warning2) +not ok 10 - testFailOnWarningsWithData(warning2) --- # Received a warning that resulted in a failure: warning2 @@ -90,7 +90,7 @@ warning2 # warning3 # warning1 # warning2 -not ok 13 - testFailOnWarningsWithData(warning3) +not ok 11 - testFailOnWarningsWithData(warning3) --- # Received a warning that resulted in a failure: warning3 @@ -98,14 +98,14 @@ warning3 file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -not ok 14 - testFailOnWarningsFailInHelper() +not ok 12 - testFailOnWarningsFailInHelper() --- # This failure message should be printed but not cause the test to abort at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -not ok 15 - testFailOnWarningsFailInHelper() +not ok 12 - testFailOnWarningsFailInHelper() --- # Received a warning that resulted in a failure: Ran out of cabbage! @@ -113,14 +113,14 @@ Ran out of cabbage! file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -not ok 16 - testFailOnWarningsFailInHelper() +not ok 12 - testFailOnWarningsFailInHelper() --- # My cabbage! :( at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -not ok 17 - testFailOnWarningsThenSkip() +not ok 13 - testFailOnWarningsThenSkip() --- # Received a warning that resulted in a failure: Ran out of cabbage! @@ -128,10 +128,10 @@ Ran out of cabbage! file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -ok 18 - testFailOnWarningsThenSkip() # SKIP My cabbage! :( -ok 19 - testFailOnWarningsAndIgnoreWarnings() -ok 20 - cleanupTestCase() -1..20 -# tests 20 +ok 13 - testFailOnWarningsThenSkip() # SKIP My cabbage! :( +ok 14 - testFailOnWarningsAndIgnoreWarnings() +ok 15 - cleanupTestCase() +1..15 +# tests 15 # pass 5 -# fail 14 +# fail 10 diff --git a/tests/auto/testlib/selftests/expected_warnings.txt b/tests/auto/testlib/selftests/expected_warnings.txt index 02361eb39d..5037eb87df 100644 --- a/tests/auto/testlib/selftests/expected_warnings.txt +++ b/tests/auto/testlib/selftests/expected_warnings.txt @@ -70,5 +70,5 @@ SKIP : tst_Warnings::testFailOnWarningsThenSkip() My cabbage! :( Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] PASS : tst_Warnings::testFailOnWarningsAndIgnoreWarnings() PASS : tst_Warnings::cleanupTestCase() -Totals: 5 passed, 14 failed, 1 skipped, 0 blacklisted, 0ms +Totals: 5 passed, 10 failed, 0 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_Warnings ********* diff --git a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp index 7b555b8293..e978867bbf 100644 --- a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp +++ b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp @@ -347,7 +347,6 @@ void tst_ExpectFail::xpassContinueSkip() const ++failed; // and *not* ++skipped QEXPECT_FAIL("", "This test should xpass", Continue); QVERIFY(true); - // FIXME: QTBUG-95661 skip-count is incremented. QSKIP("This should be reached but not increment skip-count"); } diff --git a/tests/auto/testlib/selftests/subtest/tst_subtest.cpp b/tests/auto/testlib/selftests/subtest/tst_subtest.cpp index fff8d4068f..c5f5cd6c04 100644 --- a/tests/auto/testlib/selftests/subtest/tst_subtest.cpp +++ b/tests/auto/testlib/selftests/subtest/tst_subtest.cpp @@ -49,6 +49,7 @@ private slots: void test3(); void multiFail(); + void multiSkip(); private: void logNames(const char *caller); void table_data(); @@ -136,13 +137,20 @@ void tst_Subtest::test3() void tst_Subtest::multiFail() { // Simulates tests which call a shared function that does common checks, or - // that do checks in code run asynchronously from a messae loop. + // that do checks in code run asynchronously from a message loop. for (int i = 0; i < 10; ++i) []() { QFAIL("This failure message should be repeated ten times"); }(); - // FIXME QTBUG-95661: it gets counted as eleven failures, of course. QFAIL("But this test should only contribute one to the failure count"); } +void tst_Subtest::multiSkip() +{ + // Similar to multiFail() + for (int i = 0; i < 10; ++i) + []() { QSKIP("This skip should be repeated ten times"); }(); + QSKIP("But this test should only contribute one to the skip count"); +} + QTEST_MAIN(tst_Subtest) #include "tst_subtest.moc" |