diff options
Diffstat (limited to 'tests/auto')
28 files changed, 593 insertions, 82 deletions
diff --git a/tests/auto/testlib/selftests/blacklisted/BLACKLIST b/tests/auto/testlib/selftests/blacklisted/BLACKLIST index edda648921..e8c9b5a6d1 100644 --- a/tests/auto/testlib/selftests/blacklisted/BLACKLIST +++ b/tests/auto/testlib/selftests/blacklisted/BLACKLIST @@ -9,6 +9,12 @@ obscure # no such platform; is ignored [fail] * +[multiSkip] +* + +[multiFail] +* + [xfail] * diff --git a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp index cd7e529f92..90e7876f97 100644 --- a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp +++ b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp @@ -41,6 +41,8 @@ private slots: void skip(); void fail(); void xfail(); + void multiSkip(); + void multiFail(); void xfailContinueSkip(); void xfailContinueFail(); void xpass(); @@ -86,6 +88,23 @@ void tst_Blacklisted::fail() QVERIFY2(false, "This test should BFAIL"); } +void tst_Blacklisted::multiFail() // cf. ../subtest/'s similar tests +{ + ++blacklisted; + for (int i = 0; i < 10; ++i) + []() { QFAIL("This failure message should be repeated ten times"); }(); + QFAIL("But this test should only contribute one to the blacklisted count"); +} + +void tst_Blacklisted::multiSkip() +{ + // Similar to multiFail() + ++skipped; + for (int i = 0; i < 10; ++i) + []() { QSKIP("This skip should be repeated ten times"); }(); + QSKIP("But this test should only contribute one to the skip count"); +} + void tst_Blacklisted::xfail() { ++blacklisted; @@ -121,7 +140,6 @@ void tst_Blacklisted::xpassContinueSkip() ++blacklisted; QEXPECT_FAIL("", "This test should BXPASS then SKIP", Continue); QVERIFY2(true, "This test should BXPASS then SKIP"); - // FIXME QTBUG-95661: skip gets counted QSKIP("This skip should be seen but not counted"); } @@ -130,7 +148,6 @@ void tst_Blacklisted::xpassContinueFail() ++blacklisted; QEXPECT_FAIL("", "This test should BXPASS then BFAIL", Continue); QVERIFY2(true, "This test should BXPASS then BFAIL"); - // FIXME QTBUG-95661: gets double-counted QFAIL("This fail should be seen and not counted (due to prior XPASS)"); } diff --git a/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv b/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv index caa54d0905..f682c48e0f 100644 --- a/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv +++ b/tests/auto/testlib/selftests/expected_benchlibtickcounter.csv @@ -1 +1 @@ -"threeBillionTicks","","CPUTicks",3000023453,3000023453,1 +"threeBillionTicks","","CPUTicks",3000012216,3000012216,1 diff --git a/tests/auto/testlib/selftests/expected_blacklisted.junitxml b/tests/auto/testlib/selftests/expected_blacklisted.junitxml index ce176d7301..5f7eb749cc 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.junitxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.junitxml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<testsuite name="tst_Blacklisted" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="11" failures="0" errors="0" skipped="3" time="@TEST_DURATION@"> +<testsuite name="tst_Blacklisted" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="13" failures="0" errors="0" skipped="3" time="@TEST_DURATION@"> <properties> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> @@ -16,6 +16,20 @@ </testcase> <testcase name="fail" classname="tst_Blacklisted" time="@TEST_DURATION@"/> <testcase name="xfail" classname="tst_Blacklisted" time="@TEST_DURATION@"/> + <testcase name="multiSkip" classname="tst_Blacklisted" time="@TEST_DURATION@"> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="But this test should only contribute one to the skip count"/> + </testcase> + <testcase name="multiFail" classname="tst_Blacklisted" time="@TEST_DURATION@"/> <testcase name="xfailContinueSkip" classname="tst_Blacklisted" time="@TEST_DURATION@"> <skipped message="This skip should be seen and counted"/> </testcase> @@ -27,7 +41,7 @@ <testcase name="xpassContinueFail" classname="tst_Blacklisted" time="@TEST_DURATION@"/> <testcase name="cleanupTestCase" classname="tst_Blacklisted" time="@TEST_DURATION@"> <system-out> - <![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]> + <![CDATA[Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted]]> </system-out> </testcase> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_blacklisted.lightxml b/tests/auto/testlib/selftests/expected_blacklisted.lightxml index ba5f043c32..cc22d4b089 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.lightxml +++ b/tests/auto/testlib/selftests/expected_blacklisted.lightxml @@ -33,6 +33,78 @@ <Incident type="bpass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> + <TestFunction name="multiSkip"> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the skip count]]></Description> + </Incident> + <Duration msecs="0"/> + </TestFunction> + <TestFunction name="multiFail"> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the blacklisted count]]></Description> + </Incident> + <Duration msecs="0"/> + </TestFunction> <TestFunction name="xfailContinueSkip"> <Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> <Description><![CDATA[This test should BXFAIL then SKIP]]></Description> @@ -77,7 +149,7 @@ </TestFunction> <TestFunction name="cleanupTestCase"> <Message type="qdebug" file="" line="0"> - <Description><![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]></Description> + <Description><![CDATA[Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted]]></Description> </Message> <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_blacklisted.tap b/tests/auto/testlib/selftests/expected_blacklisted.tap index af0dab417f..63a0e90e21 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.tap +++ b/tests/auto/testlib/selftests/expected_blacklisted.tap @@ -23,32 +23,120 @@ not ok 5 - xfail() # TODO This test should BXFAIL then BPASS file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -not ok 6 - xfailContinueSkip() # TODO This test should BXFAIL then SKIP +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP This skip should be repeated ten times +ok 6 - multiSkip() # SKIP But this test should only contribute one to the skip count +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO This failure message should be repeated ten times + --- + # This failure message should be repeated ten times + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 7 - multiFail() # TODO But this test should only contribute one to the blacklisted count + --- + # But this test should only contribute one to the blacklisted count + at: tst_Blacklisted::multiFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 + ... +not ok 8 - xfailContinueSkip() # TODO This test should BXFAIL then SKIP --- # This test should BXFAIL then SKIP at: tst_Blacklisted::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -ok 6 - xfailContinueSkip() # SKIP This skip should be seen and counted -not ok 7 - xfailContinueFail() # TODO This test should BXFAIL then BFAIL +ok 8 - xfailContinueSkip() # SKIP This skip should be seen and counted +not ok 9 - xfailContinueFail() # TODO This test should BXFAIL then BFAIL --- # This test should BXFAIL then BFAIL at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -not ok 7 - xfailContinueFail() # TODO This fail should be seen and counted as blacklisted +not ok 9 - xfailContinueFail() # TODO This fail should be seen and counted as blacklisted --- # This fail should be seen and counted as blacklisted at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -ok 8 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS) -ok 9 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP) -ok 10 - xpassContinueSkip() # SKIP This skip should be seen but not counted -ok 11 - xpassContinueFail() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL) +ok 10 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS) +ok 11 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP) +ok 11 - xpassContinueSkip() # SKIP This skip should be seen but not counted +ok 12 - xpassContinueFail() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then BFAIL) not ok 12 - xpassContinueFail() # TODO This fail should be seen and not counted (due to prior XPASS) --- # This fail should be seen and not counted (due to prior XPASS) @@ -56,7 +144,7 @@ not ok 12 - xpassContinueFail() # TODO This fail should be seen and not counted file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -# Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted +# Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted ok 13 - cleanupTestCase() 1..13 # tests 13 diff --git a/tests/auto/testlib/selftests/expected_blacklisted.teamcity b/tests/auto/testlib/selftests/expected_blacklisted.teamcity index a0f1833405..31790bd6b4 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.teamcity +++ b/tests/auto/testlib/selftests/expected_blacklisted.teamcity @@ -12,6 +12,41 @@ ##teamcity[testStarted name='xfail()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xfail()' flowId='tst_Blacklisted'] +##teamcity[testStarted name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testIgnored name='multiSkip()' message='But this test should only contribute one to the skip count |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Blacklisted'] +##teamcity[testStarted name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] +##teamcity[testFinished name='multiFail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='xfailContinueSkip()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xfailContinueSkip()' flowId='tst_Blacklisted'] ##teamcity[testIgnored name='xfailContinueSkip()' message='This skip should be seen and counted |[Loc: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)|]' flowId='tst_Blacklisted'] @@ -29,6 +64,6 @@ ##teamcity[testFinished name='xpassContinueFail()' flowId='tst_Blacklisted'] ##teamcity[testFinished name='xpassContinueFail()' flowId='tst_Blacklisted'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Blacklisted'] -##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted' flowId='tst_Blacklisted'] +##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted' flowId='tst_Blacklisted'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Blacklisted'] ##teamcity[testSuiteFinished name='tst_Blacklisted' flowId='tst_Blacklisted'] diff --git a/tests/auto/testlib/selftests/expected_blacklisted.txt b/tests/auto/testlib/selftests/expected_blacklisted.txt index 07c0150ee9..772ef33b73 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.txt +++ b/tests/auto/testlib/selftests/expected_blacklisted.txt @@ -10,6 +10,50 @@ BFAIL : tst_Blacklisted::fail() 'false' returned FALSE. (This test should BFAIL BXFAIL : tst_Blacklisted::xfail() This test should BXFAIL then BPASS Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BPASS : tst_Blacklisted::xfail() +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +SKIP : tst_Blacklisted::multiSkip() But this test should only contribute one to the skip count + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() This failure message should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] +BFAIL : tst_Blacklisted::multiFail() But this test should only contribute one to the blacklisted count + Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BXFAIL : tst_Blacklisted::xfailContinueSkip() This test should BXFAIL then SKIP Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] SKIP : tst_Blacklisted::xfailContinueSkip() This skip should be seen and counted @@ -28,7 +72,7 @@ BXPASS : tst_Blacklisted::xpassContinueFail() 'true' returned TRUE unexpectedly. Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] BFAIL : tst_Blacklisted::xpassContinueFail() This fail should be seen and not counted (due to prior XPASS) Loc: [qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp(0)] -QDEBUG : tst_Blacklisted::cleanupTestCase() Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted +QDEBUG : tst_Blacklisted::cleanupTestCase() Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted PASS : tst_Blacklisted::cleanupTestCase() Totals: 2 passed, 0 failed, 3 skipped, 8 blacklisted, 0ms ********* Finished testing of tst_Blacklisted ********* diff --git a/tests/auto/testlib/selftests/expected_blacklisted.xml b/tests/auto/testlib/selftests/expected_blacklisted.xml index 319bddee6d..2ef3513786 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.xml +++ b/tests/auto/testlib/selftests/expected_blacklisted.xml @@ -35,6 +35,78 @@ <Incident type="bpass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> + <TestFunction name="multiSkip"> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the skip count]]></Description> + </Incident> + <Duration msecs="0"/> + </TestFunction> + <TestFunction name="multiFail"> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[This failure message should be repeated ten times]]></Description> + </Incident> + <Incident type="bfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the blacklisted count]]></Description> + </Incident> + <Duration msecs="0"/> + </TestFunction> <TestFunction name="xfailContinueSkip"> <Incident type="bxfail" file="qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp" line="0"> <Description><![CDATA[This test should BXFAIL then SKIP]]></Description> @@ -79,7 +151,7 @@ </TestFunction> <TestFunction name="cleanupTestCase"> <Message type="qdebug" file="" line="0"> - <Description><![CDATA[Totals should add up to 11: 2 passed, 0 failed, 2 skipped, 7 blacklisted]]></Description> + <Description><![CDATA[Totals should add up to 13: 2 passed, 0 failed, 3 skipped, 8 blacklisted]]></Description> </Message> <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_expectfail.junitxml b/tests/auto/testlib/selftests/expected_expectfail.junitxml index 54f0416016..75da637ab9 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.junitxml +++ b/tests/auto/testlib/selftests/expected_expectfail.junitxml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<testsuite name="tst_ExpectFail" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="44" failures="17" errors="0" skipped="5" time="@TEST_DURATION@"> +<testsuite name="tst_ExpectFail" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="44" failures="17" errors="0" skipped="4" time="@TEST_DURATION@"> <properties> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> diff --git a/tests/auto/testlib/selftests/expected_expectfail.tap b/tests/auto/testlib/selftests/expected_expectfail.tap index c520bc25c0..8527acdaf1 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.tap +++ b/tests/auto/testlib/selftests/expected_expectfail.tap @@ -145,29 +145,29 @@ ok 32 - xpassAbortXfailContinue() # TODO 'true' returned TRUE unexpectedly. () ok 33 - xpassContinue() # TODO 'true' returned TRUE unexpectedly. () # This should be reached ok 34 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. () -ok 35 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count -ok 36 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. () -not ok 37 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals +ok 34 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count +ok 35 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. () +not ok 36 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals --- at: tst_ExpectFail::xpassContinueXfailAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -ok 37 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () -ok 38 - xpassAbortDataDrivenWithQVerify(Pass) -ok 39 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () +ok 36 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () +ok 37 - xpassAbortDataDrivenWithQVerify(Pass) +ok 38 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () # Test should Continue past XPASS # Test should simply PASS -ok 40 - xpassContinueDataDrivenWithQVerify(Pass) -ok 41 - xpassAbortDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. -ok 42 - xpassAbortDataDrivenWithQCompare(Pass) -ok 43 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. +ok 39 - xpassContinueDataDrivenWithQVerify(Pass) +ok 40 - xpassAbortDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. +ok 41 - xpassAbortDataDrivenWithQCompare(Pass) +ok 42 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. # Test should Continue past XPASS # Test should simply PASS -ok 44 - xpassContinueDataDrivenWithQCompare(Pass) +ok 43 - xpassContinueDataDrivenWithQCompare(Pass) # Totals should add up to 44: 23 passed, 17 failed, 4 skipped -ok 45 - cleanupTestCase() -1..45 -# tests 45 +ok 44 - cleanupTestCase() +1..44 +# tests 44 # pass 23 # fail 17 diff --git a/tests/auto/testlib/selftests/expected_expectfail.txt b/tests/auto/testlib/selftests/expected_expectfail.txt index 2dff4ad55f..fee6f6a30a 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.txt +++ b/tests/auto/testlib/selftests/expected_expectfail.txt @@ -110,5 +110,5 @@ QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) Test should s PASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) QDEBUG : tst_ExpectFail::cleanupTestCase() Totals should add up to 44: 23 passed, 17 failed, 4 skipped PASS : tst_ExpectFail::cleanupTestCase() -Totals: 23 passed, 17 failed, 5 skipped, 0 blacklisted, 0ms +Totals: 23 passed, 17 failed, 4 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_ExpectFail ********* diff --git a/tests/auto/testlib/selftests/expected_skipinit.lightxml b/tests/auto/testlib/selftests/expected_skipinit.lightxml index 7dd35a4ed3..c2517b07c2 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.lightxml +++ b/tests/auto/testlib/selftests/expected_skipinit.lightxml @@ -10,7 +10,6 @@ <Duration msecs="0"/> </TestFunction> <TestFunction name="cleanupTestCase"> - <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_skipinit.tap b/tests/auto/testlib/selftests/expected_skipinit.tap index 1fc38e79aa..880b4471e1 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.tap +++ b/tests/auto/testlib/selftests/expected_skipinit.tap @@ -1,8 +1,7 @@ TAP version 13 # tst_SkipInit ok 1 - initTestCase() # SKIP Skip inside initTestCase. This should skip all tests in the class. -ok 2 - cleanupTestCase() -1..2 -# tests 2 -# pass 1 +1..1 +# tests 1 +# pass 0 # fail 0 diff --git a/tests/auto/testlib/selftests/expected_skipinit.teamcity b/tests/auto/testlib/selftests/expected_skipinit.teamcity index 189953f532..4538d67f7e 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.teamcity +++ b/tests/auto/testlib/selftests/expected_skipinit.teamcity @@ -2,6 +2,4 @@ ##teamcity[testStarted name='initTestCase()' flowId='tst_SkipInit'] ##teamcity[testIgnored name='initTestCase()' message='Skip inside initTestCase. This should skip all tests in the class. |[Loc: qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(0)|]' flowId='tst_SkipInit'] ##teamcity[testFinished name='initTestCase()' flowId='tst_SkipInit'] -##teamcity[testStarted name='cleanupTestCase()' flowId='tst_SkipInit'] -##teamcity[testFinished name='cleanupTestCase()' flowId='tst_SkipInit'] ##teamcity[testSuiteFinished name='tst_SkipInit' flowId='tst_SkipInit'] diff --git a/tests/auto/testlib/selftests/expected_skipinit.txt b/tests/auto/testlib/selftests/expected_skipinit.txt index 9fc2d98006..ef5e68ac3b 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.txt +++ b/tests/auto/testlib/selftests/expected_skipinit.txt @@ -2,6 +2,5 @@ Config: Using QtTest library SKIP : tst_SkipInit::initTestCase() Skip inside initTestCase. This should skip all tests in the class. Loc: [qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(0)] -PASS : tst_SkipInit::cleanupTestCase() -Totals: 1 passed, 0 failed, 1 skipped, 0 blacklisted, 0ms +Totals: 0 passed, 0 failed, 1 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_SkipInit ********* diff --git a/tests/auto/testlib/selftests/expected_skipinit.xml b/tests/auto/testlib/selftests/expected_skipinit.xml index 7633cb9152..eb2d24213b 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.xml +++ b/tests/auto/testlib/selftests/expected_skipinit.xml @@ -12,7 +12,6 @@ <Duration msecs="0"/> </TestFunction> <TestFunction name="cleanupTestCase"> - <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_subtest.junitxml b/tests/auto/testlib/selftests/expected_subtest.junitxml index 353f739985..289053e3f9 100644 --- a/tests/auto/testlib/selftests/expected_subtest.junitxml +++ b/tests/auto/testlib/selftests/expected_subtest.junitxml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<testsuite name="tst_Subtest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="10" failures="3" errors="0" skipped="0" time="@TEST_DURATION@"> +<testsuite name="tst_Subtest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="11" failures="3" errors="0" skipped="1" time="@TEST_DURATION@"> <properties> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> @@ -82,6 +82,23 @@ <![CDATA[cleanup multiFail (null)]]> </system-out> </testcase> + <testcase name="multiSkip" classname="tst_Subtest" time="@TEST_DURATION@"> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="This skip should be repeated ten times"/> + <skipped message="But this test should only contribute one to the skip count"/> + <system-out> + <![CDATA[init multiSkip (null)]]> + <![CDATA[cleanup multiSkip (null)]]> + </system-out> + </testcase> <testcase name="cleanupTestCase" classname="tst_Subtest" time="@TEST_DURATION@"> <system-out> <![CDATA[cleanupTestCase cleanupTestCase (null)]]> diff --git a/tests/auto/testlib/selftests/expected_subtest.lightxml b/tests/auto/testlib/selftests/expected_subtest.lightxml index 36c502b2f7..f666ba5b12 100644 --- a/tests/auto/testlib/selftests/expected_subtest.lightxml +++ b/tests/auto/testlib/selftests/expected_subtest.lightxml @@ -195,6 +195,48 @@ </Message> <Duration msecs="0"/> </TestFunction> + <TestFunction name="multiSkip"> + <Message type="qdebug" file="" line="0"> + <Description><![CDATA[init multiSkip (null)]]></Description> + </Message> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the skip count]]></Description> + </Incident> + <Message type="qdebug" file="" line="0"> + <Description><![CDATA[cleanup multiSkip (null)]]></Description> + </Message> + <Duration msecs="0"/> + </TestFunction> <TestFunction name="cleanupTestCase"> <Message type="qdebug" file="" line="0"> <Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description> diff --git a/tests/auto/testlib/selftests/expected_subtest.tap b/tests/auto/testlib/selftests/expected_subtest.tap index ebbc1d1067..ee69487233 100644 --- a/tests/auto/testlib/selftests/expected_subtest.tap +++ b/tests/auto/testlib/selftests/expected_subtest.tap @@ -68,70 +68,70 @@ not ok 9 - multiFail() file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 10 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 11 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 12 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 13 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 14 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 15 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 16 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 17 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 18 - multiFail() +not ok 9 - multiFail() --- # This failure message should be repeated ten times at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp line: 0 ... -not ok 19 - multiFail() +not ok 9 - multiFail() --- # But this test should only contribute one to the failure count at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0) @@ -139,9 +139,22 @@ not ok 19 - multiFail() line: 0 ... # cleanup multiFail (null) +# init multiSkip (null) +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP This skip should be repeated ten times +ok 10 - multiSkip() # SKIP But this test should only contribute one to the skip count +# cleanup multiSkip (null) # cleanupTestCase cleanupTestCase (null) -ok 20 - cleanupTestCase() -1..20 -# tests 20 +ok 11 - cleanupTestCase() +1..11 +# tests 11 # pass 7 -# fail 13 +# fail 3 diff --git a/tests/auto/testlib/selftests/expected_subtest.teamcity b/tests/auto/testlib/selftests/expected_subtest.teamcity index 580e5695dd..e58444ecf1 100644 --- a/tests/auto/testlib/selftests/expected_subtest.teamcity +++ b/tests/auto/testlib/selftests/expected_subtest.teamcity @@ -49,7 +49,31 @@ ##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] ##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='But this test should only contribute one to the failure count' flowId='tst_Subtest'] ##teamcity[testFinished name='multiFail()' flowId='tst_Subtest'] +##teamcity[testStarted name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testStdOut name='multiSkip()' out='QDEBUG: cleanup multiFail (null)|nQDEBUG: init multiSkip (null)' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='This skip should be repeated ten times |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] +##teamcity[testIgnored name='multiSkip()' message='But this test should only contribute one to the skip count |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' flowId='tst_Subtest'] +##teamcity[testFinished name='multiSkip()' flowId='tst_Subtest'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Subtest'] -##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup multiFail (null)|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest'] +##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup multiSkip (null)|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Subtest'] ##teamcity[testSuiteFinished name='tst_Subtest' flowId='tst_Subtest'] diff --git a/tests/auto/testlib/selftests/expected_subtest.txt b/tests/auto/testlib/selftests/expected_subtest.txt index d35d7b4d24..365bbf087a 100644 --- a/tests/auto/testlib/selftests/expected_subtest.txt +++ b/tests/auto/testlib/selftests/expected_subtest.txt @@ -68,7 +68,31 @@ FAIL! : tst_Subtest::multiFail() This failure message should be repeated ten ti FAIL! : tst_Subtest::multiFail() But this test should only contribute one to the failure count Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] QDEBUG : tst_Subtest::multiFail() cleanup multiFail (null) +QDEBUG : tst_Subtest::multiSkip() init multiSkip (null) +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() This skip should be repeated ten times + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +SKIP : tst_Subtest::multiSkip() But this test should only contribute one to the skip count + Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)] +QDEBUG : tst_Subtest::multiSkip() cleanup multiSkip (null) QDEBUG : tst_Subtest::cleanupTestCase() cleanupTestCase cleanupTestCase (null) PASS : tst_Subtest::cleanupTestCase() -Totals: 7 passed, 13 failed, 0 skipped, 0 blacklisted, 0ms +Totals: 7 passed, 3 failed, 1 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_Subtest ********* diff --git a/tests/auto/testlib/selftests/expected_subtest.xml b/tests/auto/testlib/selftests/expected_subtest.xml index 975ba60af7..8e4f9ae563 100644 --- a/tests/auto/testlib/selftests/expected_subtest.xml +++ b/tests/auto/testlib/selftests/expected_subtest.xml @@ -197,6 +197,48 @@ </Message> <Duration msecs="0"/> </TestFunction> + <TestFunction name="multiSkip"> + <Message type="qdebug" file="" line="0"> + <Description><![CDATA[init multiSkip (null)]]></Description> + </Message> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[This skip should be repeated ten times]]></Description> + </Incident> + <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0"> + <Description><![CDATA[But this test should only contribute one to the skip count]]></Description> + </Incident> + <Message type="qdebug" file="" line="0"> + <Description><![CDATA[cleanup multiSkip (null)]]></Description> + </Message> + <Duration msecs="0"/> + </TestFunction> <TestFunction name="cleanupTestCase"> <Message type="qdebug" file="" line="0"> <Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description> diff --git a/tests/auto/testlib/selftests/expected_warnings.junitxml b/tests/auto/testlib/selftests/expected_warnings.junitxml index 92285a4937..1799be8279 100644 --- a/tests/auto/testlib/selftests/expected_warnings.junitxml +++ b/tests/auto/testlib/selftests/expected_warnings.junitxml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="15" failures="10" errors="0" skipped="1" time="@TEST_DURATION@"> +<testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="15" failures="10" errors="0" skipped="0" time="@TEST_DURATION@"> <properties> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> diff --git a/tests/auto/testlib/selftests/expected_warnings.tap b/tests/auto/testlib/selftests/expected_warnings.tap index 4dbebb45b3..161aa6db2b 100644 --- a/tests/auto/testlib/selftests/expected_warnings.tap +++ b/tests/auto/testlib/selftests/expected_warnings.tap @@ -47,7 +47,7 @@ Ran out of cabbage! # Ran out of tortillas! # Ran out of oil! # nope -not ok 8 - testFailOnWarnings() +not ok 7 - testFailOnWarnings() --- # Received a warning that resulted in a failure: Ran out of biscuits! @@ -55,7 +55,7 @@ Ran out of biscuits! file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -not ok 9 - testFailOnWarnings() +not ok 7 - testFailOnWarnings() --- # Received a warning that resulted in a failure: Running low on toothpaste! @@ -67,8 +67,8 @@ Running low on toothpaste! # Running low on toothpaste! # Running low on toothpaste! # Ran out of muffins! -ok 10 - testFailOnWarningsCleared() -not ok 11 - testFailOnWarningsWithData(warning1) +ok 8 - testFailOnWarningsCleared() +not ok 9 - testFailOnWarningsWithData(warning1) --- # Received a warning that resulted in a failure: warning1 @@ -79,7 +79,7 @@ warning1 # warning2 # warning3 # warning1 -not ok 12 - testFailOnWarningsWithData(warning2) +not ok 10 - testFailOnWarningsWithData(warning2) --- # Received a warning that resulted in a failure: warning2 @@ -90,7 +90,7 @@ warning2 # warning3 # warning1 # warning2 -not ok 13 - testFailOnWarningsWithData(warning3) +not ok 11 - testFailOnWarningsWithData(warning3) --- # Received a warning that resulted in a failure: warning3 @@ -98,14 +98,14 @@ warning3 file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -not ok 14 - testFailOnWarningsFailInHelper() +not ok 12 - testFailOnWarningsFailInHelper() --- # This failure message should be printed but not cause the test to abort at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -not ok 15 - testFailOnWarningsFailInHelper() +not ok 12 - testFailOnWarningsFailInHelper() --- # Received a warning that resulted in a failure: Ran out of cabbage! @@ -113,14 +113,14 @@ Ran out of cabbage! file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -not ok 16 - testFailOnWarningsFailInHelper() +not ok 12 - testFailOnWarningsFailInHelper() --- # My cabbage! :( at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -not ok 17 - testFailOnWarningsThenSkip() +not ok 13 - testFailOnWarningsThenSkip() --- # Received a warning that resulted in a failure: Ran out of cabbage! @@ -128,10 +128,10 @@ Ran out of cabbage! file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp line: 0 ... -ok 18 - testFailOnWarningsThenSkip() # SKIP My cabbage! :( -ok 19 - testFailOnWarningsAndIgnoreWarnings() -ok 20 - cleanupTestCase() -1..20 -# tests 20 +ok 13 - testFailOnWarningsThenSkip() # SKIP My cabbage! :( +ok 14 - testFailOnWarningsAndIgnoreWarnings() +ok 15 - cleanupTestCase() +1..15 +# tests 15 # pass 5 -# fail 14 +# fail 10 diff --git a/tests/auto/testlib/selftests/expected_warnings.txt b/tests/auto/testlib/selftests/expected_warnings.txt index 02361eb39d..5037eb87df 100644 --- a/tests/auto/testlib/selftests/expected_warnings.txt +++ b/tests/auto/testlib/selftests/expected_warnings.txt @@ -70,5 +70,5 @@ SKIP : tst_Warnings::testFailOnWarningsThenSkip() My cabbage! :( Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] PASS : tst_Warnings::testFailOnWarningsAndIgnoreWarnings() PASS : tst_Warnings::cleanupTestCase() -Totals: 5 passed, 14 failed, 1 skipped, 0 blacklisted, 0ms +Totals: 5 passed, 10 failed, 0 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_Warnings ********* diff --git a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp index 7b555b8293..e978867bbf 100644 --- a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp +++ b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp @@ -347,7 +347,6 @@ void tst_ExpectFail::xpassContinueSkip() const ++failed; // and *not* ++skipped QEXPECT_FAIL("", "This test should xpass", Continue); QVERIFY(true); - // FIXME: QTBUG-95661 skip-count is incremented. QSKIP("This should be reached but not increment skip-count"); } diff --git a/tests/auto/testlib/selftests/subtest/tst_subtest.cpp b/tests/auto/testlib/selftests/subtest/tst_subtest.cpp index fff8d4068f..c5f5cd6c04 100644 --- a/tests/auto/testlib/selftests/subtest/tst_subtest.cpp +++ b/tests/auto/testlib/selftests/subtest/tst_subtest.cpp @@ -49,6 +49,7 @@ private slots: void test3(); void multiFail(); + void multiSkip(); private: void logNames(const char *caller); void table_data(); @@ -136,13 +137,20 @@ void tst_Subtest::test3() void tst_Subtest::multiFail() { // Simulates tests which call a shared function that does common checks, or - // that do checks in code run asynchronously from a messae loop. + // that do checks in code run asynchronously from a message loop. for (int i = 0; i < 10; ++i) []() { QFAIL("This failure message should be repeated ten times"); }(); - // FIXME QTBUG-95661: it gets counted as eleven failures, of course. QFAIL("But this test should only contribute one to the failure count"); } +void tst_Subtest::multiSkip() +{ + // Similar to multiFail() + for (int i = 0; i < 10; ++i) + []() { QSKIP("This skip should be repeated ten times"); }(); + QSKIP("But this test should only contribute one to the skip count"); +} + QTEST_MAIN(tst_Subtest) #include "tst_subtest.moc" |