diff options
| author | Edward Welbourne <edward.welbourne@qt.io> | 2021-08-12 14:39:40 +0200 |
|---|---|---|
| committer | Edward Welbourne <edward.welbourne@qt.io> | 2021-08-17 16:28:27 +0200 |
| commit | ebcc8413f21556e0128c713dedfd97442ae93479 (patch) | |
| tree | d9694064b04d15c962e4350d379ca62416e0adde /tests/auto/testlib | |
| parent | a7484002a3645c91bf7b970f9194d427d7466981 (diff) | |
Expand testing of QEXPECT_FAIL()
Test both Abort and Continue cases. Test more with successive marked
as expected failure. Test cases with a QSKIP after the marked check.
Unify data functions where practical.
Change-Id: I2eade5e4dd0907d23e37137ce3d93cd5ca79f802
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
Diffstat (limited to 'tests/auto/testlib')
| -rw-r--r-- | tests/auto/testlib/selftests/expected_expectfail.junitxml | 138 | ||||
| -rw-r--r-- | tests/auto/testlib/selftests/expected_expectfail.lightxml | 205 | ||||
| -rw-r--r-- | tests/auto/testlib/selftests/expected_expectfail.tap | 140 | ||||
| -rw-r--r-- | tests/auto/testlib/selftests/expected_expectfail.teamcity | 155 | ||||
| -rw-r--r-- | tests/auto/testlib/selftests/expected_expectfail.txt | 108 | ||||
| -rw-r--r-- | tests/auto/testlib/selftests/expected_expectfail.xml | 205 | ||||
| -rw-r--r-- | tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp | 285 |
7 files changed, 981 insertions, 255 deletions
diff --git a/tests/auto/testlib/selftests/expected_expectfail.junitxml b/tests/auto/testlib/selftests/expected_expectfail.junitxml index fd4cdb3488..31521263ee 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.junitxml +++ b/tests/auto/testlib/selftests/expected_expectfail.junitxml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<testsuite name="tst_ExpectFail" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="24" failures="6" errors="0" skipped="0" time="@TEST_DURATION@"> +<testsuite name="tst_ExpectFail" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="44" failures="17" errors="0" skipped="5" time="@TEST_DURATION@"> <properties> <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/> <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/> @@ -19,66 +19,166 @@ <![CDATA[This should xfail]]> </system-out> </testcase> + <testcase name="xfailContinueSkip" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <skipped message="This skip should be reported and counted"/> + <system-out> + <![CDATA[This should xfail then skip]]> + </system-out> + </testcase> + <testcase name="xfailAbortSkip" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <system-out> + <![CDATA[This should xfail]]> + </system-out> + </testcase> <testcase name="xfailTwice" classname="tst_ExpectFail" time="@TEST_DURATION@"> <failure type="fail" message="Already expecting a fail"/> </testcase> + <testcase name="xfailDataDrivenTwice(Pass Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <failure type="fail" message="Already expecting a fail"/> + </testcase> + <testcase name="xfailDataDrivenTwice(Pass Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <failure type="fail" message="Already expecting a fail"/> + </testcase> + <testcase name="xfailDataDrivenTwice(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <failure type="fail" message="Already expecting a fail"/> + </testcase> + <testcase name="xfailDataDrivenTwice(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <failure type="fail" message="Already expecting a fail"/> + </testcase> <testcase name="xfailWithQString" classname="tst_ExpectFail" time="@TEST_DURATION@"> <system-out> <![CDATA[A string]]> <![CDATA[Bug 5 (The message)]]> </system-out> </testcase> - <testcase name="xfailDataDrivenWithQVerify(Pass 1)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> - <testcase name="xfailDataDrivenWithQVerify(Pass 2)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> - <testcase name="xfailDataDrivenWithQVerify(Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <testcase name="xfailDataDrivenWithQString(Pass Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <skipped message="Each Continue or Pass reports this and increments skip-count"/> + </testcase> + <testcase name="xfailDataDrivenWithQString(Pass Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <skipped message="Each Continue or Pass reports this and increments skip-count"/> + </testcase> + <testcase name="xfailDataDrivenWithQString(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <system-out> + <![CDATA[A string]]> + </system-out> + </testcase> + <testcase name="xfailDataDrivenWithQString(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <skipped message="Each Continue or Pass reports this and increments skip-count"/> + <system-out> + <![CDATA[A string]]> + <![CDATA[Bug 5 (The message)]]> + </system-out> + </testcase> + <testcase name="xfailDataDrivenWithQVerify(Pass Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> + <testcase name="xfailDataDrivenWithQVerify(Pass Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> + <testcase name="xfailDataDrivenWithQVerify(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"> <system-out> <![CDATA[This test should xfail]]> </system-out> </testcase> - <testcase name="xfailDataDrivenWithQVerify(Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <testcase name="xfailDataDrivenWithQVerify(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"> <system-out> <![CDATA[This test should xfail]]> </system-out> </testcase> - <testcase name="xfailDataDrivenWithQCompare(Pass 1)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> - <testcase name="xfailDataDrivenWithQCompare(Pass 2)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> - <testcase name="xfailDataDrivenWithQCompare(Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <testcase name="xfailDataDrivenWithQCompare(Pass Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> + <testcase name="xfailDataDrivenWithQCompare(Pass Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> + <testcase name="xfailDataDrivenWithQCompare(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"> <system-out> <![CDATA[This test should xfail]]> </system-out> </testcase> - <testcase name="xfailDataDrivenWithQCompare(Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <testcase name="xfailDataDrivenWithQCompare(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"> <system-out> <![CDATA[This test should xfail]]> </system-out> </testcase> - <testcase name="xfailOnWrongRow(right row)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> - <testcase name="xfailOnAnyRow(first row)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <testcase name="xfailOnWrongRow(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <system-out> + <![CDATA[Should pass (*not* xpass), despite test-case name]]> + </system-out> + </testcase> + <testcase name="xfailOnWrongRow(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <system-out> + <![CDATA[Should pass (*not* xpass), despite test-case name]]> + </system-out> + </testcase> + <testcase name="xfailOnAnyRow(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"> <system-out> <![CDATA[This test should xfail]]> </system-out> </testcase> - <testcase name="xfailOnAnyRow(second row)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <testcase name="xfailOnAnyRow(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"> <system-out> <![CDATA[This test should xfail]]> </system-out> </testcase> - <testcase name="xfailWithoutVerify(first row)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <testcase name="xfailWithoutCheck(Fail Abort)" classname="tst_ExpectFail" time="@TEST_DURATION@"> <failure type="fail" message="QEXPECT_FAIL was called without any subsequent verification statements"/> + <system-out> + <![CDATA[Should fail (*not* xfail), despite test-case name]]> + </system-out> </testcase> - <testcase name="xfailWithoutVerify(second row)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <testcase name="xfailWithoutCheck(Fail Continue)" classname="tst_ExpectFail" time="@TEST_DURATION@"> <failure type="fail" message="QEXPECT_FAIL was called without any subsequent verification statements"/> + <system-out> + <![CDATA[Should fail (*not* xfail), despite test-case name]]> + </system-out> + </testcase> + <testcase name="xpassAbort" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <failure type="xpass" message="'true' returned TRUE unexpectedly. ()"/> + </testcase> + <testcase name="xpassAbortSkip" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <failure type="xpass" message="'true' returned TRUE unexpectedly. ()"/> + </testcase> + <testcase name="xpassAbortXfailContinue" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <failure type="xpass" message="'true' returned TRUE unexpectedly. ()"/> + </testcase> + <testcase name="xpassContinue" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <failure type="xpass" message="'true' returned TRUE unexpectedly. ()"/> + <system-out> + <![CDATA[This should be reached]]> + </system-out> + </testcase> + <testcase name="xpassContinueSkip" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <failure type="xpass" message="'true' returned TRUE unexpectedly. ()"/> + <skipped message="This should be reached but not increment skip-count"/> </testcase> - <testcase name="xpass" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <testcase name="xpassContinueXfailAbort" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <failure type="xpass" message="'true' returned TRUE unexpectedly. ()"/> + <system-out> + <![CDATA[This test should xfail but not add to totals]]> + </system-out> + </testcase> + <testcase name="xpassAbortDataDrivenWithQVerify(XPass)" classname="tst_ExpectFail" time="@TEST_DURATION@"> <failure type="xpass" message="'true' returned TRUE unexpectedly. ()"/> </testcase> - <testcase name="xpassDataDrivenWithQVerify(XPass)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <testcase name="xpassAbortDataDrivenWithQVerify(Pass)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> + <testcase name="xpassContinueDataDrivenWithQVerify(XPass)" classname="tst_ExpectFail" time="@TEST_DURATION@"> <failure type="xpass" message="'true' returned TRUE unexpectedly. ()"/> + <system-out> + <![CDATA[Test should Continue past XPASS]]> + </system-out> + </testcase> + <testcase name="xpassContinueDataDrivenWithQVerify(Pass)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <system-out> + <![CDATA[Test should simply PASS]]> + </system-out> + </testcase> + <testcase name="xpassAbortDataDrivenWithQCompare(XPass)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <failure type="xpass" message="QCOMPARE(1, 1) returned TRUE unexpectedly."/> </testcase> - <testcase name="xpassDataDrivenWithQVerify(Pass)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> - <testcase name="xpassDataDrivenWithQCompare(XPass)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <testcase name="xpassAbortDataDrivenWithQCompare(Pass)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> + <testcase name="xpassContinueDataDrivenWithQCompare(XPass)" classname="tst_ExpectFail" time="@TEST_DURATION@"> <failure type="xpass" message="QCOMPARE(1, 1) returned TRUE unexpectedly."/> + <system-out> + <![CDATA[Test should Continue past XPASS]]> + </system-out> + </testcase> + <testcase name="xpassContinueDataDrivenWithQCompare(Pass)" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <system-out> + <![CDATA[Test should simply PASS]]> + </system-out> </testcase> - <testcase name="xpassDataDrivenWithQCompare(Pass)" classname="tst_ExpectFail" time="@TEST_DURATION@"/> <testcase name="cleanupTestCase" classname="tst_ExpectFail" time="@TEST_DURATION@"/> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_expectfail.lightxml b/tests/auto/testlib/selftests/expected_expectfail.lightxml index 7c91656b89..36636e13c4 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.lightxml +++ b/tests/auto/testlib/selftests/expected_expectfail.lightxml @@ -30,12 +30,47 @@ <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> +<TestFunction name="xfailContinueSkip"> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA[This should xfail then skip]]></Description> +</Incident> +<Message type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA[This skip should be reported and counted]]></Description> +</Message> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xfailAbortSkip"> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA[This should xfail]]></Description> +</Incident> +<Incident type="pass" file="" line="0" /> + <Duration msecs="0"/> +</TestFunction> <TestFunction name="xfailTwice"> <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> <Description><![CDATA[Already expecting a fail]]></Description> </Incident> <Duration msecs="0"/> </TestFunction> +<TestFunction name="xfailDataDrivenTwice"> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Pass Abort]]></DataTag> + <Description><![CDATA[Already expecting a fail]]></Description> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Pass Continue]]></DataTag> + <Description><![CDATA[Already expecting a fail]]></Description> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Abort]]></DataTag> + <Description><![CDATA[Already expecting a fail]]></Description> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[Already expecting a fail]]></Description> +</Incident> + <Duration msecs="0"/> +</TestFunction> <TestFunction name="xfailWithQString"> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> <Description><![CDATA[A string]]></Description> @@ -46,93 +81,181 @@ <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> +<TestFunction name="xfailDataDrivenWithQString"> +<Message type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Pass Abort]]></DataTag> + <Description><![CDATA[Each Continue or Pass reports this and increments skip-count]]></Description> +</Message> +<Message type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Pass Continue]]></DataTag> + <Description><![CDATA[Each Continue or Pass reports this and increments skip-count]]></Description> +</Message> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Abort]]></DataTag> + <Description><![CDATA[A string]]></Description> +</Incident> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[Fail Abort]]></DataTag> +</Incident> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[A string]]></Description> +</Incident> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[Bug 5 (The message)]]></Description> +</Incident> +<Message type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[Each Continue or Pass reports this and increments skip-count]]></Description> +</Message> + <Duration msecs="0"/> +</TestFunction> <TestFunction name="xfailDataDrivenWithQVerify"> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Pass 1]]></DataTag> + <DataTag><![CDATA[Pass Abort]]></DataTag> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Pass 2]]></DataTag> + <DataTag><![CDATA[Pass Continue]]></DataTag> </Incident> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[Abort]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Abort]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> </Incident> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[Continue]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Continue]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> </Incident> <Duration msecs="0"/> </TestFunction> <TestFunction name="xfailDataDrivenWithQCompare"> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Pass 1]]></DataTag> + <DataTag><![CDATA[Pass Abort]]></DataTag> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Pass 2]]></DataTag> + <DataTag><![CDATA[Pass Continue]]></DataTag> </Incident> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[Abort]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Abort]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> </Incident> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[Continue]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Continue]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> </Incident> <Duration msecs="0"/> </TestFunction> <TestFunction name="xfailOnWrongRow"> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Fail Abort]]></DataTag> + <Description><![CDATA[Should pass (*not* xpass), despite test-case name]]></Description> +</Message> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[right row]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> +</Incident> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[Should pass (*not* xpass), despite test-case name]]></Description> +</Message> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> </Incident> <Duration msecs="0"/> </TestFunction> <TestFunction name="xfailOnAnyRow"> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[first row]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[first row]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> </Incident> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[second row]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[second row]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> </Incident> <Duration msecs="0"/> </TestFunction> -<TestFunction name="xfailWithoutVerify"> +<TestFunction name="xfailWithoutCheck"> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Fail Abort]]></DataTag> + <Description><![CDATA[Should fail (*not* xfail), despite test-case name]]></Description> +</Message> <Incident type="fail" file="" line="0"> - <DataTag><![CDATA[first row]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> <Description><![CDATA[QEXPECT_FAIL was called without any subsequent verification statements]]></Description> </Incident> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[Should fail (*not* xfail), despite test-case name]]></Description> +</Message> <Incident type="fail" file="" line="0"> - <DataTag><![CDATA[second row]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> <Description><![CDATA[QEXPECT_FAIL was called without any subsequent verification statements]]></Description> </Incident> <Duration msecs="0"/> </TestFunction> -<TestFunction name="xpass"> +<TestFunction name="xpassAbort"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassAbortSkip"> <Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> </Incident> <Duration msecs="0"/> </TestFunction> -<TestFunction name="xpassDataDrivenWithQVerify"> +<TestFunction name="xpassAbortXfailContinue"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassContinue"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> +<Message type="qdebug" file="" line="0"> + <Description><![CDATA[This should be reached]]></Description> +</Message> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassContinueSkip"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> +<Message type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA[This should be reached but not increment skip-count]]></Description> +</Message> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassContinueXfailAbort"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA[This test should xfail but not add to totals]]></Description> +</Incident> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassAbortDataDrivenWithQVerify"> <Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> <DataTag><![CDATA[XPass]]></DataTag> <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> @@ -142,7 +265,25 @@ </Incident> <Duration msecs="0"/> </TestFunction> -<TestFunction name="xpassDataDrivenWithQCompare"> +<TestFunction name="xpassContinueDataDrivenWithQVerify"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[XPass]]></DataTag> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[XPass]]></DataTag> + <Description><![CDATA[Test should Continue past XPASS]]></Description> +</Message> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Pass]]></DataTag> + <Description><![CDATA[Test should simply PASS]]></Description> +</Message> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[Pass]]></DataTag> +</Incident> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassAbortDataDrivenWithQCompare"> <Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> <DataTag><![CDATA[XPass]]></DataTag> <Description><![CDATA[QCOMPARE(1, 1) returned TRUE unexpectedly.]]></Description> @@ -152,6 +293,24 @@ </Incident> <Duration msecs="0"/> </TestFunction> +<TestFunction name="xpassContinueDataDrivenWithQCompare"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[XPass]]></DataTag> + <Description><![CDATA[QCOMPARE(1, 1) returned TRUE unexpectedly.]]></Description> +</Incident> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[XPass]]></DataTag> + <Description><![CDATA[Test should Continue past XPASS]]></Description> +</Message> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Pass]]></DataTag> + <Description><![CDATA[Test should simply PASS]]></Description> +</Message> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[Pass]]></DataTag> +</Incident> + <Duration msecs="0"/> +</TestFunction> <TestFunction name="cleanupTestCase"> <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expected_expectfail.tap b/tests/auto/testlib/selftests/expected_expectfail.tap index 2ad422b7f9..984815909a 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.tap +++ b/tests/auto/testlib/selftests/expected_expectfail.tap @@ -16,81 +16,169 @@ not ok 3 - xfailAndAbort() # TODO This should xfail file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 4 - xfailTwice() +not ok 4 - xfailContinueSkip() # TODO This should xfail then skip + --- + at: tst_ExpectFail::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +ok 4 - xfailContinueSkip() # SKIP This skip should be reported and counted +not ok 5 - xfailAbortSkip() # TODO This should xfail + --- + at: tst_ExpectFail::xfailAbortSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 6 - xfailTwice() --- # Already expecting a fail at: tst_ExpectFail::xfailTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 5 - xfailWithQString() # TODO A string +not ok 7 - xfailDataDrivenTwice(Pass Abort) + --- + # Already expecting a fail + at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 8 - xfailDataDrivenTwice(Pass Continue) + --- + # Already expecting a fail + at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 9 - xfailDataDrivenTwice(Fail Abort) + --- + # Already expecting a fail + at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 10 - xfailDataDrivenTwice(Fail Continue) + --- + # Already expecting a fail + at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 11 - xfailWithQString() # TODO A string --- at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 5 - xfailWithQString() # TODO Bug 5 (The message) +not ok 11 - xfailWithQString() # TODO Bug 5 (The message) --- at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -ok 6 - xfailDataDrivenWithQVerify(Pass 1) -ok 7 - xfailDataDrivenWithQVerify(Pass 2) -not ok 8 - xfailDataDrivenWithQVerify(Abort) # TODO This test should xfail +ok 12 - xfailDataDrivenWithQString(Pass Abort) # SKIP Each Continue or Pass reports this and increments skip-count +ok 13 - xfailDataDrivenWithQString(Pass Continue) # SKIP Each Continue or Pass reports this and increments skip-count +not ok 14 - xfailDataDrivenWithQString(Fail Abort) # TODO A string + --- + at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO A string + --- + at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO Bug 5 (The message) + --- + at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +ok 15 - xfailDataDrivenWithQString(Fail Continue) # SKIP Each Continue or Pass reports this and increments skip-count +ok 16 - xfailDataDrivenWithQVerify(Pass Abort) +ok 17 - xfailDataDrivenWithQVerify(Pass Continue) +not ok 18 - xfailDataDrivenWithQVerify(Fail Abort) # TODO This test should xfail --- at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 9 - xfailDataDrivenWithQVerify(Continue) # TODO This test should xfail +not ok 19 - xfailDataDrivenWithQVerify(Fail Continue) # TODO This test should xfail --- at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -ok 10 - xfailDataDrivenWithQCompare(Pass 1) -ok 11 - xfailDataDrivenWithQCompare(Pass 2) -not ok 12 - xfailDataDrivenWithQCompare(Abort) # TODO This test should xfail +ok 20 - xfailDataDrivenWithQCompare(Pass Abort) +ok 21 - xfailDataDrivenWithQCompare(Pass Continue) +not ok 22 - xfailDataDrivenWithQCompare(Fail Abort) # TODO This test should xfail --- at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 13 - xfailDataDrivenWithQCompare(Continue) # TODO This test should xfail +not ok 23 - xfailDataDrivenWithQCompare(Fail Continue) # TODO This test should xfail --- at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -ok 14 - xfailOnWrongRow(right row) -not ok 15 - xfailOnAnyRow(first row) # TODO This test should xfail +# Should pass (*not* xpass), despite test-case name +ok 24 - xfailOnWrongRow(Fail Abort) +# Should pass (*not* xpass), despite test-case name +ok 25 - xfailOnWrongRow(Fail Continue) +not ok 26 - xfailOnAnyRow(Fail Abort) # TODO This test should xfail --- at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 16 - xfailOnAnyRow(second row) # TODO This test should xfail +not ok 27 - xfailOnAnyRow(Fail Continue) # TODO This test should xfail --- at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 ... -not ok 17 - xfailWithoutVerify(first row) +# Should fail (*not* xfail), despite test-case name +not ok 28 - xfailWithoutCheck(Fail Abort) --- # QEXPECT_FAIL was called without any subsequent verification statements ... -not ok 18 - xfailWithoutVerify(second row) +# Should fail (*not* xfail), despite test-case name +not ok 29 - xfailWithoutCheck(Fail Continue) --- # QEXPECT_FAIL was called without any subsequent verification statements ... -ok 19 - xpass() # TODO 'true' returned TRUE unexpectedly. () -ok 20 - xpassDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () -ok 21 - xpassDataDrivenWithQVerify(Pass) -ok 22 - xpassDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. -ok 23 - xpassDataDrivenWithQCompare(Pass) -ok 24 - cleanupTestCase() -1..24 -# tests 24 -# pass 18 -# fail 6 +ok 30 - xpassAbort() # TODO 'true' returned TRUE unexpectedly. () +ok 31 - xpassAbortSkip() # TODO 'true' returned TRUE unexpectedly. () +ok 32 - xpassAbortXfailContinue() # TODO 'true' returned TRUE unexpectedly. () +ok 33 - xpassContinue() # TODO 'true' returned TRUE unexpectedly. () +# This should be reached +ok 34 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. () +ok 35 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count +ok 36 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. () +not ok 37 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals + --- + at: tst_ExpectFail::xpassContinueXfailAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + ... +ok 37 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () +ok 38 - xpassAbortDataDrivenWithQVerify(Pass) +ok 39 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () +# Test should Continue past XPASS +# Test should simply PASS +ok 40 - xpassContinueDataDrivenWithQVerify(Pass) +ok 41 - xpassAbortDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. +ok 42 - xpassAbortDataDrivenWithQCompare(Pass) +ok 43 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly. +# Test should Continue past XPASS +# Test should simply PASS +ok 44 - xpassContinueDataDrivenWithQCompare(Pass) +ok 45 - cleanupTestCase() +1..45 +# tests 45 +# pass 23 +# fail 17 diff --git a/tests/auto/testlib/selftests/expected_expectfail.teamcity b/tests/auto/testlib/selftests/expected_expectfail.teamcity index cac2c34491..738ef664ec 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.teamcity +++ b/tests/auto/testlib/selftests/expected_expectfail.teamcity @@ -7,59 +7,120 @@ ##teamcity[testStarted name='xfailAndAbort()' flowId='tst_ExpectFail'] ##teamcity[testStdOut name='xfailAndAbort()' out='QDEBUG: begin|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This should xfail' flowId='tst_ExpectFail'] ##teamcity[testFinished name='xfailAndAbort()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailContinueSkip()' flowId='tst_ExpectFail'] +##teamcity[testIgnored name='xfailContinueSkip()' message='This skip should be reported and counted |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailAbortSkip()' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailAbortSkip()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This should xfail then skip|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailAbortSkip()' flowId='tst_ExpectFail'] ##teamcity[testStarted name='xfailTwice()' flowId='tst_ExpectFail'] ##teamcity[testFailed name='xfailTwice()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail'] ##teamcity[testFinished name='xfailTwice()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenTwice(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailDataDrivenTwice(Pass Abort)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenTwice(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenTwice(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailDataDrivenTwice(Pass Continue)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenTwice(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenTwice(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailDataDrivenTwice(Fail Abort)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenTwice(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenTwice(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailDataDrivenTwice(Fail Continue)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='Already expecting a fail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenTwice(Fail Continue)' flowId='tst_ExpectFail'] ##teamcity[testStarted name='xfailWithQString()' flowId='tst_ExpectFail'] ##teamcity[testStdOut name='xfailWithQString()' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: A string|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: Bug 5 (The message)' flowId='tst_ExpectFail'] ##teamcity[testFinished name='xfailWithQString()' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQVerify(Pass 1)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQVerify(Pass 1)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQVerify(Pass 2)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQVerify(Pass 2)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQVerify(Abort)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQVerify(Abort)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQVerify(Continue)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQVerify(Continue)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQCompare(Pass 1)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQCompare(Pass 1)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQCompare(Pass 2)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQCompare(Pass 2)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQCompare(Abort)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailDataDrivenWithQCompare(Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQCompare(Abort)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailDataDrivenWithQCompare(Continue)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailDataDrivenWithQCompare(Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailDataDrivenWithQCompare(Continue)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailOnWrongRow(right row)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailOnWrongRow(right row)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailOnAnyRow(first row)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailOnAnyRow(first row)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailOnAnyRow(first row)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailOnAnyRow(second row)' flowId='tst_ExpectFail'] -##teamcity[testStdOut name='xfailOnAnyRow(second row)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailOnAnyRow(second row)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailWithoutVerify(first row)' flowId='tst_ExpectFail'] -##teamcity[testFailed name='xfailWithoutVerify(first row)' message='Failure!' details='QEXPECT_FAIL was called without any subsequent verification statements' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailWithoutVerify(first row)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xfailWithoutVerify(second row)' flowId='tst_ExpectFail'] -##teamcity[testFailed name='xfailWithoutVerify(second row)' message='Failure!' details='QEXPECT_FAIL was called without any subsequent verification statements' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xfailWithoutVerify(second row)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xpass()' flowId='tst_ExpectFail'] -##teamcity[testFailed name='xpass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xpass()' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xpassDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] -##teamcity[testFailed name='xpassDataDrivenWithQVerify(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xpassDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xpassDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xpassDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xpassDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] -##teamcity[testFailed name='xpassDataDrivenWithQCompare(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='QCOMPARE(1, 1) returned TRUE unexpectedly.' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xpassDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] -##teamcity[testStarted name='xpassDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] -##teamcity[testFinished name='xpassDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] +##teamcity[testIgnored name='xfailDataDrivenWithQString(Pass Abort)' message='Each Continue or Pass reports this and increments skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail'] +##teamcity[testIgnored name='xfailDataDrivenWithQString(Pass Continue)' message='Each Continue or Pass reports this and increments skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQString(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQString(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: A string' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQString(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQString(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testIgnored name='xfailDataDrivenWithQString(Fail Continue)' message='Each Continue or Pass reports this and increments skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQVerify(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Pass Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: A string|nXFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: Bug 5 (The message)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQVerify(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQVerify(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQVerify(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQVerify(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQVerify(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQVerify(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQVerify(Fail Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQVerify(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQCompare(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQCompare(Pass Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQCompare(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQCompare(Pass Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQCompare(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQCompare(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQCompare(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailDataDrivenWithQCompare(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailDataDrivenWithQCompare(Fail Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailDataDrivenWithQCompare(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailOnWrongRow(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailOnWrongRow(Fail Abort)' out='QDEBUG: Should pass (*not* xpass), despite test-case name' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailOnWrongRow(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailOnWrongRow(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailOnWrongRow(Fail Continue)' out='QDEBUG: Should pass (*not* xpass), despite test-case name' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailOnWrongRow(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailOnAnyRow(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailOnAnyRow(Fail Abort)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailOnAnyRow(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailOnAnyRow(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailOnAnyRow(Fail Continue)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailOnAnyRow(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailWithoutCheck(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailWithoutCheck(Fail Abort)' message='Failure!' details='QEXPECT_FAIL was called without any subsequent verification statements' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailWithoutCheck(Fail Abort)' out='QDEBUG: Should fail (*not* xfail), despite test-case name' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailWithoutCheck(Fail Abort)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xfailWithoutCheck(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xfailWithoutCheck(Fail Continue)' message='Failure!' details='QEXPECT_FAIL was called without any subsequent verification statements' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xfailWithoutCheck(Fail Continue)' out='QDEBUG: Should fail (*not* xfail), despite test-case name' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xfailWithoutCheck(Fail Continue)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbort()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassAbort()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbort()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortSkip()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassAbortSkip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortSkip()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortXfailContinue()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassAbortXfailContinue()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortXfailContinue()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinue()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassContinue()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinue()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueSkip()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassContinueSkip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xpassContinueSkip()' out='QDEBUG: This should be reached' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueSkip()' flowId='tst_ExpectFail'] +##teamcity[testIgnored name='xpassContinueSkip()' message='This should be reached but not increment skip-count |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueXfailAbort()' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassContinueXfailAbort()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueXfailAbort()' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassAbortDataDrivenWithQVerify(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xpassAbortDataDrivenWithQVerify(XPass)' out='XFAIL |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]: This test should xfail but not add to totals' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassContinueDataDrivenWithQVerify(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='|'true|' returned TRUE unexpectedly. ()' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueDataDrivenWithQVerify(XPass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xpassContinueDataDrivenWithQVerify(Pass)' out='QDEBUG: Test should Continue past XPASS|nQDEBUG: Test should simply PASS' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueDataDrivenWithQVerify(Pass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassAbortDataDrivenWithQCompare(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='QCOMPARE(1, 1) returned TRUE unexpectedly.' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassAbortDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassAbortDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] +##teamcity[testFailed name='xpassContinueDataDrivenWithQCompare(XPass)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)|]' details='QCOMPARE(1, 1) returned TRUE unexpectedly.' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueDataDrivenWithQCompare(XPass)' flowId='tst_ExpectFail'] +##teamcity[testStarted name='xpassContinueDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='xpassContinueDataDrivenWithQCompare(Pass)' out='QDEBUG: Test should Continue past XPASS|nQDEBUG: Test should simply PASS' flowId='tst_ExpectFail'] +##teamcity[testFinished name='xpassContinueDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_ExpectFail'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_ExpectFail'] ##teamcity[testSuiteFinished name='tst_ExpectFail' flowId='tst_ExpectFail'] diff --git a/tests/auto/testlib/selftests/expected_expectfail.txt b/tests/auto/testlib/selftests/expected_expectfail.txt index 75d41e5733..83f673f2f3 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.txt +++ b/tests/auto/testlib/selftests/expected_expectfail.txt @@ -10,46 +10,104 @@ QDEBUG : tst_ExpectFail::xfailAndAbort() begin XFAIL : tst_ExpectFail::xfailAndAbort() This should xfail Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] PASS : tst_ExpectFail::xfailAndAbort() +XFAIL : tst_ExpectFail::xfailContinueSkip() This should xfail then skip + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +SKIP : tst_ExpectFail::xfailContinueSkip() This skip should be reported and counted + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XFAIL : tst_ExpectFail::xfailAbortSkip() This should xfail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailAbortSkip() FAIL! : tst_ExpectFail::xfailTwice() Already expecting a fail Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +FAIL! : tst_ExpectFail::xfailDataDrivenTwice(Pass Abort) Already expecting a fail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +FAIL! : tst_ExpectFail::xfailDataDrivenTwice(Pass Continue) Already expecting a fail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +FAIL! : tst_ExpectFail::xfailDataDrivenTwice(Fail Abort) Already expecting a fail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +FAIL! : tst_ExpectFail::xfailDataDrivenTwice(Fail Continue) Already expecting a fail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] XFAIL : tst_ExpectFail::xfailWithQString() A string Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] XFAIL : tst_ExpectFail::xfailWithQString() Bug 5 (The message) Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] PASS : tst_ExpectFail::xfailWithQString() -PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Pass 1) -PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Pass 2) -XFAIL : tst_ExpectFail::xfailDataDrivenWithQVerify(Abort) This test should xfail +SKIP : tst_ExpectFail::xfailDataDrivenWithQString(Pass Abort) Each Continue or Pass reports this and increments skip-count + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +SKIP : tst_ExpectFail::xfailDataDrivenWithQString(Pass Continue) Each Continue or Pass reports this and increments skip-count + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XFAIL : tst_ExpectFail::xfailDataDrivenWithQString(Fail Abort) A string + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailDataDrivenWithQString(Fail Abort) +XFAIL : tst_ExpectFail::xfailDataDrivenWithQString(Fail Continue) A string + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XFAIL : tst_ExpectFail::xfailDataDrivenWithQString(Fail Continue) Bug 5 (The message) + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +SKIP : tst_ExpectFail::xfailDataDrivenWithQString(Fail Continue) Each Continue or Pass reports this and increments skip-count + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Pass Abort) +PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Pass Continue) +XFAIL : tst_ExpectFail::xfailDataDrivenWithQVerify(Fail Abort) This test should xfail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Fail Abort) +XFAIL : tst_ExpectFail::xfailDataDrivenWithQVerify(Fail Continue) This test should xfail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Fail Continue) +PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Pass Abort) +PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Pass Continue) +XFAIL : tst_ExpectFail::xfailDataDrivenWithQCompare(Fail Abort) This test should xfail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Fail Abort) +XFAIL : tst_ExpectFail::xfailDataDrivenWithQCompare(Fail Continue) This test should xfail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Fail Continue) +QDEBUG : tst_ExpectFail::xfailOnWrongRow(Fail Abort) Should pass (*not* xpass), despite test-case name +PASS : tst_ExpectFail::xfailOnWrongRow(Fail Abort) +QDEBUG : tst_ExpectFail::xfailOnWrongRow(Fail Continue) Should pass (*not* xpass), despite test-case name +PASS : tst_ExpectFail::xfailOnWrongRow(Fail Continue) +XFAIL : tst_ExpectFail::xfailOnAnyRow(Fail Abort) This test should xfail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailOnAnyRow(Fail Abort) +XFAIL : tst_ExpectFail::xfailOnAnyRow(Fail Continue) This test should xfail + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +PASS : tst_ExpectFail::xfailOnAnyRow(Fail Continue) +QDEBUG : tst_ExpectFail::xfailWithoutCheck(Fail Abort) Should fail (*not* xfail), despite test-case name +FAIL! : tst_ExpectFail::xfailWithoutCheck(Fail Abort) QEXPECT_FAIL was called without any subsequent verification statements +QDEBUG : tst_ExpectFail::xfailWithoutCheck(Fail Continue) Should fail (*not* xfail), despite test-case name +FAIL! : tst_ExpectFail::xfailWithoutCheck(Fail Continue) QEXPECT_FAIL was called without any subsequent verification statements +XPASS : tst_ExpectFail::xpassAbort() 'true' returned TRUE unexpectedly. () + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XPASS : tst_ExpectFail::xpassAbortSkip() 'true' returned TRUE unexpectedly. () + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XPASS : tst_ExpectFail::xpassAbortXfailContinue() 'true' returned TRUE unexpectedly. () + Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] +XPASS : tst_ExpectFail::xpassContinue() 'true' returned TRUE unexpectedly. () Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Abort) -XFAIL : tst_ExpectFail::xfailDataDrivenWithQVerify(Continue) This test should xfail +QDEBUG : tst_ExpectFail::xpassContinue() This should be reached +XPASS : tst_ExpectFail::xpassContinueSkip() 'true' returned TRUE unexpectedly. () Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailDataDrivenWithQVerify(Continue) -PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Pass 1) -PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Pass 2) -XFAIL : tst_ExpectFail::xfailDataDrivenWithQCompare(Abort) This test should xfail +SKIP : tst_ExpectFail::xpassContinueSkip() This should be reached but not increment skip-count Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Abort) -XFAIL : tst_ExpectFail::xfailDataDrivenWithQCompare(Continue) This test should xfail +XPASS : tst_ExpectFail::xpassContinueXfailAbort() 'true' returned TRUE unexpectedly. () Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailDataDrivenWithQCompare(Continue) -PASS : tst_ExpectFail::xfailOnWrongRow(right row) -XFAIL : tst_ExpectFail::xfailOnAnyRow(first row) This test should xfail +XFAIL : tst_ExpectFail::xpassContinueXfailAbort() This test should xfail but not add to totals Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailOnAnyRow(first row) -XFAIL : tst_ExpectFail::xfailOnAnyRow(second row) This test should xfail +XPASS : tst_ExpectFail::xpassAbortDataDrivenWithQVerify(XPass) 'true' returned TRUE unexpectedly. () Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xfailOnAnyRow(second row) -FAIL! : tst_ExpectFail::xfailWithoutVerify(first row) QEXPECT_FAIL was called without any subsequent verification statements -FAIL! : tst_ExpectFail::xfailWithoutVerify(second row) QEXPECT_FAIL was called without any subsequent verification statements -XPASS : tst_ExpectFail::xpass() 'true' returned TRUE unexpectedly. () +PASS : tst_ExpectFail::xpassAbortDataDrivenWithQVerify(Pass) +XPASS : tst_ExpectFail::xpassContinueDataDrivenWithQVerify(XPass) 'true' returned TRUE unexpectedly. () Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -XPASS : tst_ExpectFail::xpassDataDrivenWithQVerify(XPass) 'true' returned TRUE unexpectedly. () +QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQVerify(XPass) Test should Continue past XPASS +QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQVerify(Pass) Test should simply PASS +PASS : tst_ExpectFail::xpassContinueDataDrivenWithQVerify(Pass) +XPASS : tst_ExpectFail::xpassAbortDataDrivenWithQCompare(XPass) QCOMPARE(1, 1) returned TRUE unexpectedly. Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xpassDataDrivenWithQVerify(Pass) -XPASS : tst_ExpectFail::xpassDataDrivenWithQCompare(XPass) QCOMPARE(1, 1) returned TRUE unexpectedly. +PASS : tst_ExpectFail::xpassAbortDataDrivenWithQCompare(Pass) +XPASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) QCOMPARE(1, 1) returned TRUE unexpectedly. Loc: [qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(0)] -PASS : tst_ExpectFail::xpassDataDrivenWithQCompare(Pass) +QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) Test should Continue past XPASS +QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) Test should simply PASS +PASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) PASS : tst_ExpectFail::cleanupTestCase() -Totals: 18 passed, 6 failed, 0 skipped, 0 blacklisted, 0ms +Totals: 23 passed, 17 failed, 5 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_ExpectFail ********* diff --git a/tests/auto/testlib/selftests/expected_expectfail.xml b/tests/auto/testlib/selftests/expected_expectfail.xml index ccfd942c92..27aae6b75c 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.xml +++ b/tests/auto/testlib/selftests/expected_expectfail.xml @@ -32,12 +32,47 @@ <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> +<TestFunction name="xfailContinueSkip"> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA[This should xfail then skip]]></Description> +</Incident> +<Message type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA[This skip should be reported and counted]]></Description> +</Message> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xfailAbortSkip"> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA[This should xfail]]></Description> +</Incident> +<Incident type="pass" file="" line="0" /> + <Duration msecs="0"/> +</TestFunction> <TestFunction name="xfailTwice"> <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> <Description><![CDATA[Already expecting a fail]]></Description> </Incident> <Duration msecs="0"/> </TestFunction> +<TestFunction name="xfailDataDrivenTwice"> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Pass Abort]]></DataTag> + <Description><![CDATA[Already expecting a fail]]></Description> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Pass Continue]]></DataTag> + <Description><![CDATA[Already expecting a fail]]></Description> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Abort]]></DataTag> + <Description><![CDATA[Already expecting a fail]]></Description> +</Incident> +<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[Already expecting a fail]]></Description> +</Incident> + <Duration msecs="0"/> +</TestFunction> <TestFunction name="xfailWithQString"> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> <Description><![CDATA[A string]]></Description> @@ -48,93 +83,181 @@ <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> +<TestFunction name="xfailDataDrivenWithQString"> +<Message type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Pass Abort]]></DataTag> + <Description><![CDATA[Each Continue or Pass reports this and increments skip-count]]></Description> +</Message> +<Message type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Pass Continue]]></DataTag> + <Description><![CDATA[Each Continue or Pass reports this and increments skip-count]]></Description> +</Message> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Abort]]></DataTag> + <Description><![CDATA[A string]]></Description> +</Incident> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[Fail Abort]]></DataTag> +</Incident> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[A string]]></Description> +</Incident> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[Bug 5 (The message)]]></Description> +</Incident> +<Message type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[Each Continue or Pass reports this and increments skip-count]]></Description> +</Message> + <Duration msecs="0"/> +</TestFunction> <TestFunction name="xfailDataDrivenWithQVerify"> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Pass 1]]></DataTag> + <DataTag><![CDATA[Pass Abort]]></DataTag> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Pass 2]]></DataTag> + <DataTag><![CDATA[Pass Continue]]></DataTag> </Incident> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[Abort]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Abort]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> </Incident> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[Continue]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Continue]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> </Incident> <Duration msecs="0"/> </TestFunction> <TestFunction name="xfailDataDrivenWithQCompare"> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Pass 1]]></DataTag> + <DataTag><![CDATA[Pass Abort]]></DataTag> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Pass 2]]></DataTag> + <DataTag><![CDATA[Pass Continue]]></DataTag> </Incident> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[Abort]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Abort]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> </Incident> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[Continue]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[Continue]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> </Incident> <Duration msecs="0"/> </TestFunction> <TestFunction name="xfailOnWrongRow"> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Fail Abort]]></DataTag> + <Description><![CDATA[Should pass (*not* xpass), despite test-case name]]></Description> +</Message> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[right row]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> +</Incident> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[Should pass (*not* xpass), despite test-case name]]></Description> +</Message> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> </Incident> <Duration msecs="0"/> </TestFunction> <TestFunction name="xfailOnAnyRow"> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[first row]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[first row]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> </Incident> <Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> - <DataTag><![CDATA[second row]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> <Description><![CDATA[This test should xfail]]></Description> </Incident> <Incident type="pass" file="" line="0"> - <DataTag><![CDATA[second row]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> </Incident> <Duration msecs="0"/> </TestFunction> -<TestFunction name="xfailWithoutVerify"> +<TestFunction name="xfailWithoutCheck"> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Fail Abort]]></DataTag> + <Description><![CDATA[Should fail (*not* xfail), despite test-case name]]></Description> +</Message> <Incident type="fail" file="" line="0"> - <DataTag><![CDATA[first row]]></DataTag> + <DataTag><![CDATA[Fail Abort]]></DataTag> <Description><![CDATA[QEXPECT_FAIL was called without any subsequent verification statements]]></Description> </Incident> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Fail Continue]]></DataTag> + <Description><![CDATA[Should fail (*not* xfail), despite test-case name]]></Description> +</Message> <Incident type="fail" file="" line="0"> - <DataTag><![CDATA[second row]]></DataTag> + <DataTag><![CDATA[Fail Continue]]></DataTag> <Description><![CDATA[QEXPECT_FAIL was called without any subsequent verification statements]]></Description> </Incident> <Duration msecs="0"/> </TestFunction> -<TestFunction name="xpass"> +<TestFunction name="xpassAbort"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassAbortSkip"> <Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> </Incident> <Duration msecs="0"/> </TestFunction> -<TestFunction name="xpassDataDrivenWithQVerify"> +<TestFunction name="xpassAbortXfailContinue"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassContinue"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> +<Message type="qdebug" file="" line="0"> + <Description><![CDATA[This should be reached]]></Description> +</Message> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassContinueSkip"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> +<Message type="skip" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA[This should be reached but not increment skip-count]]></Description> +</Message> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassContinueXfailAbort"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> +<Incident type="xfail" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <Description><![CDATA[This test should xfail but not add to totals]]></Description> +</Incident> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassAbortDataDrivenWithQVerify"> <Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> <DataTag><![CDATA[XPass]]></DataTag> <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> @@ -144,7 +267,25 @@ </Incident> <Duration msecs="0"/> </TestFunction> -<TestFunction name="xpassDataDrivenWithQCompare"> +<TestFunction name="xpassContinueDataDrivenWithQVerify"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[XPass]]></DataTag> + <Description><![CDATA['true' returned TRUE unexpectedly. ()]]></Description> +</Incident> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[XPass]]></DataTag> + <Description><![CDATA[Test should Continue past XPASS]]></Description> +</Message> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Pass]]></DataTag> + <Description><![CDATA[Test should simply PASS]]></Description> +</Message> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[Pass]]></DataTag> +</Incident> + <Duration msecs="0"/> +</TestFunction> +<TestFunction name="xpassAbortDataDrivenWithQCompare"> <Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> <DataTag><![CDATA[XPass]]></DataTag> <Description><![CDATA[QCOMPARE(1, 1) returned TRUE unexpectedly.]]></Description> @@ -154,6 +295,24 @@ </Incident> <Duration msecs="0"/> </TestFunction> +<TestFunction name="xpassContinueDataDrivenWithQCompare"> +<Incident type="xpass" file="qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="0"> + <DataTag><![CDATA[XPass]]></DataTag> + <Description><![CDATA[QCOMPARE(1, 1) returned TRUE unexpectedly.]]></Description> +</Incident> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[XPass]]></DataTag> + <Description><![CDATA[Test should Continue past XPASS]]></Description> +</Message> +<Message type="qdebug" file="" line="0"> + <DataTag><![CDATA[Pass]]></DataTag> + <Description><![CDATA[Test should simply PASS]]></Description> +</Message> +<Incident type="pass" file="" line="0"> + <DataTag><![CDATA[Pass]]></DataTag> +</Incident> + <Duration msecs="0"/> +</TestFunction> <TestFunction name="cleanupTestCase"> <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> diff --git a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp index fb6300b4a8..b97244b476 100644 --- a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp +++ b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2021 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the test suite of the Qt Toolkit. @@ -39,23 +39,42 @@ class tst_ExpectFail: public QObject private slots: void xfailAndContinue() const; void xfailAndAbort() const; + void xfailContinueSkip() const; + void xfailAbortSkip() const; void xfailTwice() const; + void xfailDataDrivenTwice() const; + void xfailDataDrivenTwice_data() const { xfailDataDriven_data(false); } void xfailWithQString() const; - void xfailDataDrivenWithQVerify_data() const; + void xfailDataDrivenWithQString_data() const { xfailDataDriven_data(false); } + void xfailDataDrivenWithQString() const; + void xfailDataDrivenWithQVerify_data() const { xfailDataDriven_data(false); } void xfailDataDrivenWithQVerify() const; - void xfailDataDrivenWithQCompare_data() const; + void xfailDataDrivenWithQCompare_data() const { xfailDataDriven_data(false); } void xfailDataDrivenWithQCompare() const; - void xfailOnWrongRow_data() const; + void xfailOnWrongRow_data() const { xfailDataDriven_data(true); } void xfailOnWrongRow() const; - void xfailOnAnyRow_data() const; + void xfailOnAnyRow_data() const { xfailDataDriven_data(true); } void xfailOnAnyRow() const; - void xfailWithoutVerify_data() const; - void xfailWithoutVerify() const; - void xpass() const; - void xpassDataDrivenWithQVerify_data() const; - void xpassDataDrivenWithQVerify() const; - void xpassDataDrivenWithQCompare_data() const; - void xpassDataDrivenWithQCompare() const; + void xfailWithoutCheck_data() const { xfailDataDriven_data(true); } + void xfailWithoutCheck() const; + void xpassAbort() const; + void xpassAbortSkip() const; + void xpassAbortXfailContinue() const; + void xpassContinue() const; + void xpassContinueSkip() const; + void xpassContinueXfailAbort() const; + void xpassAbortDataDrivenWithQVerify_data() const { xpassDataDriven_data(); } + void xpassAbortDataDrivenWithQVerify() const; + void xpassContinueDataDrivenWithQVerify_data() const { xpassDataDriven_data(); } + void xpassContinueDataDrivenWithQVerify() const; + void xpassAbortDataDrivenWithQCompare_data() const { xpassDataDriven_data(); } + void xpassAbortDataDrivenWithQCompare() const; + void xpassContinueDataDrivenWithQCompare_data() const { xpassDataDriven_data(); } + void xpassContinueDataDrivenWithQCompare() const; + +private: + void xfailDataDriven_data(bool failOnly) const; + void xpassDataDriven_data() const; }; void tst_ExpectFail::xfailAndContinue() const @@ -76,11 +95,48 @@ void tst_ExpectFail::xfailAndAbort() const QVERIFY2(false, "This should not be reached"); } +void tst_ExpectFail::xfailContinueSkip() const +{ + QEXPECT_FAIL("", "This should xfail then skip", Continue); + QVERIFY(false); + QSKIP("This skip should be reported and counted"); +} + +void tst_ExpectFail::xfailAbortSkip() const +{ + QEXPECT_FAIL("", "This should xfail", Abort); + QVERIFY(false); + + // If we get here the test did not correctly abort on the previous QVERIFY. + QSKIP("This skip should not be reached"); +} + void tst_ExpectFail::xfailTwice() const { - QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Abort); + QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Continue); QEXPECT_FAIL("", "Calling QEXPECT_FAIL when already expecting a failure is " - "an error and should abort this test function", Abort); + "an error and should abort this test function", Continue); + + // If we get here the test did not correctly abort on the double call to QEXPECT_FAIL. + QVERIFY2(false, "This should not be reached"); +} + +void tst_ExpectFail::xfailDataDrivenTwice() const +{ + // Same with data-driven cases (twist semantics of unused shouldPass; we + // have four combinations to test): + QEXPECT_FAIL("Pass Abort", "Calling QEXPECT_FAIL once on a test-case is fine", Abort); + QEXPECT_FAIL("Pass Abort", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Continue); + QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL once on a test-case is fine", Abort); + QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Abort); + QEXPECT_FAIL("Pass Continue", "Calling QEXPECT_FAIL once on a test-case is fine", Continue); + QEXPECT_FAIL("Pass Continue", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Abort); + QEXPECT_FAIL("Fail Continue", "Calling QEXPECT_FAIL once on a test-case is fine", Continue); + QEXPECT_FAIL("Fail Continue", "Calling QEXPECT_FAIL when already expecting a failure is " + "an error and should abort this test function", Continue); // If we get here the test did not correctly abort on the double call to QEXPECT_FAIL. QVERIFY2(false, "This should not be reached"); @@ -92,134 +148,155 @@ void tst_ExpectFail::xfailWithQString() const QVERIFY(false); int bugNo = 5; - QString msg("The message"); - QEXPECT_FAIL( "", QString("Bug %1 (%2)").arg(bugNo).arg(msg).toLatin1().constData(), Continue); + QLatin1String msg("The message"); + QEXPECT_FAIL("", QString("Bug %1 (%2)").arg(bugNo).arg(msg).toLatin1().constData(), Continue); QVERIFY(false); } -void tst_ExpectFail::xfailDataDrivenWithQVerify_data() const +void tst_ExpectFail::xfailDataDrivenWithQString() const { - QTest::addColumn<bool>("shouldPass"); - QTest::addColumn<QTest::TestFailMode>("failMode"); + // This test does not (yet) distinguish the two Pass cases. + QFETCH(bool, shouldPass); + + QEXPECT_FAIL("Fail Abort", QString("A string").toLatin1().constData(), Abort); + QEXPECT_FAIL("Fail Continue", QString("A string").toLatin1().constData(), Continue); + // TODO: why aren't QVERIFY2()'s smessages seen ? + QVERIFY2(shouldPass, "Both Fail cases should XFAIL here"); + // Fail Abort should be gone now. + + int bugNo = 5; + QLatin1String msg("The message"); + QEXPECT_FAIL("Fail Continue", qPrintable(QString("Bug %1 (%2)").arg(bugNo).arg(msg)), Continue); + QVERIFY2(shouldPass, "Only Fail Continue should see this"); - QTest::newRow("Pass 1") << true << QTest::Abort; - QTest::newRow("Pass 2") << true << QTest::Continue; - QTest::newRow("Abort") << false << QTest::Abort; - QTest::newRow("Continue") << false << QTest::Continue; + // FAIL is a pass, and SKIP trumps pass. + QSKIP("Each Continue or Pass reports this and increments skip-count"); } void tst_ExpectFail::xfailDataDrivenWithQVerify() const { + // This test does not (yet) distinguish the two Pass cases. QFETCH(bool, shouldPass); QFETCH(QTest::TestFailMode, failMode); - // You can't pass a variable as the last parameter of QEXPECT_FAIL, - // because the macro adds "QTest::" in front of the last parameter. - // That is why the following code appears to be a little strange. - if (!shouldPass) { - if (failMode == QTest::Abort) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Abort); - else - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Continue); - } - - QVERIFY(shouldPass); + QEXPECT_FAIL("Fail Abort", "This test should xfail", Abort); + QEXPECT_FAIL("Fail Continue", "This test should xfail", Continue); + QVERIFY2(shouldPass, "Both Fail cases should XFAIL here"); - // If we get here, we either expected to pass or we expected to + // If we get here, either we expected to pass or we expected to // fail and the failure mode was Continue. if (!shouldPass) QCOMPARE(failMode, QTest::Continue); } -void tst_ExpectFail::xfailDataDrivenWithQCompare_data() const +void tst_ExpectFail::xfailDataDriven_data(bool failOnly) const { QTest::addColumn<bool>("shouldPass"); QTest::addColumn<QTest::TestFailMode>("failMode"); - QTest::newRow("Pass 1") << true << QTest::Abort; - QTest::newRow("Pass 2") << true << QTest::Continue; - QTest::newRow("Abort") << false << QTest::Abort; - QTest::newRow("Continue") << false << QTest::Continue; + if (!failOnly) { + QTest::newRow("Pass Abort") << true << QTest::Abort; + QTest::newRow("Pass Continue") << true << QTest::Continue; + } + QTest::newRow("Fail Abort") << false << QTest::Abort; + QTest::newRow("Fail Continue") << false << QTest::Continue; } void tst_ExpectFail::xfailDataDrivenWithQCompare() const { + // This test does not (yet) distinguish the two Pass cases. QFETCH(bool, shouldPass); QFETCH(QTest::TestFailMode, failMode); - // You can't pass a variable as the last parameter of QEXPECT_FAIL, - // because the macro adds "QTest::" in front of the last parameter. - // That is why the following code appears to be a little strange. - if (!shouldPass) { - if (failMode == QTest::Abort) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Abort); - else - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xfail", Continue); - } + QEXPECT_FAIL("Fail Abort", "This test should xfail", Abort); + QEXPECT_FAIL("Fail Continue", "This test should xfail", Continue); QCOMPARE(1, shouldPass ? 1 : 2); - // If we get here, we either expected to pass or we expected to + // If we get here, either we expected to pass or we expected to // fail and the failure mode was Continue. if (!shouldPass) QCOMPARE(failMode, QTest::Continue); } -void tst_ExpectFail::xfailOnWrongRow_data() const -{ - QTest::addColumn<int>("dummy"); - - QTest::newRow("right row") << 0; -} - void tst_ExpectFail::xfailOnWrongRow() const { - // QEXPECT_FAIL for a row that is not the current row should be ignored. - QEXPECT_FAIL("wrong row", "This xfail should be ignored", Abort); - QVERIFY(true); -} - -void tst_ExpectFail::xfailOnAnyRow_data() const -{ - QTest::addColumn<int>("dummy"); - - QTest::newRow("first row") << 0; - QTest::newRow("second row") << 1; + qDebug("Should pass (*not* xpass), despite test-case name"); + // QEXPECT_FAIL for a row that does not exist should be ignored. + // (It might be conditional data(), so exist in other circumstances.) + QFETCH(QTest::TestFailMode, failMode); + // You can't pass a variable as the last parameter of QEXPECT_FAIL, + // because the macro adds "QTest::" in front of the last parameter. + // That is why the following code appears to be a little strange. + if (failMode == QTest::Abort) + QEXPECT_FAIL("wrong row", "This xfail should be ignored", Abort); + else + QEXPECT_FAIL("wrong row", "This xfail should be ignored", Continue); + QTEST(false, "shouldPass"); // _data skips the passing tests as pointless } void tst_ExpectFail::xfailOnAnyRow() const { // In a data-driven test, passing an empty first parameter to QEXPECT_FAIL // should mean that the failure is expected for all data rows. - QEXPECT_FAIL("", "This test should xfail", Abort); - QVERIFY(false); + QFETCH(QTest::TestFailMode, failMode); + // You can't pass a variable as the last parameter of QEXPECT_FAIL, + // because the macro adds "QTest::" in front of the last parameter. + // That is why the following code appears to be a little strange. + if (failMode == QTest::Abort) + QEXPECT_FAIL("", "This test should xfail", Abort); + else + QEXPECT_FAIL("", "This test should xfail", Continue); + QTEST(true, "shouldPass"); // _data skips the passing tests as pointless } -void tst_ExpectFail::xfailWithoutVerify_data() const +void tst_ExpectFail::xfailWithoutCheck() const { - QTest::addColumn<int>("dummy"); + qDebug("Should fail (*not* xfail), despite test-case name"); + QTEST(false, "shouldPass"); // _data skips the passing tests as pass/fail is irrelevant + QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL without any subsequent check is an error", + Abort); + QEXPECT_FAIL("Fail Continue", "Calling QEXPECT_FAIL without any subsequent check is an error", + Continue); +} + +void tst_ExpectFail::xpassAbort() const +{ + QEXPECT_FAIL("", "This test should xpass", Abort); + QVERIFY(true); - QTest::newRow("first row") << 0; - QTest::newRow("second row") << 1; + // If we get here the test did not correctly abort on the previous + // unexpected pass. + QVERIFY2(false, "This should not be reached"); } -void tst_ExpectFail::xfailWithoutVerify() const +void tst_ExpectFail::xpassAbortSkip() const { + QEXPECT_FAIL("", "This test should xpass", Abort); QVERIFY(true); - QEXPECT_FAIL("", "This expected failure should be ignored", Abort); + + QSKIP("This should not be reached (and not add to skip-count)"); } -void tst_ExpectFail::xpass() const +void tst_ExpectFail::xpassAbortXfailContinue() const { QEXPECT_FAIL("", "This test should xpass", Abort); QVERIFY(true); // If we get here the test did not correctly abort on the previous // unexpected pass. + QEXPECT_FAIL("", "This should not be reached", Continue); QVERIFY2(false, "This should not be reached"); } -void tst_ExpectFail::xpassDataDrivenWithQVerify_data() const +void tst_ExpectFail::xpassContinue() const +{ + QEXPECT_FAIL("", "This test should xpass", Continue); + QVERIFY(true); + qDebug("This should be reached"); +} + +void tst_ExpectFail::xpassDataDriven_data() const { QTest::addColumn<bool>("shouldXPass"); @@ -227,38 +304,62 @@ void tst_ExpectFail::xpassDataDrivenWithQVerify_data() const QTest::newRow("Pass") << false; } -void tst_ExpectFail::xpassDataDrivenWithQVerify() const +void tst_ExpectFail::xpassContinueSkip() const { - QFETCH(bool, shouldXPass); + QEXPECT_FAIL("", "This test should xpass", Continue); + QVERIFY(true); + QSKIP("This should be reached but not increment skip-count"); +} - if (shouldXPass) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xpass", Abort); +void tst_ExpectFail::xpassContinueXfailAbort() const +{ + QEXPECT_FAIL("", "This test should xpass", Continue); + QVERIFY(true); + QEXPECT_FAIL("", "This test should xfail but not add to totals", Abort); + QVERIFY(false); + QVERIFY2(false, "This should not be reached"); +} +void tst_ExpectFail::xpassAbortDataDrivenWithQVerify() const +{ + QFETCH(bool, shouldXPass); + + QEXPECT_FAIL("XPass", "This test-row should xpass", Abort); QVERIFY(true); // We should only get here if the test wasn't supposed to xpass. - QVERIFY2(!shouldXPass, "Test failed to terminate on XPASS"); + QVERIFY2(!shouldXPass, "Test failed to Abort on XPASS"); } -void tst_ExpectFail::xpassDataDrivenWithQCompare_data() const +void tst_ExpectFail::xpassContinueDataDrivenWithQVerify() const { - QTest::addColumn<bool>("shouldXPass"); + QFETCH(bool, shouldXPass); - QTest::newRow("XPass") << true; - QTest::newRow("Pass") << false; + QEXPECT_FAIL("XPass", "This test-row should xpass", Continue); + QVERIFY(true); + + qDebug(shouldXPass ? "Test should Continue past XPASS" : "Test should simply PASS"); } -void tst_ExpectFail::xpassDataDrivenWithQCompare() const +void tst_ExpectFail::xpassAbortDataDrivenWithQCompare() const { QFETCH(bool, shouldXPass); - if (shouldXPass) - QEXPECT_FAIL(QTest::currentDataTag(), "This test should xpass", Abort); - + QEXPECT_FAIL("XPass", "This test should xpass", Abort); QCOMPARE(1, 1); // We should only get here if the test wasn't supposed to xpass. - QVERIFY2(!shouldXPass, "Test failed to terminate on XPASS"); + QVERIFY2(!shouldXPass, "Test failed to Abort on XPASS"); +} + +void tst_ExpectFail::xpassContinueDataDrivenWithQCompare() const +{ + QFETCH(bool, shouldXPass); + + QEXPECT_FAIL("XPass", "This test should xpass", Continue); + QCOMPARE(1, 1); + + qDebug(shouldXPass ? "Test should Continue past XPASS" : "Test should simply PASS"); } QTEST_MAIN(tst_ExpectFail) |