diff options
7 files changed, 71 insertions, 1 deletions
diff --git a/tests/auto/testlib/selftests/expected_expectfail.junitxml b/tests/auto/testlib/selftests/expected_expectfail.junitxml index 31521263ee..54f0416016 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.junitxml +++ b/tests/auto/testlib/selftests/expected_expectfail.junitxml @@ -180,5 +180,9 @@ <![CDATA[Test should simply PASS]]> </system-out> </testcase> - <testcase name="cleanupTestCase" classname="tst_ExpectFail" time="@TEST_DURATION@"/> + <testcase name="cleanupTestCase" classname="tst_ExpectFail" time="@TEST_DURATION@"> + <system-out> + <![CDATA[Totals should add up to 44: 23 passed, 17 failed, 4 skipped]]> + </system-out> + </testcase> </testsuite> diff --git a/tests/auto/testlib/selftests/expected_expectfail.lightxml b/tests/auto/testlib/selftests/expected_expectfail.lightxml index 36636e13c4..4c2beec6ca 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.lightxml +++ b/tests/auto/testlib/selftests/expected_expectfail.lightxml @@ -312,6 +312,9 @@ <Duration msecs="0"/> </TestFunction> <TestFunction name="cleanupTestCase"> +<Message type="qdebug" file="" line="0"> + <Description><![CDATA[Totals should add up to 44: 23 passed, 17 failed, 4 skipped]]></Description> +</Message> <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> diff --git a/tests/auto/testlib/selftests/expected_expectfail.tap b/tests/auto/testlib/selftests/expected_expectfail.tap index 984815909a..ef464e41b0 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.tap +++ b/tests/auto/testlib/selftests/expected_expectfail.tap @@ -177,6 +177,7 @@ ok 43 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returne # Test should Continue past XPASS # Test should simply PASS ok 44 - xpassContinueDataDrivenWithQCompare(Pass) +# Totals should add up to 44: 23 passed, 17 failed, 4 skipped ok 45 - cleanupTestCase() 1..45 # tests 45 diff --git a/tests/auto/testlib/selftests/expected_expectfail.teamcity b/tests/auto/testlib/selftests/expected_expectfail.teamcity index 738ef664ec..774318bdb6 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.teamcity +++ b/tests/auto/testlib/selftests/expected_expectfail.teamcity @@ -122,5 +122,6 @@ ##teamcity[testStdOut name='xpassContinueDataDrivenWithQCompare(Pass)' out='QDEBUG: Test should Continue past XPASS|nQDEBUG: Test should simply PASS' flowId='tst_ExpectFail'] ##teamcity[testFinished name='xpassContinueDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_ExpectFail'] +##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 44: 23 passed, 17 failed, 4 skipped' flowId='tst_ExpectFail'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_ExpectFail'] ##teamcity[testSuiteFinished name='tst_ExpectFail' flowId='tst_ExpectFail'] diff --git a/tests/auto/testlib/selftests/expected_expectfail.txt b/tests/auto/testlib/selftests/expected_expectfail.txt index 83f673f2f3..2dff4ad55f 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.txt +++ b/tests/auto/testlib/selftests/expected_expectfail.txt @@ -108,6 +108,7 @@ XPASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) QCOMPARE(1, QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) Test should Continue past XPASS QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) Test should simply PASS PASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) +QDEBUG : tst_ExpectFail::cleanupTestCase() Totals should add up to 44: 23 passed, 17 failed, 4 skipped PASS : tst_ExpectFail::cleanupTestCase() Totals: 23 passed, 17 failed, 5 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_ExpectFail ********* diff --git a/tests/auto/testlib/selftests/expected_expectfail.xml b/tests/auto/testlib/selftests/expected_expectfail.xml index 27aae6b75c..9c25b28981 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.xml +++ b/tests/auto/testlib/selftests/expected_expectfail.xml @@ -314,6 +314,9 @@ <Duration msecs="0"/> </TestFunction> <TestFunction name="cleanupTestCase"> +<Message type="qdebug" file="" line="0"> + <Description><![CDATA[Totals should add up to 44: 23 passed, 17 failed, 4 skipped]]></Description> +</Message> <Incident type="pass" file="" line="0" /> <Duration msecs="0"/> </TestFunction> diff --git a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp index b97244b476..7b555b8293 100644 --- a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp +++ b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp @@ -37,6 +37,8 @@ class tst_ExpectFail: public QObject Q_OBJECT private slots: + void cleanupTestCase() const; + void init() const; void xfailAndContinue() const; void xfailAndAbort() const; void xfailContinueSkip() const; @@ -76,9 +78,25 @@ private: void xfailDataDriven_data(bool failOnly) const; void xpassDataDriven_data() const; }; +static int casesTested = 2; +// What the totals line's numbers *should* be: +static int passed = 2, failed = 0, skipped = 0; +// Total and passed get {init,cleanup}TestCase() in addition to the actual tests. + +void tst_ExpectFail::init() const +{ + ++casesTested; +} + +void tst_ExpectFail::cleanupTestCase() const +{ + qDebug("Totals should add up to %d: %d passed, %d failed, %d skipped", + casesTested, passed, failed, skipped); +} void tst_ExpectFail::xfailAndContinue() const { + ++passed; qDebug("begin"); QEXPECT_FAIL("", "This should xfail", Continue); QVERIFY(false); @@ -87,6 +105,7 @@ void tst_ExpectFail::xfailAndContinue() const void tst_ExpectFail::xfailAndAbort() const { + ++passed; qDebug("begin"); QEXPECT_FAIL("", "This should xfail", Abort); QVERIFY(false); @@ -97,6 +116,7 @@ void tst_ExpectFail::xfailAndAbort() const void tst_ExpectFail::xfailContinueSkip() const { + ++skipped; QEXPECT_FAIL("", "This should xfail then skip", Continue); QVERIFY(false); QSKIP("This skip should be reported and counted"); @@ -104,6 +124,7 @@ void tst_ExpectFail::xfailContinueSkip() const void tst_ExpectFail::xfailAbortSkip() const { + ++passed; QEXPECT_FAIL("", "This should xfail", Abort); QVERIFY(false); @@ -113,6 +134,7 @@ void tst_ExpectFail::xfailAbortSkip() const void tst_ExpectFail::xfailTwice() const { + ++failed; QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Continue); QEXPECT_FAIL("", "Calling QEXPECT_FAIL when already expecting a failure is " "an error and should abort this test function", Continue); @@ -123,6 +145,7 @@ void tst_ExpectFail::xfailTwice() const void tst_ExpectFail::xfailDataDrivenTwice() const { + ++failed; // Same with data-driven cases (twist semantics of unused shouldPass; we // have four combinations to test): QEXPECT_FAIL("Pass Abort", "Calling QEXPECT_FAIL once on a test-case is fine", Abort); @@ -144,6 +167,7 @@ void tst_ExpectFail::xfailDataDrivenTwice() const void tst_ExpectFail::xfailWithQString() const { + ++passed; QEXPECT_FAIL("", QString("A string").toLatin1().constData(), Continue); QVERIFY(false); @@ -157,6 +181,11 @@ void tst_ExpectFail::xfailDataDrivenWithQString() const { // This test does not (yet) distinguish the two Pass cases. QFETCH(bool, shouldPass); + QFETCH(QTest::TestFailMode, failMode); + if (shouldPass || failMode == QTest::Continue) + ++skipped; + else + ++passed; QEXPECT_FAIL("Fail Abort", QString("A string").toLatin1().constData(), Abort); QEXPECT_FAIL("Fail Continue", QString("A string").toLatin1().constData(), Continue); @@ -176,6 +205,7 @@ void tst_ExpectFail::xfailDataDrivenWithQString() const void tst_ExpectFail::xfailDataDrivenWithQVerify() const { // This test does not (yet) distinguish the two Pass cases. + ++passed; QFETCH(bool, shouldPass); QFETCH(QTest::TestFailMode, failMode); @@ -205,6 +235,7 @@ void tst_ExpectFail::xfailDataDriven_data(bool failOnly) const void tst_ExpectFail::xfailDataDrivenWithQCompare() const { // This test does not (yet) distinguish the two Pass cases. + ++passed; QFETCH(bool, shouldPass); QFETCH(QTest::TestFailMode, failMode); @@ -221,6 +252,7 @@ void tst_ExpectFail::xfailDataDrivenWithQCompare() const void tst_ExpectFail::xfailOnWrongRow() const { + ++passed; qDebug("Should pass (*not* xpass), despite test-case name"); // QEXPECT_FAIL for a row that does not exist should be ignored. // (It might be conditional data(), so exist in other circumstances.) @@ -237,6 +269,7 @@ void tst_ExpectFail::xfailOnWrongRow() const void tst_ExpectFail::xfailOnAnyRow() const { + ++passed; // In a data-driven test, passing an empty first parameter to QEXPECT_FAIL // should mean that the failure is expected for all data rows. QFETCH(QTest::TestFailMode, failMode); @@ -252,6 +285,7 @@ void tst_ExpectFail::xfailOnAnyRow() const void tst_ExpectFail::xfailWithoutCheck() const { + ++failed; qDebug("Should fail (*not* xfail), despite test-case name"); QTEST(false, "shouldPass"); // _data skips the passing tests as pass/fail is irrelevant QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL without any subsequent check is an error", @@ -262,6 +296,7 @@ void tst_ExpectFail::xfailWithoutCheck() const void tst_ExpectFail::xpassAbort() const { + ++failed; QEXPECT_FAIL("", "This test should xpass", Abort); QVERIFY(true); @@ -272,6 +307,7 @@ void tst_ExpectFail::xpassAbort() const void tst_ExpectFail::xpassAbortSkip() const { + ++failed; QEXPECT_FAIL("", "This test should xpass", Abort); QVERIFY(true); @@ -280,6 +316,7 @@ void tst_ExpectFail::xpassAbortSkip() const void tst_ExpectFail::xpassAbortXfailContinue() const { + ++failed; QEXPECT_FAIL("", "This test should xpass", Abort); QVERIFY(true); @@ -291,6 +328,7 @@ void tst_ExpectFail::xpassAbortXfailContinue() const void tst_ExpectFail::xpassContinue() const { + ++failed; QEXPECT_FAIL("", "This test should xpass", Continue); QVERIFY(true); qDebug("This should be reached"); @@ -306,13 +344,16 @@ void tst_ExpectFail::xpassDataDriven_data() const void tst_ExpectFail::xpassContinueSkip() const { + ++failed; // and *not* ++skipped QEXPECT_FAIL("", "This test should xpass", Continue); QVERIFY(true); + // FIXME: QTBUG-95661 skip-count is incremented. QSKIP("This should be reached but not increment skip-count"); } void tst_ExpectFail::xpassContinueXfailAbort() const { + ++failed; QEXPECT_FAIL("", "This test should xpass", Continue); QVERIFY(true); QEXPECT_FAIL("", "This test should xfail but not add to totals", Abort); @@ -323,6 +364,10 @@ void tst_ExpectFail::xpassContinueXfailAbort() const void tst_ExpectFail::xpassAbortDataDrivenWithQVerify() const { QFETCH(bool, shouldXPass); + if (shouldXPass) + ++failed; + else + ++passed; QEXPECT_FAIL("XPass", "This test-row should xpass", Abort); QVERIFY(true); @@ -334,6 +379,10 @@ void tst_ExpectFail::xpassAbortDataDrivenWithQVerify() const void tst_ExpectFail::xpassContinueDataDrivenWithQVerify() const { QFETCH(bool, shouldXPass); + if (shouldXPass) + ++failed; + else + ++passed; QEXPECT_FAIL("XPass", "This test-row should xpass", Continue); QVERIFY(true); @@ -344,6 +393,10 @@ void tst_ExpectFail::xpassContinueDataDrivenWithQVerify() const void tst_ExpectFail::xpassAbortDataDrivenWithQCompare() const { QFETCH(bool, shouldXPass); + if (shouldXPass) + ++failed; + else + ++passed; QEXPECT_FAIL("XPass", "This test should xpass", Abort); QCOMPARE(1, 1); @@ -355,6 +408,10 @@ void tst_ExpectFail::xpassAbortDataDrivenWithQCompare() const void tst_ExpectFail::xpassContinueDataDrivenWithQCompare() const { QFETCH(bool, shouldXPass); + if (shouldXPass) + ++failed; + else + ++passed; QEXPECT_FAIL("XPass", "This test should xpass", Continue); QCOMPARE(1, 1); |