summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorEdward Welbourne <edward.welbourne@qt.io>2021-08-12 14:56:46 +0200
committerEdward Welbourne <edward.welbourne@qt.io>2021-08-27 20:16:18 +0200
commitbe72cb9cfdacf90c145bb02d34ab2e2cb79f3935 (patch)
tree43408c706345d9ee7e2d9b96caeef15f7629a284
parent7af79ba09148dbfe5c1fc8b130d564dcca7ae4b3 (diff)
Include check for the Totals line in testlib's expectedfail selftest
Count how many we expect to pass, fail and be skipped, so we can report that for comparison with the actual totals line: this reveals some double-counting, marked with a FIXME. Task-number: QTBUG-95661 Change-Id: I8b0f13ded5202ed476d8abfee70ed60f9e639bf9 Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io> Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
-rw-r--r--tests/auto/testlib/selftests/expected_expectfail.junitxml6
-rw-r--r--tests/auto/testlib/selftests/expected_expectfail.lightxml3
-rw-r--r--tests/auto/testlib/selftests/expected_expectfail.tap1
-rw-r--r--tests/auto/testlib/selftests/expected_expectfail.teamcity1
-rw-r--r--tests/auto/testlib/selftests/expected_expectfail.txt1
-rw-r--r--tests/auto/testlib/selftests/expected_expectfail.xml3
-rw-r--r--tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp57
7 files changed, 71 insertions, 1 deletions
diff --git a/tests/auto/testlib/selftests/expected_expectfail.junitxml b/tests/auto/testlib/selftests/expected_expectfail.junitxml
index 31521263ee..54f0416016 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.junitxml
+++ b/tests/auto/testlib/selftests/expected_expectfail.junitxml
@@ -180,5 +180,9 @@
<![CDATA[Test should simply PASS]]>
</system-out>
</testcase>
- <testcase name="cleanupTestCase" classname="tst_ExpectFail" time="@TEST_DURATION@"/>
+ <testcase name="cleanupTestCase" classname="tst_ExpectFail" time="@TEST_DURATION@">
+ <system-out>
+ <![CDATA[Totals should add up to 44: 23 passed, 17 failed, 4 skipped]]>
+ </system-out>
+ </testcase>
</testsuite>
diff --git a/tests/auto/testlib/selftests/expected_expectfail.lightxml b/tests/auto/testlib/selftests/expected_expectfail.lightxml
index 36636e13c4..4c2beec6ca 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.lightxml
+++ b/tests/auto/testlib/selftests/expected_expectfail.lightxml
@@ -312,6 +312,9 @@
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase">
+<Message type="qdebug" file="" line="0">
+ <Description><![CDATA[Totals should add up to 44: 23 passed, 17 failed, 4 skipped]]></Description>
+</Message>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_expectfail.tap b/tests/auto/testlib/selftests/expected_expectfail.tap
index 984815909a..ef464e41b0 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.tap
+++ b/tests/auto/testlib/selftests/expected_expectfail.tap
@@ -177,6 +177,7 @@ ok 43 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returne
# Test should Continue past XPASS
# Test should simply PASS
ok 44 - xpassContinueDataDrivenWithQCompare(Pass)
+# Totals should add up to 44: 23 passed, 17 failed, 4 skipped
ok 45 - cleanupTestCase()
1..45
# tests 45
diff --git a/tests/auto/testlib/selftests/expected_expectfail.teamcity b/tests/auto/testlib/selftests/expected_expectfail.teamcity
index 738ef664ec..774318bdb6 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.teamcity
+++ b/tests/auto/testlib/selftests/expected_expectfail.teamcity
@@ -122,5 +122,6 @@
##teamcity[testStdOut name='xpassContinueDataDrivenWithQCompare(Pass)' out='QDEBUG: Test should Continue past XPASS|nQDEBUG: Test should simply PASS' flowId='tst_ExpectFail']
##teamcity[testFinished name='xpassContinueDataDrivenWithQCompare(Pass)' flowId='tst_ExpectFail']
##teamcity[testStarted name='cleanupTestCase()' flowId='tst_ExpectFail']
+##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: Totals should add up to 44: 23 passed, 17 failed, 4 skipped' flowId='tst_ExpectFail']
##teamcity[testFinished name='cleanupTestCase()' flowId='tst_ExpectFail']
##teamcity[testSuiteFinished name='tst_ExpectFail' flowId='tst_ExpectFail']
diff --git a/tests/auto/testlib/selftests/expected_expectfail.txt b/tests/auto/testlib/selftests/expected_expectfail.txt
index 83f673f2f3..2dff4ad55f 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.txt
+++ b/tests/auto/testlib/selftests/expected_expectfail.txt
@@ -108,6 +108,7 @@ XPASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) QCOMPARE(1,
QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(XPass) Test should Continue past XPASS
QDEBUG : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass) Test should simply PASS
PASS : tst_ExpectFail::xpassContinueDataDrivenWithQCompare(Pass)
+QDEBUG : tst_ExpectFail::cleanupTestCase() Totals should add up to 44: 23 passed, 17 failed, 4 skipped
PASS : tst_ExpectFail::cleanupTestCase()
Totals: 23 passed, 17 failed, 5 skipped, 0 blacklisted, 0ms
********* Finished testing of tst_ExpectFail *********
diff --git a/tests/auto/testlib/selftests/expected_expectfail.xml b/tests/auto/testlib/selftests/expected_expectfail.xml
index 27aae6b75c..9c25b28981 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.xml
+++ b/tests/auto/testlib/selftests/expected_expectfail.xml
@@ -314,6 +314,9 @@
<Duration msecs="0"/>
</TestFunction>
<TestFunction name="cleanupTestCase">
+<Message type="qdebug" file="" line="0">
+ <Description><![CDATA[Totals should add up to 44: 23 passed, 17 failed, 4 skipped]]></Description>
+</Message>
<Incident type="pass" file="" line="0" />
<Duration msecs="0"/>
</TestFunction>
diff --git a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
index b97244b476..7b555b8293 100644
--- a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+++ b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
@@ -37,6 +37,8 @@ class tst_ExpectFail: public QObject
Q_OBJECT
private slots:
+ void cleanupTestCase() const;
+ void init() const;
void xfailAndContinue() const;
void xfailAndAbort() const;
void xfailContinueSkip() const;
@@ -76,9 +78,25 @@ private:
void xfailDataDriven_data(bool failOnly) const;
void xpassDataDriven_data() const;
};
+static int casesTested = 2;
+// What the totals line's numbers *should* be:
+static int passed = 2, failed = 0, skipped = 0;
+// Total and passed get {init,cleanup}TestCase() in addition to the actual tests.
+
+void tst_ExpectFail::init() const
+{
+ ++casesTested;
+}
+
+void tst_ExpectFail::cleanupTestCase() const
+{
+ qDebug("Totals should add up to %d: %d passed, %d failed, %d skipped",
+ casesTested, passed, failed, skipped);
+}
void tst_ExpectFail::xfailAndContinue() const
{
+ ++passed;
qDebug("begin");
QEXPECT_FAIL("", "This should xfail", Continue);
QVERIFY(false);
@@ -87,6 +105,7 @@ void tst_ExpectFail::xfailAndContinue() const
void tst_ExpectFail::xfailAndAbort() const
{
+ ++passed;
qDebug("begin");
QEXPECT_FAIL("", "This should xfail", Abort);
QVERIFY(false);
@@ -97,6 +116,7 @@ void tst_ExpectFail::xfailAndAbort() const
void tst_ExpectFail::xfailContinueSkip() const
{
+ ++skipped;
QEXPECT_FAIL("", "This should xfail then skip", Continue);
QVERIFY(false);
QSKIP("This skip should be reported and counted");
@@ -104,6 +124,7 @@ void tst_ExpectFail::xfailContinueSkip() const
void tst_ExpectFail::xfailAbortSkip() const
{
+ ++passed;
QEXPECT_FAIL("", "This should xfail", Abort);
QVERIFY(false);
@@ -113,6 +134,7 @@ void tst_ExpectFail::xfailAbortSkip() const
void tst_ExpectFail::xfailTwice() const
{
+ ++failed;
QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Continue);
QEXPECT_FAIL("", "Calling QEXPECT_FAIL when already expecting a failure is "
"an error and should abort this test function", Continue);
@@ -123,6 +145,7 @@ void tst_ExpectFail::xfailTwice() const
void tst_ExpectFail::xfailDataDrivenTwice() const
{
+ ++failed;
// Same with data-driven cases (twist semantics of unused shouldPass; we
// have four combinations to test):
QEXPECT_FAIL("Pass Abort", "Calling QEXPECT_FAIL once on a test-case is fine", Abort);
@@ -144,6 +167,7 @@ void tst_ExpectFail::xfailDataDrivenTwice() const
void tst_ExpectFail::xfailWithQString() const
{
+ ++passed;
QEXPECT_FAIL("", QString("A string").toLatin1().constData(), Continue);
QVERIFY(false);
@@ -157,6 +181,11 @@ void tst_ExpectFail::xfailDataDrivenWithQString() const
{
// This test does not (yet) distinguish the two Pass cases.
QFETCH(bool, shouldPass);
+ QFETCH(QTest::TestFailMode, failMode);
+ if (shouldPass || failMode == QTest::Continue)
+ ++skipped;
+ else
+ ++passed;
QEXPECT_FAIL("Fail Abort", QString("A string").toLatin1().constData(), Abort);
QEXPECT_FAIL("Fail Continue", QString("A string").toLatin1().constData(), Continue);
@@ -176,6 +205,7 @@ void tst_ExpectFail::xfailDataDrivenWithQString() const
void tst_ExpectFail::xfailDataDrivenWithQVerify() const
{
// This test does not (yet) distinguish the two Pass cases.
+ ++passed;
QFETCH(bool, shouldPass);
QFETCH(QTest::TestFailMode, failMode);
@@ -205,6 +235,7 @@ void tst_ExpectFail::xfailDataDriven_data(bool failOnly) const
void tst_ExpectFail::xfailDataDrivenWithQCompare() const
{
// This test does not (yet) distinguish the two Pass cases.
+ ++passed;
QFETCH(bool, shouldPass);
QFETCH(QTest::TestFailMode, failMode);
@@ -221,6 +252,7 @@ void tst_ExpectFail::xfailDataDrivenWithQCompare() const
void tst_ExpectFail::xfailOnWrongRow() const
{
+ ++passed;
qDebug("Should pass (*not* xpass), despite test-case name");
// QEXPECT_FAIL for a row that does not exist should be ignored.
// (It might be conditional data(), so exist in other circumstances.)
@@ -237,6 +269,7 @@ void tst_ExpectFail::xfailOnWrongRow() const
void tst_ExpectFail::xfailOnAnyRow() const
{
+ ++passed;
// In a data-driven test, passing an empty first parameter to QEXPECT_FAIL
// should mean that the failure is expected for all data rows.
QFETCH(QTest::TestFailMode, failMode);
@@ -252,6 +285,7 @@ void tst_ExpectFail::xfailOnAnyRow() const
void tst_ExpectFail::xfailWithoutCheck() const
{
+ ++failed;
qDebug("Should fail (*not* xfail), despite test-case name");
QTEST(false, "shouldPass"); // _data skips the passing tests as pass/fail is irrelevant
QEXPECT_FAIL("Fail Abort", "Calling QEXPECT_FAIL without any subsequent check is an error",
@@ -262,6 +296,7 @@ void tst_ExpectFail::xfailWithoutCheck() const
void tst_ExpectFail::xpassAbort() const
{
+ ++failed;
QEXPECT_FAIL("", "This test should xpass", Abort);
QVERIFY(true);
@@ -272,6 +307,7 @@ void tst_ExpectFail::xpassAbort() const
void tst_ExpectFail::xpassAbortSkip() const
{
+ ++failed;
QEXPECT_FAIL("", "This test should xpass", Abort);
QVERIFY(true);
@@ -280,6 +316,7 @@ void tst_ExpectFail::xpassAbortSkip() const
void tst_ExpectFail::xpassAbortXfailContinue() const
{
+ ++failed;
QEXPECT_FAIL("", "This test should xpass", Abort);
QVERIFY(true);
@@ -291,6 +328,7 @@ void tst_ExpectFail::xpassAbortXfailContinue() const
void tst_ExpectFail::xpassContinue() const
{
+ ++failed;
QEXPECT_FAIL("", "This test should xpass", Continue);
QVERIFY(true);
qDebug("This should be reached");
@@ -306,13 +344,16 @@ void tst_ExpectFail::xpassDataDriven_data() const
void tst_ExpectFail::xpassContinueSkip() const
{
+ ++failed; // and *not* ++skipped
QEXPECT_FAIL("", "This test should xpass", Continue);
QVERIFY(true);
+ // FIXME: QTBUG-95661 skip-count is incremented.
QSKIP("This should be reached but not increment skip-count");
}
void tst_ExpectFail::xpassContinueXfailAbort() const
{
+ ++failed;
QEXPECT_FAIL("", "This test should xpass", Continue);
QVERIFY(true);
QEXPECT_FAIL("", "This test should xfail but not add to totals", Abort);
@@ -323,6 +364,10 @@ void tst_ExpectFail::xpassContinueXfailAbort() const
void tst_ExpectFail::xpassAbortDataDrivenWithQVerify() const
{
QFETCH(bool, shouldXPass);
+ if (shouldXPass)
+ ++failed;
+ else
+ ++passed;
QEXPECT_FAIL("XPass", "This test-row should xpass", Abort);
QVERIFY(true);
@@ -334,6 +379,10 @@ void tst_ExpectFail::xpassAbortDataDrivenWithQVerify() const
void tst_ExpectFail::xpassContinueDataDrivenWithQVerify() const
{
QFETCH(bool, shouldXPass);
+ if (shouldXPass)
+ ++failed;
+ else
+ ++passed;
QEXPECT_FAIL("XPass", "This test-row should xpass", Continue);
QVERIFY(true);
@@ -344,6 +393,10 @@ void tst_ExpectFail::xpassContinueDataDrivenWithQVerify() const
void tst_ExpectFail::xpassAbortDataDrivenWithQCompare() const
{
QFETCH(bool, shouldXPass);
+ if (shouldXPass)
+ ++failed;
+ else
+ ++passed;
QEXPECT_FAIL("XPass", "This test should xpass", Abort);
QCOMPARE(1, 1);
@@ -355,6 +408,10 @@ void tst_ExpectFail::xpassAbortDataDrivenWithQCompare() const
void tst_ExpectFail::xpassContinueDataDrivenWithQCompare() const
{
QFETCH(bool, shouldXPass);
+ if (shouldXPass)
+ ++failed;
+ else
+ ++passed;
QEXPECT_FAIL("XPass", "This test should xpass", Continue);
QCOMPARE(1, 1);