Expand testing of QEXPECT_FAIL()

Test both Abort and Continue cases. Test more with successive marked
as expected failure. Test cases with a QSKIP after the marked check.
Unify data functions where practical.

Change-Id: I2eade5e4dd0907d23e37137ce3d93cd5ca79f802
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>

1 files changed, 114 insertions, 26 deletions

diff --git a/tests/auto/testlib/selftests/expected_expectfail.tap b/tests/auto/testlib/selftests/expected_expectfail.tap
index 2ad422b7f9..984815909a 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.tap
+++ b/tests/auto/testlib/selftests/expected_expectfail.tap
@@ -16,81 +16,169 @@ not ok 3 - xfailAndAbort() # TODO This should xfail
   file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
   line: 0
   ...
-not ok 4 - xfailTwice()
+not ok 4 - xfailContinueSkip() # TODO This should xfail then skip
+  ---
+  at: tst_ExpectFail::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+  line: 0
+  ...
+ok 4 - xfailContinueSkip() # SKIP This skip should be reported and counted
+not ok 5 - xfailAbortSkip() # TODO This should xfail
+  ---
+  at: tst_ExpectFail::xfailAbortSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+  line: 0
+  ...
+not ok 6 - xfailTwice()
   ---
   # Already expecting a fail
   at: tst_ExpectFail::xfailTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
   line: 0
   ...
-not ok 5 - xfailWithQString() # TODO A string
+not ok 7 - xfailDataDrivenTwice(Pass Abort)
+  ---
+  # Already expecting a fail
+  at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+  line: 0
+  ...
+not ok 8 - xfailDataDrivenTwice(Pass Continue)
+  ---
+  # Already expecting a fail
+  at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+  line: 0
+  ...
+not ok 9 - xfailDataDrivenTwice(Fail Abort)
+  ---
+  # Already expecting a fail
+  at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+  line: 0
+  ...
+not ok 10 - xfailDataDrivenTwice(Fail Continue)
+  ---
+  # Already expecting a fail
+  at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+  line: 0
+  ...
+not ok 11 - xfailWithQString() # TODO A string
   ---
   at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
   line: 0
   ...
-not ok 5 - xfailWithQString() # TODO Bug 5 (The message)
+not ok 11 - xfailWithQString() # TODO Bug 5 (The message)
   ---
   at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
   line: 0
   ...
-ok 6 - xfailDataDrivenWithQVerify(Pass 1)
-ok 7 - xfailDataDrivenWithQVerify(Pass 2)
-not ok 8 - xfailDataDrivenWithQVerify(Abort) # TODO This test should xfail
+ok 12 - xfailDataDrivenWithQString(Pass Abort) # SKIP Each Continue or Pass reports this and increments skip-count
+ok 13 - xfailDataDrivenWithQString(Pass Continue) # SKIP Each Continue or Pass reports this and increments skip-count
+not ok 14 - xfailDataDrivenWithQString(Fail Abort) # TODO A string
+  ---
+  at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+  line: 0
+  ...
+not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO A string
+  ---
+  at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+  line: 0
+  ...
+not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO Bug 5 (The message)
+  ---
+  at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+  line: 0
+  ...
+ok 15 - xfailDataDrivenWithQString(Fail Continue) # SKIP Each Continue or Pass reports this and increments skip-count
+ok 16 - xfailDataDrivenWithQVerify(Pass Abort)
+ok 17 - xfailDataDrivenWithQVerify(Pass Continue)
+not ok 18 - xfailDataDrivenWithQVerify(Fail Abort) # TODO This test should xfail
   ---
   at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
   line: 0
   ...
-not ok 9 - xfailDataDrivenWithQVerify(Continue) # TODO This test should xfail
+not ok 19 - xfailDataDrivenWithQVerify(Fail Continue) # TODO This test should xfail
   ---
   at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
   line: 0
   ...
-ok 10 - xfailDataDrivenWithQCompare(Pass 1)
-ok 11 - xfailDataDrivenWithQCompare(Pass 2)
-not ok 12 - xfailDataDrivenWithQCompare(Abort) # TODO This test should xfail
+ok 20 - xfailDataDrivenWithQCompare(Pass Abort)
+ok 21 - xfailDataDrivenWithQCompare(Pass Continue)
+not ok 22 - xfailDataDrivenWithQCompare(Fail Abort) # TODO This test should xfail
   ---
   at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
   line: 0
   ...
-not ok 13 - xfailDataDrivenWithQCompare(Continue) # TODO This test should xfail
+not ok 23 - xfailDataDrivenWithQCompare(Fail Continue) # TODO This test should xfail
   ---
   at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
   line: 0
   ...
-ok 14 - xfailOnWrongRow(right row)
-not ok 15 - xfailOnAnyRow(first row) # TODO This test should xfail
+# Should pass (*not* xpass), despite test-case name
+ok 24 - xfailOnWrongRow(Fail Abort)
+# Should pass (*not* xpass), despite test-case name
+ok 25 - xfailOnWrongRow(Fail Continue)
+not ok 26 - xfailOnAnyRow(Fail Abort) # TODO This test should xfail
   ---
   at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
   line: 0
   ...
-not ok 16 - xfailOnAnyRow(second row) # TODO This test should xfail
+not ok 27 - xfailOnAnyRow(Fail Continue) # TODO This test should xfail
   ---
   at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
   line: 0
   ...
-not ok 17 - xfailWithoutVerify(first row)
+# Should fail (*not* xfail), despite test-case name
+not ok 28 - xfailWithoutCheck(Fail Abort)
   ---
   # QEXPECT_FAIL was called without any subsequent verification statements
   ...
-not ok 18 - xfailWithoutVerify(second row)
+# Should fail (*not* xfail), despite test-case name
+not ok 29 - xfailWithoutCheck(Fail Continue)
   ---
   # QEXPECT_FAIL was called without any subsequent verification statements
   ...
-ok 19 - xpass() # TODO 'true' returned TRUE unexpectedly. ()
-ok 20 - xpassDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
-ok 21 - xpassDataDrivenWithQVerify(Pass)
-ok 22 - xpassDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
-ok 23 - xpassDataDrivenWithQCompare(Pass)
-ok 24 - cleanupTestCase()
-1..24
-# tests 24
-# pass 18
-# fail 6
+ok 30 - xpassAbort() # TODO 'true' returned TRUE unexpectedly. ()
+ok 31 - xpassAbortSkip() # TODO 'true' returned TRUE unexpectedly. ()
+ok 32 - xpassAbortXfailContinue() # TODO 'true' returned TRUE unexpectedly. ()
+ok 33 - xpassContinue() # TODO 'true' returned TRUE unexpectedly. ()
+# This should be reached
+ok 34 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. ()
+ok 35 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count
+ok 36 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. ()
+not ok 37 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals
+  ---
+  at: tst_ExpectFail::xpassContinueXfailAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+  line: 0
+  ...
+ok 37 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
+ok 38 - xpassAbortDataDrivenWithQVerify(Pass)
+ok 39 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
+# Test should Continue past XPASS
+# Test should simply PASS
+ok 40 - xpassContinueDataDrivenWithQVerify(Pass)
+ok 41 - xpassAbortDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
+ok 42 - xpassAbortDataDrivenWithQCompare(Pass)
+ok 43 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
+# Test should Continue past XPASS
+# Test should simply PASS
+ok 44 - xpassContinueDataDrivenWithQCompare(Pass)
+ok 45 - cleanupTestCase()
+1..45
+# tests 45
+# pass 23
+# fail 17