summaryrefslogtreecommitdiffstats
path: root/tests/auto/testlib/selftests/expected_expectfail.tap
diff options
context:
space:
mode:
Diffstat (limited to 'tests/auto/testlib/selftests/expected_expectfail.tap')
-rw-r--r--tests/auto/testlib/selftests/expected_expectfail.tap288
1 files changed, 227 insertions, 61 deletions
diff --git a/tests/auto/testlib/selftests/expected_expectfail.tap b/tests/auto/testlib/selftests/expected_expectfail.tap
index bc449a7452..8ad34c5f0a 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.tap
+++ b/tests/auto/testlib/selftests/expected_expectfail.tap
@@ -1,102 +1,268 @@
TAP version 13
# tst_ExpectFail
ok 1 - initTestCase()
-# begin
not ok 2 - xfailAndContinue() # TODO This should xfail
---
- at: tst_ExpectFail::xfailAndContinue() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:65)
- file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
- line: 65
+ extensions:
+ messages:
+ - severity: debug
+ message: begin
+ - severity: xfail
+ message: This should xfail
+ at: tst_ExpectFail::xfailAndContinue() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
+ - severity: debug
+ message: after
...
-# after
-# begin
not ok 3 - xfailAndAbort() # TODO This should xfail
---
- at: tst_ExpectFail::xfailAndAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:73)
+ extensions:
+ messages:
+ - severity: debug
+ message: begin
+ - severity: xfail
+ message: This should xfail
+ at: tst_ExpectFail::xfailAndAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
+ ...
+not ok 4 - xfailContinueSkip() # SKIP This skip should be reported and counted
+ ---
+ at: tst_ExpectFail::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
- line: 73
+ line: 0
+ extensions:
+ messages:
+ - severity: xfail
+ message: This should xfail then skip
+ at: tst_ExpectFail::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
+ ...
+not ok 5 - xfailAbortSkip() # TODO This should xfail
+ ---
+ extensions:
+ messages:
+ - severity: xfail
+ message: This should xfail
+ at: tst_ExpectFail::xfailAbortSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
...
-not ok 4 - xfailTwice()
+not ok 6 - xfailTwice()
---
# Already expecting a fail
- at: tst_ExpectFail::xfailTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:83)
+ at: tst_ExpectFail::xfailTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
- line: 83
+ line: 0
...
-not ok 5 - xfailWithQString() # TODO A string
+not ok 7 - xfailDataDrivenTwice(Pass Abort)
---
- at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:92)
+ # Already expecting a fail
+ at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
- line: 92
+ line: 0
...
-not ok 5 - xfailWithQString() # TODO Bug 5 (The message)
+not ok 8 - xfailDataDrivenTwice(Pass Continue)
---
- at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:97)
+ # Already expecting a fail
+ at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
- line: 97
+ line: 0
...
-ok 6 - xfailDataDrivenWithQVerify(Pass 1)
-ok 7 - xfailDataDrivenWithQVerify(Pass 2)
-not ok 8 - xfailDataDrivenWithQVerify(Abort) # TODO This test should xfail
+not ok 9 - xfailDataDrivenTwice(Fail Abort)
---
- at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:126)
+ # Already expecting a fail
+ at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
- line: 126
+ line: 0
...
-not ok 9 - xfailDataDrivenWithQVerify(Continue) # TODO This test should xfail
+not ok 10 - xfailDataDrivenTwice(Fail Continue)
---
- at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:126)
+ # Already expecting a fail
+ at: tst_ExpectFail::xfailDataDrivenTwice() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
- line: 126
+ line: 0
...
-ok 10 - xfailDataDrivenWithQCompare(Pass 1)
-ok 11 - xfailDataDrivenWithQCompare(Pass 2)
-not ok 12 - xfailDataDrivenWithQCompare(Abort) # TODO This test should xfail
+not ok 11 - xfailWithQString() # TODO A string
---
- at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:160)
- file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
- line: 160
+ extensions:
+ messages:
+ - severity: xfail
+ message: A string
+ at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
+ - severity: xfail
+ message: Bug 5 (The message)
+ at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
...
-not ok 13 - xfailDataDrivenWithQCompare(Continue) # TODO This test should xfail
+ok 12 - xfailDataDrivenWithQString(Pass Abort) # SKIP Each Continue or Pass reports this and increments skip-count
+ok 13 - xfailDataDrivenWithQString(Pass Continue) # SKIP Each Continue or Pass reports this and increments skip-count
+not ok 14 - xfailDataDrivenWithQString(Fail Abort) # TODO A string
---
- at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:160)
- file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
- line: 160
+ extensions:
+ messages:
+ - severity: xfail
+ message: A string
+ at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
...
-ok 14 - xfailOnWrongRow(right row)
-not ok 15 - xfailOnAnyRow(first row) # TODO This test should xfail
+not ok 15 - xfailDataDrivenWithQString(Fail Continue) # SKIP Each Continue or Pass reports this and increments skip-count
---
- at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:195)
+ at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
- line: 195
+ line: 0
+ extensions:
+ messages:
+ - severity: xfail
+ message: A string
+ at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
+ - severity: xfail
+ message: Bug 5 (The message)
+ at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
...
-not ok 16 - xfailOnAnyRow(second row) # TODO This test should xfail
+ok 16 - xfailDataDrivenWithQVerify(Pass Abort)
+ok 17 - xfailDataDrivenWithQVerify(Pass Continue)
+not ok 18 - xfailDataDrivenWithQVerify(Fail Abort) # TODO This test should xfail
---
- at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:195)
- file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
- line: 195
+ extensions:
+ messages:
+ - severity: xfail
+ message: This test should xfail
+ at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
+ ...
+not ok 19 - xfailDataDrivenWithQVerify(Fail Continue) # TODO This test should xfail
+ ---
+ extensions:
+ messages:
+ - severity: xfail
+ message: This test should xfail
+ at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
...
-not ok 17 - xfailWithoutVerify(first row)
+ok 20 - xfailDataDrivenWithQCompare(Pass Abort)
+ok 21 - xfailDataDrivenWithQCompare(Pass Continue)
+not ok 22 - xfailDataDrivenWithQCompare(Fail Abort) # TODO This test should xfail
+ ---
+ extensions:
+ messages:
+ - severity: xfail
+ message: This test should xfail
+ at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
+ ...
+not ok 23 - xfailDataDrivenWithQCompare(Fail Continue) # TODO This test should xfail
+ ---
+ extensions:
+ messages:
+ - severity: xfail
+ message: This test should xfail
+ at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
+ ...
+ok 24 - xfailOnWrongRow(Fail Abort)
+ ---
+ extensions:
+ messages:
+ - severity: debug
+ message: Should pass (*not* xpass), despite test-case name
+ ...
+ok 25 - xfailOnWrongRow(Fail Continue)
+ ---
+ extensions:
+ messages:
+ - severity: debug
+ message: Should pass (*not* xpass), despite test-case name
+ ...
+not ok 26 - xfailOnAnyRow(Fail Abort) # TODO This test should xfail
+ ---
+ extensions:
+ messages:
+ - severity: xfail
+ message: This test should xfail
+ at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
+ ...
+not ok 27 - xfailOnAnyRow(Fail Continue) # TODO This test should xfail
+ ---
+ extensions:
+ messages:
+ - severity: xfail
+ message: This test should xfail
+ at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
+ file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+ line: 0
+ ...
+not ok 28 - xfailWithoutCheck(Fail Abort)
---
# QEXPECT_FAIL was called without any subsequent verification statements
- at: tst_ExpectFail::xfailWithoutVerify() (Unknown File:0)
- file: Unknown File
- line: 0
+ extensions:
+ messages:
+ - severity: debug
+ message: Should fail (*not* xfail), despite test-case name
...
-not ok 18 - xfailWithoutVerify(second row)
+not ok 29 - xfailWithoutCheck(Fail Continue)
---
# QEXPECT_FAIL was called without any subsequent verification statements
- at: tst_ExpectFail::xfailWithoutVerify() (Unknown File:0)
- file: Unknown File
- line: 0
+ extensions:
+ messages:
+ - severity: debug
+ message: Should fail (*not* xfail), despite test-case name
+ ...
+ok 30 - xpassAbort() # TODO 'true' returned TRUE unexpectedly. ()
+ok 31 - xpassAbortSkip() # TODO 'true' returned TRUE unexpectedly. ()
+ok 32 - xpassAbortXfailContinue() # TODO 'true' returned TRUE unexpectedly. ()
+ok 33 - xpassContinue() # TODO 'true' returned TRUE unexpectedly. ()
+# debug: This should be reached
+ok 34 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. ()
+ok 34 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count
+ok 35 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. ()
+ # xfail: This test should xfail but not add to totals
+ok 36 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
+ok 37 - xpassAbortDataDrivenWithQVerify(Pass)
+ok 38 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
+# debug: Test should Continue past XPASS
+ok 39 - xpassContinueDataDrivenWithQVerify(Pass)
+ ---
+ extensions:
+ messages:
+ - severity: debug
+ message: Test should simply PASS
+ ...
+ok 40 - xpassAbortDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
+ok 41 - xpassAbortDataDrivenWithQCompare(Pass)
+ok 42 - xpassContinueDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
+# debug: Test should Continue past XPASS
+ok 43 - xpassContinueDataDrivenWithQCompare(Pass)
+ ---
+ extensions:
+ messages:
+ - severity: debug
+ message: Test should simply PASS
+ ...
+ok 44 - cleanupTestCase()
+ ---
+ extensions:
+ messages:
+ - severity: debug
+ message: Totals should add up to 44: 23 passed, 17 failed, 4 skipped
...
-ok 19 - xpass() # TODO 'true' returned TRUE unexpectedly. ()
-ok 20 - xpassDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
-ok 21 - xpassDataDrivenWithQVerify(Pass)
-ok 22 - xpassDataDrivenWithQCompare(XPass) # TODO QCOMPARE(1, 1) returned TRUE unexpectedly.
-ok 23 - xpassDataDrivenWithQCompare(Pass)
-ok 24 - cleanupTestCase()
-1..24
-# tests 24
-# pass 18
-# fail 6
+1..44
+# tests 44
+# pass 23
+# fail 17