testlib: Skip test function if init() fails.

Prior to this commit, the following statement in the qtestlib
documentation was untrue: "If init() fails, the following testfunction
will not be executed, the test will proceed to the next testfunction."
If init() called QSKIP, the test function would be skipped, but if
init() reported a failure, the test function would still be executed
(even though doing so could be unsafe).

This commit makes testlib skip a test function if init() reports a
failure and enhances the selftests to cover skips and fails in both
init() and cleanup().

Task-number: QTBUG-20371
Change-Id: Id1cc8464ae0b8c257ae1b74dbe9189a501f5366b
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>

5 files changed, 273 insertions, 38 deletions

diff --git a/tests/auto/testlib/selftests/counting/tst_counting.cpp b/tests/auto/testlib/selftests/counting/tst_counting.cpp
index 6758b533bb..fa61fce173 100644
--- a/tests/auto/testlib/selftests/counting/tst_counting.cpp
+++ b/tests/auto/testlib/selftests/counting/tst_counting.cpp
@@ -47,6 +47,8 @@ class tst_Counting : public QObject
     Q_OBJECT
 
 private slots:
+    // The following test functions exercise each possible combination of test
+    // results for two data rows.
     void testPassPass_data();
     void testPassPass();
 
@@ -74,6 +76,19 @@ private slots:
     void testFailFail_data();
     void testFailFail();
 
+    // The following test functions test skips and fails in the special
+    // init() and cleanup() slots.
+    void init();
+    void cleanup();
+    void testFailInInit_data();
+    void testFailInInit();
+    void testFailInCleanup_data();
+    void testFailInCleanup();
+    void testSkipInInit_data();
+    void testSkipInInit();
+    void testSkipInCleanup_data();
+    void testSkipInCleanup();
+
 private:
     void helper();
 };
@@ -212,5 +227,77 @@ void tst_Counting::testFailFail()
     helper();
 }
 
+void tst_Counting::init()
+{
+    if (strcmp(QTest::currentTestFunction(), "testFailInInit") == 0 && strcmp(QTest::currentDataTag(), "fail") == 0)
+        QFAIL("Fail in init()");
+    else if (strcmp(QTest::currentTestFunction(), "testSkipInInit") == 0 && strcmp(QTest::currentDataTag(), "skip") == 0)
+        QSKIP("Skip in init()");
+}
+
+void tst_Counting::cleanup()
+{
+    if (strcmp(QTest::currentTestFunction(), "testFailInCleanup") == 0 && strcmp(QTest::currentDataTag(), "fail") == 0)
+        QFAIL("Fail in cleanup()");
+    else if (strcmp(QTest::currentTestFunction(), "testSkipInCleanup") == 0 && strcmp(QTest::currentDataTag(), "skip") == 0)
+        QSKIP("Skip in cleanup()");
+}
+
+void tst_Counting::testFailInInit_data()
+{
+    QTest::addColumn<bool>("dummy");
+    QTest::newRow("before") << true;
+    QTest::newRow("fail") << true;
+    QTest::newRow("after") << true;
+}
+
+void tst_Counting::testFailInInit()
+{
+    if (strcmp(QTest::currentDataTag(), "fail") == 0)
+        QFAIL("This test function should have been skipped due to QFAIL in init()");
+}
+
+void tst_Counting::testFailInCleanup_data()
+{
+    QTest::addColumn<bool>("dummy");
+    QTest::newRow("before") << true;
+    QTest::newRow("fail") << true;
+    QTest::newRow("after") << true;
+}
+
+void tst_Counting::testFailInCleanup()
+{
+    if (strcmp(QTest::currentDataTag(), "fail") == 0)
+        qDebug() << "This test function should execute and then QFAIL in cleanup()";
+}
+
+void tst_Counting::testSkipInInit_data()
+{
+    QTest::addColumn<bool>("dummy");
+    QTest::newRow("before") << true;
+    QTest::newRow("skip") << true;
+    QTest::newRow("after") << true;
+}
+
+void tst_Counting::testSkipInInit()
+{
+    if (strcmp(QTest::currentDataTag(), "skip") == 0)
+        QFAIL("This test function should have been skipped due to QSKIP in init()");
+}
+
+void tst_Counting::testSkipInCleanup_data()
+{
+    QTest::addColumn<bool>("dummy");
+    QTest::newRow("before") << true;
+    QTest::newRow("skip") << true;
+    QTest::newRow("after") << true;
+}
+
+void tst_Counting::testSkipInCleanup()
+{
+    if (strcmp(QTest::currentDataTag(), "skip") == 0)
+        qDebug() << "This test function should execute and then QSKIP in cleanup()";
+}
+
 QTEST_MAIN(tst_Counting)
 #include "tst_counting.moc"
diff --git a/tests/auto/testlib/selftests/expected_counting.lightxml b/tests/auto/testlib/selftests/expected_counting.lightxml
index fd80292b90..e7b1136417 100644
--- a/tests/auto/testlib/selftests/expected_counting.lightxml
+++ b/tests/auto/testlib/selftests/expected_counting.lightxml
@@ -17,7 +17,7 @@
 <Incident type="pass" file="" line="0">
     <DataTag><![CDATA[row 1]]></DataTag>
 </Incident>
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
@@ -26,13 +26,13 @@
 <Incident type="pass" file="" line="0">
     <DataTag><![CDATA[row 1]]></DataTag>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
 <TestFunction name="testSkipPass">
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
@@ -41,27 +41,27 @@
 </Incident>
 </TestFunction>
 <TestFunction name="testSkipSkip">
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
 </TestFunction>
 <TestFunction name="testSkipFail">
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
 <TestFunction name="testFailPass">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
@@ -70,25 +70,81 @@
 </Incident>
 </TestFunction>
 <TestFunction name="testFailSkip">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
 </TestFunction>
 <TestFunction name="testFailFail">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
+<TestFunction name="testFailInInit">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="233">
+    <DataTag><![CDATA[fail]]></DataTag>
+    <Description><![CDATA[Fail in init()]]></Description>
+</Incident>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
+<TestFunction name="testFailInCleanup">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Message type="qdebug" file="" line="0">
+    <DataTag><![CDATA[fail]]></DataTag>
+    <Description><![CDATA[This test function should execute and then QFAIL in cleanup() ]]></Description>
+</Message>
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="241">
+    <DataTag><![CDATA[fail]]></DataTag>
+    <Description><![CDATA[Fail in cleanup()]]></Description>
+</Incident>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
+<TestFunction name="testSkipInInit">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="235">
+    <DataTag><![CDATA[skip]]></DataTag>
+    <Description><![CDATA[Skip in init()]]></Description>
+</Message>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
+<TestFunction name="testSkipInCleanup">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Message type="qdebug" file="" line="0">
+    <DataTag><![CDATA[skip]]></DataTag>
+    <Description><![CDATA[This test function should execute and then QSKIP in cleanup() ]]></Description>
+</Message>
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="243">
+    <DataTag><![CDATA[skip]]></DataTag>
+    <Description><![CDATA[Skip in cleanup()]]></Description>
+</Message>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Incident type="pass" file="" line="0" />
 </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_counting.txt b/tests/auto/testlib/selftests/expected_counting.txt
index bd70a44b2e..5c17e3c257 100644
--- a/tests/auto/testlib/selftests/expected_counting.txt
+++ b/tests/auto/testlib/selftests/expected_counting.txt
@@ -5,32 +5,50 @@ PASS   : tst_Counting::testPassPass(row 1)
 PASS   : tst_Counting::testPassPass(row 2)
 PASS   : tst_Counting::testPassSkip(row 1)
 SKIP   : tst_Counting::testPassSkip(row 2) Skipping
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)]
 PASS   : tst_Counting::testPassFail(row 1)
 FAIL!  : tst_Counting::testPassFail(row 2) 'false' returned FALSE. ()
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)]
 SKIP   : tst_Counting::testSkipPass(row 1) Skipping
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)]
 PASS   : tst_Counting::testSkipPass(row 2)
 SKIP   : tst_Counting::testSkipSkip(row 1) Skipping
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)]
 SKIP   : tst_Counting::testSkipSkip(row 2) Skipping
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)]
 SKIP   : tst_Counting::testSkipFail(row 1) Skipping
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)]
 FAIL!  : tst_Counting::testSkipFail(row 2) 'false' returned FALSE. ()
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)]
 FAIL!  : tst_Counting::testFailPass(row 1) 'false' returned FALSE. ()
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)]
 PASS   : tst_Counting::testFailPass(row 2)
 FAIL!  : tst_Counting::testFailSkip(row 1) 'false' returned FALSE. ()
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)]
 SKIP   : tst_Counting::testFailSkip(row 2) Skipping
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)]
 FAIL!  : tst_Counting::testFailFail(row 1) 'false' returned FALSE. ()
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)]
 FAIL!  : tst_Counting::testFailFail(row 2) 'false' returned FALSE. ()
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)]
+PASS   : tst_Counting::testFailInInit(before)
+FAIL!  : tst_Counting::testFailInInit(fail) Fail in init()
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(233)]
+PASS   : tst_Counting::testFailInInit(after)
+PASS   : tst_Counting::testFailInCleanup(before)
+QDEBUG : tst_Counting::testFailInCleanup(fail) This test function should execute and then QFAIL in cleanup() 
+FAIL!  : tst_Counting::testFailInCleanup(fail) Fail in cleanup()
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(241)]
+PASS   : tst_Counting::testFailInCleanup(after)
+PASS   : tst_Counting::testSkipInInit(before)
+SKIP   : tst_Counting::testSkipInInit(skip) Skip in init()
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(235)]
+PASS   : tst_Counting::testSkipInInit(after)
+PASS   : tst_Counting::testSkipInCleanup(before)
+QDEBUG : tst_Counting::testSkipInCleanup(skip) This test function should execute and then QSKIP in cleanup() 
+SKIP   : tst_Counting::testSkipInCleanup(skip) Skip in cleanup()
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(243)]
+PASS   : tst_Counting::testSkipInCleanup(after)
 PASS   : tst_Counting::cleanupTestCase()
-Totals: 8 passed, 6 failed, 6 skipped
+Totals: 16 passed, 8 failed, 8 skipped
 ********* Finished testing of tst_Counting *********
diff --git a/tests/auto/testlib/selftests/expected_counting.xml b/tests/auto/testlib/selftests/expected_counting.xml
index c5460a7346..a97296807d 100644
--- a/tests/auto/testlib/selftests/expected_counting.xml
+++ b/tests/auto/testlib/selftests/expected_counting.xml
@@ -19,7 +19,7 @@
 <Incident type="pass" file="" line="0">
     <DataTag><![CDATA[row 1]]></DataTag>
 </Incident>
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
@@ -28,13 +28,13 @@
 <Incident type="pass" file="" line="0">
     <DataTag><![CDATA[row 1]]></DataTag>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
 <TestFunction name="testSkipPass">
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
@@ -43,27 +43,27 @@
 </Incident>
 </TestFunction>
 <TestFunction name="testSkipSkip">
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
 </TestFunction>
 <TestFunction name="testSkipFail">
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
 <TestFunction name="testFailPass">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
@@ -72,25 +72,81 @@
 </Incident>
 </TestFunction>
 <TestFunction name="testFailSkip">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
 </TestFunction>
 <TestFunction name="testFailFail">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
+<TestFunction name="testFailInInit">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="233">
+    <DataTag><![CDATA[fail]]></DataTag>
+    <Description><![CDATA[Fail in init()]]></Description>
+</Incident>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
+<TestFunction name="testFailInCleanup">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Message type="qdebug" file="" line="0">
+    <DataTag><![CDATA[fail]]></DataTag>
+    <Description><![CDATA[This test function should execute and then QFAIL in cleanup() ]]></Description>
+</Message>
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="241">
+    <DataTag><![CDATA[fail]]></DataTag>
+    <Description><![CDATA[Fail in cleanup()]]></Description>
+</Incident>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
+<TestFunction name="testSkipInInit">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="235">
+    <DataTag><![CDATA[skip]]></DataTag>
+    <Description><![CDATA[Skip in init()]]></Description>
+</Message>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
+<TestFunction name="testSkipInCleanup">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Message type="qdebug" file="" line="0">
+    <DataTag><![CDATA[skip]]></DataTag>
+    <Description><![CDATA[This test function should execute and then QSKIP in cleanup() ]]></Description>
+</Message>
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="243">
+    <DataTag><![CDATA[skip]]></DataTag>
+    <Description><![CDATA[Skip in cleanup()]]></Description>
+</Message>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Incident type="pass" file="" line="0" />
 </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_counting.xunitxml b/tests/auto/testlib/selftests/expected_counting.xunitxml
index 496546700e..f317ed5923 100644
--- a/tests/auto/testlib/selftests/expected_counting.xunitxml
+++ b/tests/auto/testlib/selftests/expected_counting.xunitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite errors="6" failures="6" tests="11" name="tst_Counting">
+<testsuite errors="10" failures="8" tests="15" name="tst_Counting">
   <properties>
     <property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
     <property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
@@ -34,6 +34,20 @@
     <failure tag="row 1" message="&apos;false&apos; returned FALSE. ()" result="fail"/>
     <failure tag="row 2" message="&apos;false&apos; returned FALSE. ()" result="fail"/>
   </testcase>
+  <testcase result="fail" name="testFailInInit">
+    <failure tag="fail" message="Fail in init()" result="fail"/>
+  </testcase>
+  <testcase result="fail" name="testFailInCleanup">
+    <!-- tag="fail" message="This test function should execute and then QFAIL in cleanup() " type="qdebug" -->
+    <failure tag="fail" message="Fail in cleanup()" result="fail"/>
+  </testcase>
+  <testcase result="pass" name="testSkipInInit">
+    <!-- tag="skip" message="Skip in init()" type="skip" -->
+  </testcase>
+  <testcase result="pass" name="testSkipInCleanup">
+    <!-- tag="skip" message="This test function should execute and then QSKIP in cleanup() " type="qdebug" -->
+    <!-- tag="skip" message="Skip in cleanup()" type="skip" -->
+  </testcase>
   <testcase result="pass" name="cleanupTestCase"/>
   <system-err>
 <![CDATA[Skipping]]>
@@ -42,5 +56,9 @@
 <![CDATA[Skipping]]>
 <![CDATA[Skipping]]>
 <![CDATA[Skipping]]>
+<![CDATA[This test function should execute and then QFAIL in cleanup() ]]>
+<![CDATA[Skip in init()]]>
+<![CDATA[This test function should execute and then QSKIP in cleanup() ]]>
+<![CDATA[Skip in cleanup()]]>
   </system-err>
 </testsuite>