testlib: Skip test function if init() fails.

Prior to this commit, the following statement in the qtestlib
documentation was untrue: "If init() fails, the following testfunction
will not be executed, the test will proceed to the next testfunction."
If init() called QSKIP, the test function would be skipped, but if
init() reported a failure, the test function would still be executed
(even though doing so could be unsafe).

This commit makes testlib skip a test function if init() reports a
failure and enhances the selftests to cover skips and fails in both
init() and cleanup().

Task-number: QTBUG-20371
Change-Id: Id1cc8464ae0b8c257ae1b74dbe9189a501f5366b
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>

1 files changed, 68 insertions, 12 deletions

diff --git a/tests/auto/testlib/selftests/expected_counting.xml b/tests/auto/testlib/selftests/expected_counting.xml
index c5460a7346..a97296807d 100644
--- a/tests/auto/testlib/selftests/expected_counting.xml
+++ b/tests/auto/testlib/selftests/expected_counting.xml
@@ -19,7 +19,7 @@
 <Incident type="pass" file="" line="0">
     <DataTag><![CDATA[row 1]]></DataTag>
 </Incident>
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
@@ -28,13 +28,13 @@
 <Incident type="pass" file="" line="0">
     <DataTag><![CDATA[row 1]]></DataTag>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
 <TestFunction name="testSkipPass">
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
@@ -43,27 +43,27 @@
 </Incident>
 </TestFunction>
 <TestFunction name="testSkipSkip">
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
 </TestFunction>
 <TestFunction name="testSkipFail">
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
 <TestFunction name="testFailPass">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
@@ -72,25 +72,81 @@
 </Incident>
 </TestFunction>
 <TestFunction name="testFailSkip">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
-<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="102">
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="117">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA[Skipping]]></Description>
 </Message>
 </TestFunction>
 <TestFunction name="testFailFail">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 1]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="99">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="114">
     <DataTag><![CDATA[row 2]]></DataTag>
     <Description><![CDATA['false' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
+<TestFunction name="testFailInInit">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="233">
+    <DataTag><![CDATA[fail]]></DataTag>
+    <Description><![CDATA[Fail in init()]]></Description>
+</Incident>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
+<TestFunction name="testFailInCleanup">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Message type="qdebug" file="" line="0">
+    <DataTag><![CDATA[fail]]></DataTag>
+    <Description><![CDATA[This test function should execute and then QFAIL in cleanup() ]]></Description>
+</Message>
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="241">
+    <DataTag><![CDATA[fail]]></DataTag>
+    <Description><![CDATA[Fail in cleanup()]]></Description>
+</Incident>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
+<TestFunction name="testSkipInInit">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="235">
+    <DataTag><![CDATA[skip]]></DataTag>
+    <Description><![CDATA[Skip in init()]]></Description>
+</Message>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
+<TestFunction name="testSkipInCleanup">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[before]]></DataTag>
+</Incident>
+<Message type="qdebug" file="" line="0">
+    <DataTag><![CDATA[skip]]></DataTag>
+    <Description><![CDATA[This test function should execute and then QSKIP in cleanup() ]]></Description>
+</Message>
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp" line="243">
+    <DataTag><![CDATA[skip]]></DataTag>
+    <Description><![CDATA[Skip in cleanup()]]></Description>
+</Message>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[after]]></DataTag>
+</Incident>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Incident type="pass" file="" line="0" />
 </TestFunction>