testlib: Skip test function if init() fails.

Prior to this commit, the following statement in the qtestlib
documentation was untrue: "If init() fails, the following testfunction
will not be executed, the test will proceed to the next testfunction."
If init() called QSKIP, the test function would be skipped, but if
init() reported a failure, the test function would still be executed
(even though doing so could be unsafe).

This commit makes testlib skip a test function if init() reports a
failure and enhances the selftests to cover skips and fails in both
init() and cleanup().

Task-number: QTBUG-20371
Change-Id: Id1cc8464ae0b8c257ae1b74dbe9189a501f5366b
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>

1 files changed, 87 insertions, 0 deletions

diff --git a/tests/auto/testlib/selftests/counting/tst_counting.cpp b/tests/auto/testlib/selftests/counting/tst_counting.cpp
index 6758b533bb..fa61fce173 100644
--- a/tests/auto/testlib/selftests/counting/tst_counting.cpp
+++ b/tests/auto/testlib/selftests/counting/tst_counting.cpp
@@ -47,6 +47,8 @@ class tst_Counting : public QObject
     Q_OBJECT
 
 private slots:
+    // The following test functions exercise each possible combination of test
+    // results for two data rows.
     void testPassPass_data();
     void testPassPass();
 
@@ -74,6 +76,19 @@ private slots:
     void testFailFail_data();
     void testFailFail();
 
+    // The following test functions test skips and fails in the special
+    // init() and cleanup() slots.
+    void init();
+    void cleanup();
+    void testFailInInit_data();
+    void testFailInInit();
+    void testFailInCleanup_data();
+    void testFailInCleanup();
+    void testSkipInInit_data();
+    void testSkipInInit();
+    void testSkipInCleanup_data();
+    void testSkipInCleanup();
+
 private:
     void helper();
 };
@@ -212,5 +227,77 @@ void tst_Counting::testFailFail()
     helper();
 }
 
+void tst_Counting::init()
+{
+    if (strcmp(QTest::currentTestFunction(), "testFailInInit") == 0 && strcmp(QTest::currentDataTag(), "fail") == 0)
+        QFAIL("Fail in init()");
+    else if (strcmp(QTest::currentTestFunction(), "testSkipInInit") == 0 && strcmp(QTest::currentDataTag(), "skip") == 0)
+        QSKIP("Skip in init()");
+}
+
+void tst_Counting::cleanup()
+{
+    if (strcmp(QTest::currentTestFunction(), "testFailInCleanup") == 0 && strcmp(QTest::currentDataTag(), "fail") == 0)
+        QFAIL("Fail in cleanup()");
+    else if (strcmp(QTest::currentTestFunction(), "testSkipInCleanup") == 0 && strcmp(QTest::currentDataTag(), "skip") == 0)
+        QSKIP("Skip in cleanup()");
+}
+
+void tst_Counting::testFailInInit_data()
+{
+    QTest::addColumn<bool>("dummy");
+    QTest::newRow("before") << true;
+    QTest::newRow("fail") << true;
+    QTest::newRow("after") << true;
+}
+
+void tst_Counting::testFailInInit()
+{
+    if (strcmp(QTest::currentDataTag(), "fail") == 0)
+        QFAIL("This test function should have been skipped due to QFAIL in init()");
+}
+
+void tst_Counting::testFailInCleanup_data()
+{
+    QTest::addColumn<bool>("dummy");
+    QTest::newRow("before") << true;
+    QTest::newRow("fail") << true;
+    QTest::newRow("after") << true;
+}
+
+void tst_Counting::testFailInCleanup()
+{
+    if (strcmp(QTest::currentDataTag(), "fail") == 0)
+        qDebug() << "This test function should execute and then QFAIL in cleanup()";
+}
+
+void tst_Counting::testSkipInInit_data()
+{
+    QTest::addColumn<bool>("dummy");
+    QTest::newRow("before") << true;
+    QTest::newRow("skip") << true;
+    QTest::newRow("after") << true;
+}
+
+void tst_Counting::testSkipInInit()
+{
+    if (strcmp(QTest::currentDataTag(), "skip") == 0)
+        QFAIL("This test function should have been skipped due to QSKIP in init()");
+}
+
+void tst_Counting::testSkipInCleanup_data()
+{
+    QTest::addColumn<bool>("dummy");
+    QTest::newRow("before") << true;
+    QTest::newRow("skip") << true;
+    QTest::newRow("after") << true;
+}
+
+void tst_Counting::testSkipInCleanup()
+{
+    if (strcmp(QTest::currentDataTag(), "skip") == 0)
+        qDebug() << "This test function should execute and then QSKIP in cleanup()";
+}
+
 QTEST_MAIN(tst_Counting)
 #include "tst_counting.moc"