From 64642a4d9768b767c51c7f92dec94f655ba60195 Mon Sep 17 00:00:00 2001 From: Jason McDonald Date: Thu, 16 Feb 2012 16:00:41 +1000 Subject: testlib: Skip test function if init() fails. Prior to this commit, the following statement in the qtestlib documentation was untrue: "If init() fails, the following testfunction will not be executed, the test will proceed to the next testfunction." If init() called QSKIP, the test function would be skipped, but if init() reported a failure, the test function would still be executed (even though doing so could be unsafe). This commit makes testlib skip a test function if init() reports a failure and enhances the selftests to cover skips and fails in both init() and cleanup(). Task-number: QTBUG-20371 Change-Id: Id1cc8464ae0b8c257ae1b74dbe9189a501f5366b Reviewed-by: Rohan McGovern --- .../testlib/selftests/counting/tst_counting.cpp | 87 ++++++++++++++++++++++ .../testlib/selftests/expected_counting.lightxml | 80 +++++++++++++++++--- tests/auto/testlib/selftests/expected_counting.txt | 44 +++++++---- tests/auto/testlib/selftests/expected_counting.xml | 80 +++++++++++++++++--- .../testlib/selftests/expected_counting.xunitxml | 20 ++++- 5 files changed, 273 insertions(+), 38 deletions(-) (limited to 'tests/auto/testlib') diff --git a/tests/auto/testlib/selftests/counting/tst_counting.cpp b/tests/auto/testlib/selftests/counting/tst_counting.cpp index 6758b533bb..fa61fce173 100644 --- a/tests/auto/testlib/selftests/counting/tst_counting.cpp +++ b/tests/auto/testlib/selftests/counting/tst_counting.cpp @@ -47,6 +47,8 @@ class tst_Counting : public QObject Q_OBJECT private slots: + // The following test functions exercise each possible combination of test + // results for two data rows. void testPassPass_data(); void testPassPass(); @@ -74,6 +76,19 @@ private slots: void testFailFail_data(); void testFailFail(); + // The following test functions test skips and fails in the special + // init() and cleanup() slots. + void init(); + void cleanup(); + void testFailInInit_data(); + void testFailInInit(); + void testFailInCleanup_data(); + void testFailInCleanup(); + void testSkipInInit_data(); + void testSkipInInit(); + void testSkipInCleanup_data(); + void testSkipInCleanup(); + private: void helper(); }; @@ -212,5 +227,77 @@ void tst_Counting::testFailFail() helper(); } +void tst_Counting::init() +{ + if (strcmp(QTest::currentTestFunction(), "testFailInInit") == 0 && strcmp(QTest::currentDataTag(), "fail") == 0) + QFAIL("Fail in init()"); + else if (strcmp(QTest::currentTestFunction(), "testSkipInInit") == 0 && strcmp(QTest::currentDataTag(), "skip") == 0) + QSKIP("Skip in init()"); +} + +void tst_Counting::cleanup() +{ + if (strcmp(QTest::currentTestFunction(), "testFailInCleanup") == 0 && strcmp(QTest::currentDataTag(), "fail") == 0) + QFAIL("Fail in cleanup()"); + else if (strcmp(QTest::currentTestFunction(), "testSkipInCleanup") == 0 && strcmp(QTest::currentDataTag(), "skip") == 0) + QSKIP("Skip in cleanup()"); +} + +void tst_Counting::testFailInInit_data() +{ + QTest::addColumn("dummy"); + QTest::newRow("before") << true; + QTest::newRow("fail") << true; + QTest::newRow("after") << true; +} + +void tst_Counting::testFailInInit() +{ + if (strcmp(QTest::currentDataTag(), "fail") == 0) + QFAIL("This test function should have been skipped due to QFAIL in init()"); +} + +void tst_Counting::testFailInCleanup_data() +{ + QTest::addColumn("dummy"); + QTest::newRow("before") << true; + QTest::newRow("fail") << true; + QTest::newRow("after") << true; +} + +void tst_Counting::testFailInCleanup() +{ + if (strcmp(QTest::currentDataTag(), "fail") == 0) + qDebug() << "This test function should execute and then QFAIL in cleanup()"; +} + +void tst_Counting::testSkipInInit_data() +{ + QTest::addColumn("dummy"); + QTest::newRow("before") << true; + QTest::newRow("skip") << true; + QTest::newRow("after") << true; +} + +void tst_Counting::testSkipInInit() +{ + if (strcmp(QTest::currentDataTag(), "skip") == 0) + QFAIL("This test function should have been skipped due to QSKIP in init()"); +} + +void tst_Counting::testSkipInCleanup_data() +{ + QTest::addColumn("dummy"); + QTest::newRow("before") << true; + QTest::newRow("skip") << true; + QTest::newRow("after") << true; +} + +void tst_Counting::testSkipInCleanup() +{ + if (strcmp(QTest::currentDataTag(), "skip") == 0) + qDebug() << "This test function should execute and then QSKIP in cleanup()"; +} + QTEST_MAIN(tst_Counting) #include "tst_counting.moc" diff --git a/tests/auto/testlib/selftests/expected_counting.lightxml b/tests/auto/testlib/selftests/expected_counting.lightxml index fd80292b90..e7b1136417 100644 --- a/tests/auto/testlib/selftests/expected_counting.lightxml +++ b/tests/auto/testlib/selftests/expected_counting.lightxml @@ -17,7 +17,7 @@ - + @@ -26,13 +26,13 @@ - + - + @@ -41,27 +41,27 @@ - + - + - + - + - + @@ -70,25 +70,81 @@ - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_counting.txt b/tests/auto/testlib/selftests/expected_counting.txt index bd70a44b2e..5c17e3c257 100644 --- a/tests/auto/testlib/selftests/expected_counting.txt +++ b/tests/auto/testlib/selftests/expected_counting.txt @@ -5,32 +5,50 @@ PASS : tst_Counting::testPassPass(row 1) PASS : tst_Counting::testPassPass(row 2) PASS : tst_Counting::testPassSkip(row 1) SKIP : tst_Counting::testPassSkip(row 2) Skipping - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] PASS : tst_Counting::testPassFail(row 1) FAIL! : tst_Counting::testPassFail(row 2) 'false' returned FALSE. () - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] SKIP : tst_Counting::testSkipPass(row 1) Skipping - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] PASS : tst_Counting::testSkipPass(row 2) SKIP : tst_Counting::testSkipSkip(row 1) Skipping - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] SKIP : tst_Counting::testSkipSkip(row 2) Skipping - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] SKIP : tst_Counting::testSkipFail(row 1) Skipping - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] FAIL! : tst_Counting::testSkipFail(row 2) 'false' returned FALSE. () - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] FAIL! : tst_Counting::testFailPass(row 1) 'false' returned FALSE. () - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] PASS : tst_Counting::testFailPass(row 2) FAIL! : tst_Counting::testFailSkip(row 1) 'false' returned FALSE. () - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] SKIP : tst_Counting::testFailSkip(row 2) Skipping - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(102)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(117)] FAIL! : tst_Counting::testFailFail(row 1) 'false' returned FALSE. () - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] FAIL! : tst_Counting::testFailFail(row 2) 'false' returned FALSE. () - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(99)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(114)] +PASS : tst_Counting::testFailInInit(before) +FAIL! : tst_Counting::testFailInInit(fail) Fail in init() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(233)] +PASS : tst_Counting::testFailInInit(after) +PASS : tst_Counting::testFailInCleanup(before) +QDEBUG : tst_Counting::testFailInCleanup(fail) This test function should execute and then QFAIL in cleanup() +FAIL! : tst_Counting::testFailInCleanup(fail) Fail in cleanup() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(241)] +PASS : tst_Counting::testFailInCleanup(after) +PASS : tst_Counting::testSkipInInit(before) +SKIP : tst_Counting::testSkipInInit(skip) Skip in init() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(235)] +PASS : tst_Counting::testSkipInInit(after) +PASS : tst_Counting::testSkipInCleanup(before) +QDEBUG : tst_Counting::testSkipInCleanup(skip) This test function should execute and then QSKIP in cleanup() +SKIP : tst_Counting::testSkipInCleanup(skip) Skip in cleanup() + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/counting/tst_counting.cpp(243)] +PASS : tst_Counting::testSkipInCleanup(after) PASS : tst_Counting::cleanupTestCase() -Totals: 8 passed, 6 failed, 6 skipped +Totals: 16 passed, 8 failed, 8 skipped ********* Finished testing of tst_Counting ********* diff --git a/tests/auto/testlib/selftests/expected_counting.xml b/tests/auto/testlib/selftests/expected_counting.xml index c5460a7346..a97296807d 100644 --- a/tests/auto/testlib/selftests/expected_counting.xml +++ b/tests/auto/testlib/selftests/expected_counting.xml @@ -19,7 +19,7 @@ - + @@ -28,13 +28,13 @@ - + - + @@ -43,27 +43,27 @@ - + - + - + - + - + @@ -72,25 +72,81 @@ - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_counting.xunitxml b/tests/auto/testlib/selftests/expected_counting.xunitxml index 496546700e..f317ed5923 100644 --- a/tests/auto/testlib/selftests/expected_counting.xunitxml +++ b/tests/auto/testlib/selftests/expected_counting.xunitxml @@ -1,5 +1,5 @@ - + @@ -34,6 +34,20 @@ + + + + + + + + + + + + + + @@ -42,5 +56,9 @@ + + + + -- cgit v1.2.3