From 016cd01846539404a0eff1e3823fa2206fe2c9a9 Mon Sep 17 00:00:00 2001 From: Jason McDonald Date: Tue, 18 Oct 2011 14:53:23 +1000 Subject: Remove QTest::SkipMode from qtestlib API. When the SkipAll mode is used, tests only report a SKIP for the first line of test data and subsequent lines are not reported at all. This behaviour makes it impossible for anything post-processing test results to accurately report test pass- and run- rates because they cannot see how many lines of test data were skipped. This commit removes SkipMode. QSKIPs in regular test functions and data functions are treated the same as SkipSingle, so that every skipped line of local or global test data is reported in the test log. QSKIPs elsewhere are treated the same as SkipAll -- skipping in init() causes the next test function to be skipped entirely, and skipping in initTestCase() or initTestCase_data() causes all test functions to be skipped. This commit only changes qtestlib and the selftests. A further commit will change the autotests to remove the SkipMode parameter from QSKIP calls. Note that the change in expected output for the globaldata selftest is deliberate, as the QSKIP in the skipLocal test function has effectively changed from SkipAll to SkipSingle. Task-number: QTBUG-21851, QTBUG-21652 Change-Id: I7b1c53fe7ca9dde032810b789d967e2a402bbe5d Reviewed-by: Rohan McGovern Reviewed-by: Alex --- .../selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp | 2 +- .../benchlibtickcounter/tst_benchlibtickcounter.cpp | 2 +- tests/auto/testlib/selftests/expected_globaldata.lightxml | 12 ++++++++++++ tests/auto/testlib/selftests/expected_globaldata.txt | 6 +++++- tests/auto/testlib/selftests/expected_globaldata.xml | 12 ++++++++++++ tests/auto/testlib/selftests/expected_globaldata.xunitxml | 8 +++++++- tests/auto/testlib/selftests/expected_singleskip.txt | 2 +- tests/auto/testlib/selftests/expected_skipinit.txt | 2 +- tests/auto/testlib/selftests/expected_skipinitdata.lightxml | 2 +- tests/auto/testlib/selftests/expected_skipinitdata.txt | 4 ++-- tests/auto/testlib/selftests/expected_skipinitdata.xml | 2 +- tests/auto/testlib/selftests/expected_skipinitdata.xunitxml | 4 ++-- tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp | 6 +++--- tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp | 2 +- tests/auto/testlib/selftests/skip/tst_skip.cpp | 6 +++--- tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp | 2 +- .../auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp | 2 +- tests/auto/testlib/selftests/xunit/tst_xunit.cpp | 2 +- 18 files changed, 56 insertions(+), 22 deletions(-) (limited to 'tests') diff --git a/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp b/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp index 3505496829..1199462c99 100644 --- a/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp +++ b/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp @@ -76,7 +76,7 @@ void tst_BenchlibCallgrind::failInChildProcess() void tst_BenchlibCallgrind::twoHundredMillionInstructions() { #if !defined(__GNUC__) || !defined(__i386) - QSKIP("This test is only defined for gcc and x86.", SkipAll); + QSKIP("This test is only defined for gcc and x86."); #else QBENCHMARK { __asm__ __volatile__( diff --git a/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp b/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp index cc8b3a454f..9e274edea5 100644 --- a/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp +++ b/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp @@ -56,7 +56,7 @@ private slots: void tst_BenchlibTickCounter::threeBillionTicks() { #ifndef HAVE_TICK_COUNTER - QSKIP("Tick counter not available on this platform", SkipAll); + QSKIP("Tick counter not available on this platform"); #else QBENCHMARK { CycleCounterTicks start = getticks(); diff --git a/tests/auto/testlib/selftests/expected_globaldata.lightxml b/tests/auto/testlib/selftests/expected_globaldata.lightxml index b8208aa481..7a50f01cb8 100644 --- a/tests/auto/testlib/selftests/expected_globaldata.lightxml +++ b/tests/auto/testlib/selftests/expected_globaldata.lightxml @@ -94,6 +94,18 @@ + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_globaldata.txt b/tests/auto/testlib/selftests/expected_globaldata.txt index efd8272d5c..e3905e2fa3 100644 --- a/tests/auto/testlib/selftests/expected_globaldata.txt +++ b/tests/auto/testlib/selftests/expected_globaldata.txt @@ -25,6 +25,10 @@ QDEBUG : tst_globaldata::skipLocal(1:local 1) init skipLocal local 1 SKIP : tst_globaldata::skipLocal(1:local 1) skipping Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp(149)] QDEBUG : tst_globaldata::skipLocal(1:local 1) cleanup skipLocal local 1 +QDEBUG : tst_globaldata::skipLocal(1:local 2) init skipLocal local 2 +SKIP : tst_globaldata::skipLocal(1:local 2) skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp(149)] +QDEBUG : tst_globaldata::skipLocal(1:local 2) cleanup skipLocal local 2 QDEBUG : tst_globaldata::skipSingle(1:local 1) init skipSingle local 1 QDEBUG : tst_globaldata::skipSingle(1:local 1) global: false local: false QDEBUG : tst_globaldata::skipSingle(1:local 1) cleanup skipSingle local 1 @@ -41,5 +45,5 @@ QDEBUG : tst_globaldata::skipSingle(2:local 2) cleanup skipSingle local 2 PASS : tst_globaldata::skipSingle() QDEBUG : tst_globaldata::cleanupTestCase() cleanupTestCase cleanupTestCase (null) PASS : tst_globaldata::cleanupTestCase() -Totals: 4 passed, 0 failed, 3 skipped +Totals: 4 passed, 0 failed, 4 skipped ********* Finished testing of tst_globaldata ********* diff --git a/tests/auto/testlib/selftests/expected_globaldata.xml b/tests/auto/testlib/selftests/expected_globaldata.xml index e0c69475d6..9abbeeef35 100644 --- a/tests/auto/testlib/selftests/expected_globaldata.xml +++ b/tests/auto/testlib/selftests/expected_globaldata.xml @@ -96,6 +96,18 @@ + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_globaldata.xunitxml b/tests/auto/testlib/selftests/expected_globaldata.xunitxml index eade59734f..9e0cd149dc 100644 --- a/tests/auto/testlib/selftests/expected_globaldata.xunitxml +++ b/tests/auto/testlib/selftests/expected_globaldata.xunitxml @@ -1,5 +1,5 @@ - + @@ -32,6 +32,9 @@ + + + @@ -72,6 +75,9 @@ + + + diff --git a/tests/auto/testlib/selftests/expected_singleskip.txt b/tests/auto/testlib/selftests/expected_singleskip.txt index 1e54c469b4..b9085b1cbd 100644 --- a/tests/auto/testlib/selftests/expected_singleskip.txt +++ b/tests/auto/testlib/selftests/expected_singleskip.txt @@ -2,7 +2,7 @@ Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ PASS : tst_SingleSkip::initTestCase() SKIP : tst_SingleSkip::myTest() skipping test - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp(23)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp(56)] PASS : tst_SingleSkip::cleanupTestCase() Totals: 2 passed, 0 failed, 1 skipped ********* Finished testing of tst_SingleSkip ********* diff --git a/tests/auto/testlib/selftests/expected_skipinit.txt b/tests/auto/testlib/selftests/expected_skipinit.txt index a8b4f8ea94..a86060fa5a 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.txt +++ b/tests/auto/testlib/selftests/expected_skipinit.txt @@ -1,7 +1,7 @@ ********* Start testing of tst_SkipInit ********* Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ SKIP : tst_SkipInit::initTestCase() Skip inside initTestCase. This should skip all tests in the class. - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(22)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(55)] PASS : tst_SkipInit::cleanupTestCase() Totals: 1 passed, 0 failed, 1 skipped ********* Finished testing of tst_SkipInit ********* diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.lightxml b/tests/auto/testlib/selftests/expected_skipinitdata.lightxml index e933b7157d..cd820240fd 100644 --- a/tests/auto/testlib/selftests/expected_skipinitdata.lightxml +++ b/tests/auto/testlib/selftests/expected_skipinitdata.lightxml @@ -4,6 +4,6 @@ - + diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.txt b/tests/auto/testlib/selftests/expected_skipinitdata.txt index 8640855497..a4cbbea194 100644 --- a/tests/auto/testlib/selftests/expected_skipinitdata.txt +++ b/tests/auto/testlib/selftests/expected_skipinitdata.txt @@ -1,6 +1,6 @@ ********* Start testing of tst_SkipInitData ********* Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ -SKIP : tst_SkipInitData::initTestCase() Skip inside initTestCase. This should skip all tests in the class. - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp(23)] +SKIP : tst_SkipInitData::initTestCase() Skip inside initTestCase_data. This should skip all tests in the class. + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp(56)] Totals: 0 passed, 0 failed, 1 skipped ********* Finished testing of tst_SkipInitData ********* diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.xml b/tests/auto/testlib/selftests/expected_skipinitdata.xml index 7ad34d913f..8371700547 100644 --- a/tests/auto/testlib/selftests/expected_skipinitdata.xml +++ b/tests/auto/testlib/selftests/expected_skipinitdata.xml @@ -6,7 +6,7 @@ - + diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml b/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml index 93967eb055..699d50f7a7 100644 --- a/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml +++ b/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml @@ -5,9 +5,9 @@ - + - + diff --git a/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp b/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp index 18b91ccfe5..898cccf435 100644 --- a/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp +++ b/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp @@ -126,7 +126,7 @@ void tst_globaldata::skip_data() QTest::newRow("local 1") << false; QTest::newRow("local 2") << true; - QSKIP("skipping", SkipAll); + QSKIP("skipping"); } void tst_globaldata::skip() @@ -140,13 +140,13 @@ void tst_globaldata::skipSingle() QFETCH(bool, booll); if (booli && !booll) - QSKIP("skipping", SkipSingle); + QSKIP("skipping"); qDebug() << "global:" << booli << "local:" << booll; } void tst_globaldata::skipLocal() { - QSKIP("skipping", SkipAll); + QSKIP("skipping"); } QTEST_MAIN(tst_globaldata) diff --git a/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp b/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp index 5567b88022..7701eb81a0 100644 --- a/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp +++ b/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp @@ -53,7 +53,7 @@ private slots: void tst_SingleSkip::myTest() const { - QSKIP("skipping test", SkipAll); + QSKIP("skipping test"); } QTEST_MAIN(tst_SingleSkip) diff --git a/tests/auto/testlib/selftests/skip/tst_skip.cpp b/tests/auto/testlib/selftests/skip/tst_skip.cpp index b5b53884b7..905fe2dde4 100644 --- a/tests/auto/testlib/selftests/skip/tst_skip.cpp +++ b/tests/auto/testlib/selftests/skip/tst_skip.cpp @@ -65,7 +65,7 @@ void tst_Skip::test_data() QTest::newRow("local 1") << false; QTest::newRow("local 2") << true; - QSKIP("skipping all", SkipAll); + QSKIP("skipping all"); } void tst_Skip::test() @@ -75,7 +75,7 @@ void tst_Skip::test() void tst_Skip::emptytest_data() { - QSKIP("skipping all", SkipAll); + QSKIP("skipping all"); } void tst_Skip::emptytest() @@ -94,7 +94,7 @@ void tst_Skip::singleSkip() { QFETCH(bool, booll); if (!booll) - QSKIP("skipping one", SkipSingle); + QSKIP("skipping one"); qDebug("this line should only be reached once (%s)", booll ? "true" : "false"); } diff --git a/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp b/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp index 4cd882f747..1a9fc1eeec 100644 --- a/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp +++ b/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp @@ -52,7 +52,7 @@ private slots: void tst_SkipInit::initTestCase() const { - QSKIP("Skip inside initTestCase. This should skip all tests in the class.", SkipAll); + QSKIP("Skip inside initTestCase. This should skip all tests in the class."); } /*! \internal diff --git a/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp b/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp index fb97fc978d..b09e99b32f 100644 --- a/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp +++ b/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp @@ -53,7 +53,7 @@ private slots: void tst_SkipInitData::initTestCase_data() const { - QSKIP("Skip inside initTestCase. This should skip all tests in the class.", SkipAll); + QSKIP("Skip inside initTestCase_data. This should skip all tests in the class."); } void tst_SkipInitData::initTestCase() const diff --git a/tests/auto/testlib/selftests/xunit/tst_xunit.cpp b/tests/auto/testlib/selftests/xunit/tst_xunit.cpp index 8f2f712426..05acc8cfe7 100644 --- a/tests/auto/testlib/selftests/xunit/tst_xunit.cpp +++ b/tests/auto/testlib/selftests/xunit/tst_xunit.cpp @@ -76,7 +76,7 @@ void tst_Xunit::testFunc2() void tst_Xunit::testFunc3() { - QSKIP("skipping this function!", SkipAll); + QSKIP("skipping this function!"); } void tst_Xunit::testFunc4() -- cgit v1.2.3