Remove QTest::SkipMode from qtestlib API.

When the SkipAll mode is used, tests only report a SKIP for the first
line of test data and subsequent lines are not reported at all.  This
behaviour makes it impossible for anything post-processing test results
to accurately report test pass- and run- rates because they cannot see
how many lines of test data were skipped.

This commit removes SkipMode.  QSKIPs in regular test functions and data
functions are treated the same as SkipSingle, so that every skipped line
of local or global test data is reported in the test log.  QSKIPs
elsewhere are treated the same as SkipAll -- skipping in init() causes
the next test function to be skipped entirely, and skipping in
initTestCase() or initTestCase_data() causes all test functions to be
skipped.

This commit only changes qtestlib and the selftests. A further commit
will change the autotests to remove the SkipMode parameter from QSKIP
calls.

Note that the change in expected output for the globaldata selftest is
deliberate, as the QSKIP in the skipLocal test function has effectively
changed from SkipAll to SkipSingle.

Task-number: QTBUG-21851, QTBUG-21652
Change-Id: I7b1c53fe7ca9dde032810b789d967e2a402bbe5d
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>
Reviewed-by: Alex <alex.blasche@nokia.com>

18 files changed, 56 insertions, 22 deletions

diff --git a/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp b/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp
index 3505496829..1199462c99 100644
--- a/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp
+++ b/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp
@@ -76,7 +76,7 @@ void tst_BenchlibCallgrind::failInChildProcess()
 void tst_BenchlibCallgrind::twoHundredMillionInstructions()
 {
 #if !defined(__GNUC__) || !defined(__i386)
-    QSKIP("This test is only defined for gcc and x86.", SkipAll);
+    QSKIP("This test is only defined for gcc and x86.");
 #else
     QBENCHMARK {
         __asm__ __volatile__(
diff --git a/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp b/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp
index cc8b3a454f..9e274edea5 100644
--- a/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp
+++ b/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp
@@ -56,7 +56,7 @@ private slots:
 void tst_BenchlibTickCounter::threeBillionTicks()
 {
 #ifndef HAVE_TICK_COUNTER
-    QSKIP("Tick counter not available on this platform", SkipAll);
+    QSKIP("Tick counter not available on this platform");
 #else
     QBENCHMARK {
         CycleCounterTicks start = getticks();
diff --git a/tests/auto/testlib/selftests/expected_globaldata.lightxml b/tests/auto/testlib/selftests/expected_globaldata.lightxml
index b8208aa481..7a50f01cb8 100644
--- a/tests/auto/testlib/selftests/expected_globaldata.lightxml
+++ b/tests/auto/testlib/selftests/expected_globaldata.lightxml
@@ -94,6 +94,18 @@
     <DataTag><![CDATA[1:local 1]]></DataTag>
     <Description><![CDATA[cleanup skipLocal local 1 ]]></Description>
 </Message>
+<Message type="qdebug" file="" line="0">
+    <DataTag><![CDATA[1:local 2]]></DataTag>
+    <Description><![CDATA[init skipLocal local 2 ]]></Description>
+</Message>
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp" line="149">
+    <DataTag><![CDATA[1:local 2]]></DataTag>
+    <Description><![CDATA[skipping]]></Description>
+</Message>
+<Message type="qdebug" file="" line="0">
+    <DataTag><![CDATA[1:local 2]]></DataTag>
+    <Description><![CDATA[cleanup skipLocal local 2 ]]></Description>
+</Message>
 </TestFunction>
 <TestFunction name="skipSingle">
 <Message type="qdebug" file="" line="0">
diff --git a/tests/auto/testlib/selftests/expected_globaldata.txt b/tests/auto/testlib/selftests/expected_globaldata.txt
index efd8272d5c..e3905e2fa3 100644
--- a/tests/auto/testlib/selftests/expected_globaldata.txt
+++ b/tests/auto/testlib/selftests/expected_globaldata.txt
@@ -25,6 +25,10 @@ QDEBUG : tst_globaldata::skipLocal(1:local 1) init skipLocal local 1
 SKIP   : tst_globaldata::skipLocal(1:local 1) skipping
    Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp(149)]
 QDEBUG : tst_globaldata::skipLocal(1:local 1) cleanup skipLocal local 1 
+QDEBUG : tst_globaldata::skipLocal(1:local 2) init skipLocal local 2 
+SKIP   : tst_globaldata::skipLocal(1:local 2) skipping
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp(149)]
+QDEBUG : tst_globaldata::skipLocal(1:local 2) cleanup skipLocal local 2 
 QDEBUG : tst_globaldata::skipSingle(1:local 1) init skipSingle local 1 
 QDEBUG : tst_globaldata::skipSingle(1:local 1) global: false local: false 
 QDEBUG : tst_globaldata::skipSingle(1:local 1) cleanup skipSingle local 1 
@@ -41,5 +45,5 @@ QDEBUG : tst_globaldata::skipSingle(2:local 2) cleanup skipSingle local 2
 PASS   : tst_globaldata::skipSingle()
 QDEBUG : tst_globaldata::cleanupTestCase() cleanupTestCase cleanupTestCase (null) 
 PASS   : tst_globaldata::cleanupTestCase()
-Totals: 4 passed, 0 failed, 3 skipped
+Totals: 4 passed, 0 failed, 4 skipped
 ********* Finished testing of tst_globaldata *********
diff --git a/tests/auto/testlib/selftests/expected_globaldata.xml b/tests/auto/testlib/selftests/expected_globaldata.xml
index e0c69475d6..9abbeeef35 100644
--- a/tests/auto/testlib/selftests/expected_globaldata.xml
+++ b/tests/auto/testlib/selftests/expected_globaldata.xml
@@ -96,6 +96,18 @@
     <DataTag><![CDATA[1:local 1]]></DataTag>
     <Description><![CDATA[cleanup skipLocal local 1 ]]></Description>
 </Message>
+<Message type="qdebug" file="" line="0">
+    <DataTag><![CDATA[1:local 2]]></DataTag>
+    <Description><![CDATA[init skipLocal local 2 ]]></Description>
+</Message>
+<Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp" line="149">
+    <DataTag><![CDATA[1:local 2]]></DataTag>
+    <Description><![CDATA[skipping]]></Description>
+</Message>
+<Message type="qdebug" file="" line="0">
+    <DataTag><![CDATA[1:local 2]]></DataTag>
+    <Description><![CDATA[cleanup skipLocal local 2 ]]></Description>
+</Message>
 </TestFunction>
 <TestFunction name="skipSingle">
 <Message type="qdebug" file="" line="0">
diff --git a/tests/auto/testlib/selftests/expected_globaldata.xunitxml b/tests/auto/testlib/selftests/expected_globaldata.xunitxml
index eade59734f..9e0cd149dc 100644
--- a/tests/auto/testlib/selftests/expected_globaldata.xunitxml
+++ b/tests/auto/testlib/selftests/expected_globaldata.xunitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite errors="34" failures="0" tests="6" name="tst_globaldata">
+<testsuite errors="37" failures="0" tests="6" name="tst_globaldata">
   <properties>
     <property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
     <property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
@@ -32,6 +32,9 @@
     <!-- tag="1:local 1" message="init skipLocal local 1 " type="qdebug" -->
     <!-- tag="1:local 1" message="skipping" type="skip" -->
     <!-- tag="1:local 1" message="cleanup skipLocal local 1 " type="qdebug" -->
+    <!-- tag="1:local 2" message="init skipLocal local 2 " type="qdebug" -->
+    <!-- tag="1:local 2" message="skipping" type="skip" -->
+    <!-- tag="1:local 2" message="cleanup skipLocal local 2 " type="qdebug" -->
   </testcase>
   <testcase result="pass" name="skipSingle">
     <!-- tag="1:local 1" message="init skipSingle local 1 " type="qdebug" -->
@@ -72,6 +75,9 @@
 <![CDATA[init skipLocal local 1 ]]>
 <![CDATA[skipping]]>
 <![CDATA[cleanup skipLocal local 1 ]]>
+<![CDATA[init skipLocal local 2 ]]>
+<![CDATA[skipping]]>
+<![CDATA[cleanup skipLocal local 2 ]]>
 <![CDATA[init skipSingle local 1 ]]>
 <![CDATA[global: false local: false ]]>
 <![CDATA[cleanup skipSingle local 1 ]]>
diff --git a/tests/auto/testlib/selftests/expected_singleskip.txt b/tests/auto/testlib/selftests/expected_singleskip.txt
index 1e54c469b4..b9085b1cbd 100644
--- a/tests/auto/testlib/selftests/expected_singleskip.txt
+++ b/tests/auto/testlib/selftests/expected_singleskip.txt
@@ -2,7 +2,7 @@
 Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@
 PASS   : tst_SingleSkip::initTestCase()
 SKIP   : tst_SingleSkip::myTest() skipping test
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp(23)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp(56)]
 PASS   : tst_SingleSkip::cleanupTestCase()
 Totals: 2 passed, 0 failed, 1 skipped
 ********* Finished testing of tst_SingleSkip *********
diff --git a/tests/auto/testlib/selftests/expected_skipinit.txt b/tests/auto/testlib/selftests/expected_skipinit.txt
index a8b4f8ea94..a86060fa5a 100644
--- a/tests/auto/testlib/selftests/expected_skipinit.txt
+++ b/tests/auto/testlib/selftests/expected_skipinit.txt
@@ -1,7 +1,7 @@
 ********* Start testing of tst_SkipInit *********
 Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@
 SKIP   : tst_SkipInit::initTestCase() Skip inside initTestCase. This should skip all tests in the class.
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(22)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(55)]
 PASS   : tst_SkipInit::cleanupTestCase()
 Totals: 1 passed, 0 failed, 1 skipped
 ********* Finished testing of tst_SkipInit *********
diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.lightxml b/tests/auto/testlib/selftests/expected_skipinitdata.lightxml
index e933b7157d..cd820240fd 100644
--- a/tests/auto/testlib/selftests/expected_skipinitdata.lightxml
+++ b/tests/auto/testlib/selftests/expected_skipinitdata.lightxml
@@ -4,6 +4,6 @@
 </Environment>
 <TestFunction name="initTestCase">
 <Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp" line="56">
-    <Description><![CDATA[Skip inside initTestCase. This should skip all tests in the class.]]></Description>
+    <Description><![CDATA[Skip inside initTestCase_data. This should skip all tests in the class.]]></Description>
 </Message>
 </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.txt b/tests/auto/testlib/selftests/expected_skipinitdata.txt
index 8640855497..a4cbbea194 100644
--- a/tests/auto/testlib/selftests/expected_skipinitdata.txt
+++ b/tests/auto/testlib/selftests/expected_skipinitdata.txt
@@ -1,6 +1,6 @@
 ********* Start testing of tst_SkipInitData *********
 Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@
-SKIP   : tst_SkipInitData::initTestCase() Skip inside initTestCase. This should skip all tests in the class.
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp(23)]
+SKIP   : tst_SkipInitData::initTestCase() Skip inside initTestCase_data. This should skip all tests in the class.
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp(56)]
 Totals: 0 passed, 0 failed, 1 skipped
 ********* Finished testing of tst_SkipInitData *********
diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.xml b/tests/auto/testlib/selftests/expected_skipinitdata.xml
index 7ad34d913f..8371700547 100644
--- a/tests/auto/testlib/selftests/expected_skipinitdata.xml
+++ b/tests/auto/testlib/selftests/expected_skipinitdata.xml
@@ -6,7 +6,7 @@
 </Environment>
 <TestFunction name="initTestCase">
 <Message type="skip" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp" line="56">
-    <Description><![CDATA[Skip inside initTestCase. This should skip all tests in the class.]]></Description>
+    <Description><![CDATA[Skip inside initTestCase_data. This should skip all tests in the class.]]></Description>
 </Message>
 </TestFunction>
 </TestCase>
diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml b/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml
index 93967eb055..699d50f7a7 100644
--- a/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml
+++ b/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml
@@ -5,9 +5,9 @@
     <property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
   </properties>
   <testcase name="initTestCase">
-    <!-- message="Skip inside initTestCase. This should skip all tests in the class." type="skip" -->
+    <!-- message="Skip inside initTestCase_data. This should skip all tests in the class." type="skip" -->
   </testcase>
   <system-err>
-<![CDATA[Skip inside initTestCase. This should skip all tests in the class.]]>
+<![CDATA[Skip inside initTestCase_data. This should skip all tests in the class.]]>
   </system-err>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp b/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp
index 18b91ccfe5..898cccf435 100644
--- a/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp
+++ b/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp
@@ -126,7 +126,7 @@ void tst_globaldata::skip_data()
     QTest::newRow("local 1") << false;
     QTest::newRow("local 2") << true;
 
-    QSKIP("skipping", SkipAll);
+    QSKIP("skipping");
 }
 
 void tst_globaldata::skip()
@@ -140,13 +140,13 @@ void tst_globaldata::skipSingle()
     QFETCH(bool, booll);
 
     if (booli && !booll)
-        QSKIP("skipping", SkipSingle);
+        QSKIP("skipping");
     qDebug() << "global:" << booli << "local:" << booll;
 }
 
 void tst_globaldata::skipLocal()
 {
-    QSKIP("skipping", SkipAll);
+    QSKIP("skipping");
 }
 
 QTEST_MAIN(tst_globaldata)
diff --git a/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp b/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp
index 5567b88022..7701eb81a0 100644
--- a/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp
+++ b/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp
@@ -53,7 +53,7 @@ private slots:
 
 void tst_SingleSkip::myTest() const
 {
-    QSKIP("skipping test", SkipAll);
+    QSKIP("skipping test");
 }
 
 QTEST_MAIN(tst_SingleSkip)
diff --git a/tests/auto/testlib/selftests/skip/tst_skip.cpp b/tests/auto/testlib/selftests/skip/tst_skip.cpp
index b5b53884b7..905fe2dde4 100644
--- a/tests/auto/testlib/selftests/skip/tst_skip.cpp
+++ b/tests/auto/testlib/selftests/skip/tst_skip.cpp
@@ -65,7 +65,7 @@ void tst_Skip::test_data()
     QTest::newRow("local 1") << false;
     QTest::newRow("local 2") << true;
 
-    QSKIP("skipping all", SkipAll);
+    QSKIP("skipping all");
 }
 
 void tst_Skip::test()
@@ -75,7 +75,7 @@ void tst_Skip::test()
 
 void tst_Skip::emptytest_data()
 {
-    QSKIP("skipping all", SkipAll);
+    QSKIP("skipping all");
 }
 
 void tst_Skip::emptytest()
@@ -94,7 +94,7 @@ void tst_Skip::singleSkip()
 {
     QFETCH(bool, booll);
     if (!booll)
-        QSKIP("skipping one", SkipSingle);
+        QSKIP("skipping one");
     qDebug("this line should only be reached once (%s)", booll ? "true" : "false");
 }
 
diff --git a/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp b/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp
index 4cd882f747..1a9fc1eeec 100644
--- a/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp
+++ b/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp
@@ -52,7 +52,7 @@ private slots:
 
 void tst_SkipInit::initTestCase() const
 {
-    QSKIP("Skip inside initTestCase. This should skip all tests in the class.", SkipAll);
+    QSKIP("Skip inside initTestCase. This should skip all tests in the class.");
 }
 
 /*! \internal
diff --git a/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp b/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp
index fb97fc978d..b09e99b32f 100644
--- a/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp
+++ b/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp
@@ -53,7 +53,7 @@ private slots:
 
 void tst_SkipInitData::initTestCase_data() const
 {
-    QSKIP("Skip inside initTestCase. This should skip all tests in the class.", SkipAll);
+    QSKIP("Skip inside initTestCase_data. This should skip all tests in the class.");
 }
 
 void tst_SkipInitData::initTestCase() const
diff --git a/tests/auto/testlib/selftests/xunit/tst_xunit.cpp b/tests/auto/testlib/selftests/xunit/tst_xunit.cpp
index 8f2f712426..05acc8cfe7 100644
--- a/tests/auto/testlib/selftests/xunit/tst_xunit.cpp
+++ b/tests/auto/testlib/selftests/xunit/tst_xunit.cpp
@@ -76,7 +76,7 @@ void tst_Xunit::testFunc2()
 
 void tst_Xunit::testFunc3()
 {
-    QSKIP("skipping this function!", SkipAll);
+    QSKIP("skipping this function!");
 }
 
 void tst_Xunit::testFunc4()