Test skip and fail in cleanup() as well as in cleanupTestCase()

The skipcleanup and failcleanup tests were actually testing skip and
fail in cleanupTestCase(), not in cleanup(). Add almost-duplicate
tests and clean up so that we now have {fail,skip}cleanup(,testcase}
tests to cover all four cases. Generated expected output. The new
tests (with old names) get their fail or skip - during cleanup() -
reported against the test instead of the cleanupTestCase function.
(Results for {init,cleanup}TestCase() are always reported, even when
these slots are not defined, as no-op passes.)

Pick-to: 6.4
Change-Id: I0988d1696b50c0e2f30c45ddc25e1bd0bfd2151a
Reviewed-by: Ivan Solovev <ivan.solovev@qt.io>
Reviewed-by: Mitch Curtis <mitch.curtis@qt.io>

34 files changed, 312 insertions, 58 deletions

diff --git a/tests/auto/testlib/selftests/CMakeLists.txt b/tests/auto/testlib/selftests/CMakeLists.txt
index f7315c7db6..ceea1226f8 100644
--- a/tests/auto/testlib/selftests/CMakeLists.txt
+++ b/tests/auto/testlib/selftests/CMakeLists.txt
@@ -60,6 +60,7 @@ set(subprograms
     expectfail
     extendedcompare
     failcleanup
+    failcleanuptestcase
     faildatatype
     failfetchtype
     failinit
@@ -82,6 +83,7 @@ set(subprograms
     singleskip
     skip
     skipcleanup
+    skipcleanuptestcase
     skipinit
     skipinitdata
     sleep
diff --git a/tests/auto/testlib/selftests/expected_failcleanup.junitxml b/tests/auto/testlib/selftests/expected_failcleanup.junitxml
index a40c724a46..8ea5799363 100644
--- a/tests/auto/testlib/selftests/expected_failcleanup.junitxml
+++ b/tests/auto/testlib/selftests/expected_failcleanup.junitxml
@@ -6,8 +6,8 @@
     <property name="QtBuild" value=""/>
   </properties>
   <testcase name="initTestCase" classname="tst_FailCleanup" time="@TEST_DURATION@"/>
-  <testcase name="aTestFunction" classname="tst_FailCleanup" time="@TEST_DURATION@"/>
-  <testcase name="cleanupTestCase" classname="tst_FailCleanup" time="@TEST_DURATION@">
-    <failure type="fail" message="&apos;false&apos; returned FALSE. (Fail inside cleanupTestCase)"/>
+  <testcase name="aTestFunction" classname="tst_FailCleanup" time="@TEST_DURATION@">
+    <failure type="fail" message="&apos;false&apos; returned FALSE. (Fail inside cleanup)"/>
   </testcase>
+  <testcase name="cleanupTestCase" classname="tst_FailCleanup" time="@TEST_DURATION@"/>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/expected_failcleanup.lightxml b/tests/auto/testlib/selftests/expected_failcleanup.lightxml
index ce9da0d2a8..a9d7a1361f 100644
--- a/tests/auto/testlib/selftests/expected_failcleanup.lightxml
+++ b/tests/auto/testlib/selftests/expected_failcleanup.lightxml
@@ -8,13 +8,13 @@
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="aTestFunction">
-    <Incident type="pass" file="" line="0" />
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp" line="0">
+      <Description><![CDATA['false' returned FALSE. (Fail inside cleanup)]]></Description>
+    </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="cleanupTestCase">
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp" line="0">
-      <Description><![CDATA['false' returned FALSE. (Fail inside cleanupTestCase)]]></Description>
-    </Incident>
+    <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
   <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_failcleanup.tap b/tests/auto/testlib/selftests/expected_failcleanup.tap
index 8ec0aca6d4..64556b32b2 100644
--- a/tests/auto/testlib/selftests/expected_failcleanup.tap
+++ b/tests/auto/testlib/selftests/expected_failcleanup.tap
@@ -1,19 +1,19 @@
 TAP version 13
 # tst_FailCleanup
 ok 1 - initTestCase()
-ok 2 - aTestFunction()
-not ok 3 - cleanupTestCase()
+not ok 2 - aTestFunction()
   ---
   type: QVERIFY
-  message: Fail inside cleanupTestCase
+  message: Fail inside cleanup
   wanted: true (false)
   found: false (false)
   expected: true (false)
   actual: false (false)
-  at: tst_FailCleanup::cleanupTestCase() (qtbase/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp:0)
+  at: tst_FailCleanup::aTestFunction() (qtbase/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp
   line: 0
   ...
+ok 3 - cleanupTestCase()
 1..3
 # tests 3
 # pass 2
diff --git a/tests/auto/testlib/selftests/expected_failcleanup.teamcity b/tests/auto/testlib/selftests/expected_failcleanup.teamcity
index 1f8d353581..12d7b2f154 100644
--- a/tests/auto/testlib/selftests/expected_failcleanup.teamcity
+++ b/tests/auto/testlib/selftests/expected_failcleanup.teamcity
@@ -2,8 +2,8 @@
 ##teamcity[testStarted name='initTestCase()' flowId='tst_FailCleanup']
 ##teamcity[testFinished name='initTestCase()' flowId='tst_FailCleanup']
 ##teamcity[testStarted name='aTestFunction()' flowId='tst_FailCleanup']
+##teamcity[testFailed name='aTestFunction()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp(0)|]' details='|'false|' returned FALSE. (Fail inside cleanup)' flowId='tst_FailCleanup']
 ##teamcity[testFinished name='aTestFunction()' flowId='tst_FailCleanup']
 ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_FailCleanup']
-##teamcity[testFailed name='cleanupTestCase()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp(0)|]' details='|'false|' returned FALSE. (Fail inside cleanupTestCase)' flowId='tst_FailCleanup']
 ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_FailCleanup']
 ##teamcity[testSuiteFinished name='tst_FailCleanup' flowId='tst_FailCleanup']
diff --git a/tests/auto/testlib/selftests/expected_failcleanup.txt b/tests/auto/testlib/selftests/expected_failcleanup.txt
index a5776e59bf..4fac7940fd 100644
--- a/tests/auto/testlib/selftests/expected_failcleanup.txt
+++ b/tests/auto/testlib/selftests/expected_failcleanup.txt
@@ -1,8 +1,8 @@
 ********* Start testing of tst_FailCleanup *********
 Config: Using QtTest library
 PASS   : tst_FailCleanup::initTestCase()
-PASS   : tst_FailCleanup::aTestFunction()
-FAIL!  : tst_FailCleanup::cleanupTestCase() 'false' returned FALSE. (Fail inside cleanupTestCase)
+FAIL!  : tst_FailCleanup::aTestFunction() 'false' returned FALSE. (Fail inside cleanup)
    Loc: [qtbase/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp(0)]
+PASS   : tst_FailCleanup::cleanupTestCase()
 Totals: 2 passed, 1 failed, 0 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_FailCleanup *********
diff --git a/tests/auto/testlib/selftests/expected_failcleanup.xml b/tests/auto/testlib/selftests/expected_failcleanup.xml
index 3521802ca0..f4f041e704 100644
--- a/tests/auto/testlib/selftests/expected_failcleanup.xml
+++ b/tests/auto/testlib/selftests/expected_failcleanup.xml
@@ -10,13 +10,13 @@
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="aTestFunction">
-    <Incident type="pass" file="" line="0" />
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp" line="0">
+      <Description><![CDATA['false' returned FALSE. (Fail inside cleanup)]]></Description>
+    </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="cleanupTestCase">
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp" line="0">
-      <Description><![CDATA['false' returned FALSE. (Fail inside cleanupTestCase)]]></Description>
-    </Incident>
+    <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
   <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_failcleanuptestcase.junitxml b/tests/auto/testlib/selftests/expected_failcleanuptestcase.junitxml
new file mode 100644
index 0000000000..8e694a4dfe
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_failcleanuptestcase.junitxml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuite name="tst_FailCleanupTestCase" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="3" failures="1" errors="0" skipped="0" time="@TEST_DURATION@">
+  <properties>
+    <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
+    <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
+    <property name="QtBuild" value=""/>
+  </properties>
+  <testcase name="initTestCase" classname="tst_FailCleanupTestCase" time="@TEST_DURATION@"/>
+  <testcase name="aTestFunction" classname="tst_FailCleanupTestCase" time="@TEST_DURATION@"/>
+  <testcase name="cleanupTestCase" classname="tst_FailCleanupTestCase" time="@TEST_DURATION@">
+    <failure type="fail" message="&apos;false&apos; returned FALSE. (Fail inside cleanupTestCase)"/>
+  </testcase>
+</testsuite>
diff --git a/tests/auto/testlib/selftests/expected_failcleanuptestcase.lightxml b/tests/auto/testlib/selftests/expected_failcleanuptestcase.lightxml
new file mode 100644
index 0000000000..ed92e1e6b0
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_failcleanuptestcase.lightxml
@@ -0,0 +1,20 @@
+  <Environment>
+    <QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
+    <QtBuild/>
+    <QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
+  </Environment>
+  <TestFunction name="initTestCase">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="aTestFunction">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="cleanupTestCase">
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/failcleanuptestcase/tst_failcleanuptestcase.cpp" line="0">
+      <Description><![CDATA['false' returned FALSE. (Fail inside cleanupTestCase)]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_failcleanuptestcase.tap b/tests/auto/testlib/selftests/expected_failcleanuptestcase.tap
new file mode 100644
index 0000000000..1bad92ce1f
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_failcleanuptestcase.tap
@@ -0,0 +1,20 @@
+TAP version 13
+# tst_FailCleanupTestCase
+ok 1 - initTestCase()
+ok 2 - aTestFunction()
+not ok 3 - cleanupTestCase()
+  ---
+  type: QVERIFY
+  message: Fail inside cleanupTestCase
+  wanted: true (false)
+  found: false (false)
+  expected: true (false)
+  actual: false (false)
+  at: tst_FailCleanupTestCase::cleanupTestCase() (qtbase/tests/auto/testlib/selftests/failcleanuptestcase/tst_failcleanuptestcase.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/failcleanuptestcase/tst_failcleanuptestcase.cpp
+  line: 0
+  ...
+1..3
+# tests 3
+# pass 2
+# fail 1
diff --git a/tests/auto/testlib/selftests/expected_failcleanuptestcase.teamcity b/tests/auto/testlib/selftests/expected_failcleanuptestcase.teamcity
new file mode 100644
index 0000000000..b76c7e3f45
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_failcleanuptestcase.teamcity
@@ -0,0 +1,9 @@
+##teamcity[testSuiteStarted name='tst_FailCleanupTestCase' flowId='tst_FailCleanupTestCase']
+##teamcity[testStarted name='initTestCase()' flowId='tst_FailCleanupTestCase']
+##teamcity[testFinished name='initTestCase()' flowId='tst_FailCleanupTestCase']
+##teamcity[testStarted name='aTestFunction()' flowId='tst_FailCleanupTestCase']
+##teamcity[testFinished name='aTestFunction()' flowId='tst_FailCleanupTestCase']
+##teamcity[testStarted name='cleanupTestCase()' flowId='tst_FailCleanupTestCase']
+##teamcity[testFailed name='cleanupTestCase()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/failcleanuptestcase/tst_failcleanuptestcase.cpp(0)|]' details='|'false|' returned FALSE. (Fail inside cleanupTestCase)' flowId='tst_FailCleanupTestCase']
+##teamcity[testFinished name='cleanupTestCase()' flowId='tst_FailCleanupTestCase']
+##teamcity[testSuiteFinished name='tst_FailCleanupTestCase' flowId='tst_FailCleanupTestCase']
diff --git a/tests/auto/testlib/selftests/expected_failcleanuptestcase.txt b/tests/auto/testlib/selftests/expected_failcleanuptestcase.txt
new file mode 100644
index 0000000000..2e4619394b
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_failcleanuptestcase.txt
@@ -0,0 +1,8 @@
+********* Start testing of tst_FailCleanupTestCase *********
+Config: Using QtTest library
+PASS   : tst_FailCleanupTestCase::initTestCase()
+PASS   : tst_FailCleanupTestCase::aTestFunction()
+FAIL!  : tst_FailCleanupTestCase::cleanupTestCase() 'false' returned FALSE. (Fail inside cleanupTestCase)
+   Loc: [qtbase/tests/auto/testlib/selftests/failcleanuptestcase/tst_failcleanuptestcase.cpp(0)]
+Totals: 2 passed, 1 failed, 0 skipped, 0 blacklisted, 0ms
+********* Finished testing of tst_FailCleanupTestCase *********
diff --git a/tests/auto/testlib/selftests/expected_failcleanuptestcase.xml b/tests/auto/testlib/selftests/expected_failcleanuptestcase.xml
new file mode 100644
index 0000000000..1537440317
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_failcleanuptestcase.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<TestCase name="tst_FailCleanupTestCase">
+  <Environment>
+    <QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
+    <QtBuild/>
+    <QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
+  </Environment>
+  <TestFunction name="initTestCase">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="aTestFunction">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="cleanupTestCase">
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/failcleanuptestcase/tst_failcleanuptestcase.cpp" line="0">
+      <Description><![CDATA['false' returned FALSE. (Fail inside cleanupTestCase)]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <Duration msecs="0"/>
+</TestCase>
diff --git a/tests/auto/testlib/selftests/expected_skipcleanup.junitxml b/tests/auto/testlib/selftests/expected_skipcleanup.junitxml
index 88a5c250ff..5ef20a0b2e 100644
--- a/tests/auto/testlib/selftests/expected_skipcleanup.junitxml
+++ b/tests/auto/testlib/selftests/expected_skipcleanup.junitxml
@@ -6,8 +6,8 @@
     <property name="QtBuild" value=""/>
   </properties>
   <testcase name="initTestCase" classname="tst_SkipCleanup" time="@TEST_DURATION@"/>
-  <testcase name="aTestFunction" classname="tst_SkipCleanup" time="@TEST_DURATION@"/>
-  <testcase name="cleanupTestCase" classname="tst_SkipCleanup" time="@TEST_DURATION@">
-    <skipped message="Skip inside cleanupTestCase."/>
+  <testcase name="aTestFunction" classname="tst_SkipCleanup" time="@TEST_DURATION@">
+    <skipped message="Skip inside cleanup."/>
   </testcase>
+  <testcase name="cleanupTestCase" classname="tst_SkipCleanup" time="@TEST_DURATION@"/>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/expected_skipcleanup.lightxml b/tests/auto/testlib/selftests/expected_skipcleanup.lightxml
index f2f8330bbc..c97f125227 100644
--- a/tests/auto/testlib/selftests/expected_skipcleanup.lightxml
+++ b/tests/auto/testlib/selftests/expected_skipcleanup.lightxml
@@ -8,13 +8,13 @@
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="aTestFunction">
-    <Incident type="pass" file="" line="0" />
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/skipcleanup/tst_skipcleanup.cpp" line="0">
+      <Description><![CDATA[Skip inside cleanup.]]></Description>
+    </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="cleanupTestCase">
-    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/skipcleanup/tst_skipcleanup.cpp" line="0">
-      <Description><![CDATA[Skip inside cleanupTestCase.]]></Description>
-    </Incident>
+    <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
   <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_skipcleanup.tap b/tests/auto/testlib/selftests/expected_skipcleanup.tap
index 4a3f8f852c..36d979a84a 100644
--- a/tests/auto/testlib/selftests/expected_skipcleanup.tap
+++ b/tests/auto/testlib/selftests/expected_skipcleanup.tap
@@ -1,8 +1,8 @@
 TAP version 13
 # tst_SkipCleanup
 ok 1 - initTestCase()
-ok 2 - aTestFunction()
-ok 3 - cleanupTestCase() # SKIP Skip inside cleanupTestCase.
+ok 2 - aTestFunction() # SKIP Skip inside cleanup.
+ok 3 - cleanupTestCase()
 1..3
 # tests 3
 # pass 2
diff --git a/tests/auto/testlib/selftests/expected_skipcleanup.teamcity b/tests/auto/testlib/selftests/expected_skipcleanup.teamcity
index bdc3c0f70a..b5c3709193 100644
--- a/tests/auto/testlib/selftests/expected_skipcleanup.teamcity
+++ b/tests/auto/testlib/selftests/expected_skipcleanup.teamcity
@@ -2,8 +2,8 @@
 ##teamcity[testStarted name='initTestCase()' flowId='tst_SkipCleanup']
 ##teamcity[testFinished name='initTestCase()' flowId='tst_SkipCleanup']
 ##teamcity[testStarted name='aTestFunction()' flowId='tst_SkipCleanup']
+##teamcity[testIgnored name='aTestFunction()' message='Skip inside cleanup. |[Loc: qtbase/tests/auto/testlib/selftests/skipcleanup/tst_skipcleanup.cpp(0)|]' flowId='tst_SkipCleanup']
 ##teamcity[testFinished name='aTestFunction()' flowId='tst_SkipCleanup']
 ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_SkipCleanup']
-##teamcity[testIgnored name='cleanupTestCase()' message='Skip inside cleanupTestCase. |[Loc: qtbase/tests/auto/testlib/selftests/skipcleanup/tst_skipcleanup.cpp(0)|]' flowId='tst_SkipCleanup']
 ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_SkipCleanup']
 ##teamcity[testSuiteFinished name='tst_SkipCleanup' flowId='tst_SkipCleanup']
diff --git a/tests/auto/testlib/selftests/expected_skipcleanup.txt b/tests/auto/testlib/selftests/expected_skipcleanup.txt
index 987e2b9a14..c1bd95e5a9 100644
--- a/tests/auto/testlib/selftests/expected_skipcleanup.txt
+++ b/tests/auto/testlib/selftests/expected_skipcleanup.txt
@@ -1,8 +1,8 @@
 ********* Start testing of tst_SkipCleanup *********
 Config: Using QtTest library
 PASS   : tst_SkipCleanup::initTestCase()
-PASS   : tst_SkipCleanup::aTestFunction()
-SKIP   : tst_SkipCleanup::cleanupTestCase() Skip inside cleanupTestCase.
+SKIP   : tst_SkipCleanup::aTestFunction() Skip inside cleanup.
    Loc: [qtbase/tests/auto/testlib/selftests/skipcleanup/tst_skipcleanup.cpp(0)]
+PASS   : tst_SkipCleanup::cleanupTestCase()
 Totals: 2 passed, 0 failed, 1 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_SkipCleanup *********
diff --git a/tests/auto/testlib/selftests/expected_skipcleanup.xml b/tests/auto/testlib/selftests/expected_skipcleanup.xml
index 77bf028b12..d839a69d8f 100644
--- a/tests/auto/testlib/selftests/expected_skipcleanup.xml
+++ b/tests/auto/testlib/selftests/expected_skipcleanup.xml
@@ -10,13 +10,13 @@
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="aTestFunction">
-    <Incident type="pass" file="" line="0" />
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/skipcleanup/tst_skipcleanup.cpp" line="0">
+      <Description><![CDATA[Skip inside cleanup.]]></Description>
+    </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="cleanupTestCase">
-    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/skipcleanup/tst_skipcleanup.cpp" line="0">
-      <Description><![CDATA[Skip inside cleanupTestCase.]]></Description>
-    </Incident>
+    <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
   <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_skipcleanuptestcase.junitxml b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.junitxml
new file mode 100644
index 0000000000..6e1e10b2b2
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.junitxml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuite name="tst_SkipCleanupTestCase" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="3" failures="0" errors="0" skipped="1" time="@TEST_DURATION@">
+  <properties>
+    <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
+    <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
+    <property name="QtBuild" value=""/>
+  </properties>
+  <testcase name="initTestCase" classname="tst_SkipCleanupTestCase" time="@TEST_DURATION@"/>
+  <testcase name="aTestFunction" classname="tst_SkipCleanupTestCase" time="@TEST_DURATION@"/>
+  <testcase name="cleanupTestCase" classname="tst_SkipCleanupTestCase" time="@TEST_DURATION@">
+    <skipped message="Skip inside cleanupTestCase."/>
+  </testcase>
+</testsuite>
diff --git a/tests/auto/testlib/selftests/expected_skipcleanuptestcase.lightxml b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.lightxml
new file mode 100644
index 0000000000..29fa9ff602
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.lightxml
@@ -0,0 +1,20 @@
+  <Environment>
+    <QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
+    <QtBuild/>
+    <QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
+  </Environment>
+  <TestFunction name="initTestCase">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="aTestFunction">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="cleanupTestCase">
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/skipcleanuptestcase/tst_skipcleanuptestcase.cpp" line="0">
+      <Description><![CDATA[Skip inside cleanupTestCase.]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_skipcleanuptestcase.tap b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.tap
new file mode 100644
index 0000000000..ee3d720d22
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.tap
@@ -0,0 +1,9 @@
+TAP version 13
+# tst_SkipCleanupTestCase
+ok 1 - initTestCase()
+ok 2 - aTestFunction()
+ok 3 - cleanupTestCase() # SKIP Skip inside cleanupTestCase.
+1..3
+# tests 3
+# pass 2
+# fail 0
diff --git a/tests/auto/testlib/selftests/expected_skipcleanuptestcase.teamcity b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.teamcity
new file mode 100644
index 0000000000..6a7e6bc379
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.teamcity
@@ -0,0 +1,9 @@
+##teamcity[testSuiteStarted name='tst_SkipCleanupTestCase' flowId='tst_SkipCleanupTestCase']
+##teamcity[testStarted name='initTestCase()' flowId='tst_SkipCleanupTestCase']
+##teamcity[testFinished name='initTestCase()' flowId='tst_SkipCleanupTestCase']
+##teamcity[testStarted name='aTestFunction()' flowId='tst_SkipCleanupTestCase']
+##teamcity[testFinished name='aTestFunction()' flowId='tst_SkipCleanupTestCase']
+##teamcity[testStarted name='cleanupTestCase()' flowId='tst_SkipCleanupTestCase']
+##teamcity[testIgnored name='cleanupTestCase()' message='Skip inside cleanupTestCase. |[Loc: qtbase/tests/auto/testlib/selftests/skipcleanuptestcase/tst_skipcleanuptestcase.cpp(0)|]' flowId='tst_SkipCleanupTestCase']
+##teamcity[testFinished name='cleanupTestCase()' flowId='tst_SkipCleanupTestCase']
+##teamcity[testSuiteFinished name='tst_SkipCleanupTestCase' flowId='tst_SkipCleanupTestCase']
diff --git a/tests/auto/testlib/selftests/expected_skipcleanuptestcase.txt b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.txt
new file mode 100644
index 0000000000..4dc50dca7f
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.txt
@@ -0,0 +1,8 @@
+********* Start testing of tst_SkipCleanupTestCase *********
+Config: Using QtTest library
+PASS   : tst_SkipCleanupTestCase::initTestCase()
+PASS   : tst_SkipCleanupTestCase::aTestFunction()
+SKIP   : tst_SkipCleanupTestCase::cleanupTestCase() Skip inside cleanupTestCase.
+   Loc: [qtbase/tests/auto/testlib/selftests/skipcleanuptestcase/tst_skipcleanuptestcase.cpp(0)]
+Totals: 2 passed, 0 failed, 1 skipped, 0 blacklisted, 0ms
+********* Finished testing of tst_SkipCleanupTestCase *********
diff --git a/tests/auto/testlib/selftests/expected_skipcleanuptestcase.xml b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.xml
new file mode 100644
index 0000000000..442d1a69b7
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_skipcleanuptestcase.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<TestCase name="tst_SkipCleanupTestCase">
+  <Environment>
+    <QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
+    <QtBuild/>
+    <QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
+  </Environment>
+  <TestFunction name="initTestCase">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="aTestFunction">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="cleanupTestCase">
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/skipcleanuptestcase/tst_skipcleanuptestcase.cpp" line="0">
+      <Description><![CDATA[Skip inside cleanupTestCase.]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <Duration msecs="0"/>
+</TestCase>
diff --git a/tests/auto/testlib/selftests/failcleanup/CMakeLists.txt b/tests/auto/testlib/selftests/failcleanup/CMakeLists.txt
index e45d1b528b..81997dff89 100644
--- a/tests/auto/testlib/selftests/failcleanup/CMakeLists.txt
+++ b/tests/auto/testlib/selftests/failcleanup/CMakeLists.txt
@@ -1,12 +1,10 @@
-# Generated from failcleanup.pro.
-
 #####################################################################
 ## failcleanup Binary:
 #####################################################################
 
 qt_internal_add_executable(failcleanup
-    NO_INSTALL # special case
-    OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} # special case
+    NO_INSTALL
+    OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     SOURCES
         tst_failcleanup.cpp
     PUBLIC_LIBRARIES
@@ -16,6 +14,4 @@ qt_internal_add_executable(failcleanup
 ## Scopes:
 #####################################################################
 
-# special case begin
 qt_internal_apply_testlib_coverage_options(failcleanup)
-# special case end
diff --git a/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp b/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp
index f0c0af7549..a93c57ccda 100644
--- a/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp
+++ b/tests/auto/testlib/selftests/failcleanup/tst_failcleanup.cpp
@@ -8,7 +8,7 @@ class tst_FailCleanup: public QObject
 Q_OBJECT
 private slots:
     void aTestFunction() const;
-    void cleanupTestCase() const;
+    void cleanup() const;
 };
 
 void tst_FailCleanup::aTestFunction() const
@@ -16,9 +16,9 @@ void tst_FailCleanup::aTestFunction() const
     QVERIFY(true);
 }
 
-void tst_FailCleanup::cleanupTestCase() const
+void tst_FailCleanup::cleanup() const
 {
-    QVERIFY2(false, "Fail inside cleanupTestCase");
+    QVERIFY2(false, "Fail inside cleanup");
 }
 
 QTEST_APPLESS_MAIN(tst_FailCleanup)
diff --git a/tests/auto/testlib/selftests/failcleanuptestcase/CMakeLists.txt b/tests/auto/testlib/selftests/failcleanuptestcase/CMakeLists.txt
new file mode 100644
index 0000000000..173d9c46c3
--- /dev/null
+++ b/tests/auto/testlib/selftests/failcleanuptestcase/CMakeLists.txt
@@ -0,0 +1,17 @@
+#####################################################################
+## failcleanuptestcase Binary:
+#####################################################################
+
+qt_internal_add_executable(failcleanuptestcase
+    NO_INSTALL
+    OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    SOURCES
+        tst_failcleanuptestcase.cpp
+    PUBLIC_LIBRARIES
+        Qt::Test
+)
+
+## Scopes:
+#####################################################################
+
+qt_internal_apply_testlib_coverage_options(failcleanuptestcase)
diff --git a/tests/auto/testlib/selftests/failcleanuptestcase/tst_failcleanuptestcase.cpp b/tests/auto/testlib/selftests/failcleanuptestcase/tst_failcleanuptestcase.cpp
new file mode 100644
index 0000000000..0375122565
--- /dev/null
+++ b/tests/auto/testlib/selftests/failcleanuptestcase/tst_failcleanuptestcase.cpp
@@ -0,0 +1,25 @@
+// Copyright (C) 2016 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include <QTest>
+
+class tst_FailCleanupTestCase: public QObject
+{
+Q_OBJECT
+private slots:
+    void aTestFunction() const;
+    void cleanupTestCase() const;
+};
+
+void tst_FailCleanupTestCase::aTestFunction() const
+{
+    QVERIFY(true);
+}
+
+void tst_FailCleanupTestCase::cleanupTestCase() const
+{
+    QVERIFY2(false, "Fail inside cleanupTestCase");
+}
+
+QTEST_APPLESS_MAIN(tst_FailCleanupTestCase)
+#include "tst_failcleanuptestcase.moc"
diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py
index b1da3fd9fa..987a08b2c1 100755
--- a/tests/auto/testlib/selftests/generate_expected_output.py
+++ b/tests/auto/testlib/selftests/generate_expected_output.py
@@ -32,14 +32,15 @@ TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting',
          'benchlibwalltime', 'blacklisted', 'cmptest', 'commandlinedata',
          'counting', 'crashes', 'datatable', 'datetime', 'deleteLater',
          'deleteLater_noApp', 'differentexec', 'exceptionthrow', 'expectfail',
-         "extendedcompare", 'failcleanup', 'faildatatype', 'failfetchtype', 'failinit',
-         'failinitdata', 'fetchbogus', 'findtestdata', 'float', 'globaldata',
-         'longstring', 'maxwarnings', 'mouse', 'multiexec', 'pairdiagnostics', 'pass',
+         "extendedcompare", 'failcleanup', 'failcleanuptestcase',
+         'faildatatype', 'failfetchtype', 'failinit', 'failinitdata',
+         'fetchbogus', 'findtestdata', 'float', 'globaldata', 'longstring',
+         'maxwarnings', 'mouse', 'multiexec', 'pairdiagnostics', 'pass',
          'printdatatags', 'printdatatagswithglobaltags', 'qexecstringlist',
          'signaldumper', 'silent', 'singleskip', 'skip', 'skipcleanup',
-         'skipinit', 'skipinitdata', 'sleep', 'strcmp', 'subtest', 'testlib',
-         'tuplediagnostics', 'verbose1', 'verbose2', 'verifyexceptionthrown',
-         'warnings', 'watchdog', 'junit', 'keyboard']
+         'skipcleanuptestcase', 'skipinit', 'skipinitdata', 'sleep', 'strcmp',
+         'subtest', 'testlib', 'tuplediagnostics', 'verbose1', 'verbose2',
+         'verifyexceptionthrown', 'warnings', 'watchdog', 'junit', 'keyboard']
 
 
 class Fail (Exception): pass
diff --git a/tests/auto/testlib/selftests/skipcleanup/CMakeLists.txt b/tests/auto/testlib/selftests/skipcleanup/CMakeLists.txt
index 5af361e25a..b977c2ba3e 100644
--- a/tests/auto/testlib/selftests/skipcleanup/CMakeLists.txt
+++ b/tests/auto/testlib/selftests/skipcleanup/CMakeLists.txt
@@ -1,12 +1,10 @@
-# Generated from skipcleanup.pro.
-
 #####################################################################
 ## skipcleanup Binary:
 #####################################################################
 
 qt_internal_add_executable(skipcleanup
-    NO_INSTALL # special case
-    OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} # special case
+    NO_INSTALL
+    OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     SOURCES
         tst_skipcleanup.cpp
     PUBLIC_LIBRARIES
@@ -16,6 +14,4 @@ qt_internal_add_executable(skipcleanup
 ## Scopes:
 #####################################################################
 
-# special case begin
 qt_internal_apply_testlib_coverage_options(skipcleanup)
-# special case end
diff --git a/tests/auto/testlib/selftests/skipcleanup/tst_skipcleanup.cpp b/tests/auto/testlib/selftests/skipcleanup/tst_skipcleanup.cpp
index 1afe9a5c57..568609663a 100644
--- a/tests/auto/testlib/selftests/skipcleanup/tst_skipcleanup.cpp
+++ b/tests/auto/testlib/selftests/skipcleanup/tst_skipcleanup.cpp
@@ -8,7 +8,7 @@ class tst_SkipCleanup: public QObject
 Q_OBJECT
 private slots:
     void aTestFunction() const;
-    void cleanupTestCase() const;
+    void cleanup() const;
 };
 
 void tst_SkipCleanup::aTestFunction() const
@@ -16,9 +16,9 @@ void tst_SkipCleanup::aTestFunction() const
     QVERIFY(true);
 }
 
-void tst_SkipCleanup::cleanupTestCase() const
+void tst_SkipCleanup::cleanup() const
 {
-    QSKIP("Skip inside cleanupTestCase.");
+    QSKIP("Skip inside cleanup.");
 }
 
 QTEST_APPLESS_MAIN(tst_SkipCleanup)
diff --git a/tests/auto/testlib/selftests/skipcleanuptestcase/CMakeLists.txt b/tests/auto/testlib/selftests/skipcleanuptestcase/CMakeLists.txt
new file mode 100644
index 0000000000..1d175144ff
--- /dev/null
+++ b/tests/auto/testlib/selftests/skipcleanuptestcase/CMakeLists.txt
@@ -0,0 +1,17 @@
+#####################################################################
+## skipcleanuptestcase Binary:
+#####################################################################
+
+qt_internal_add_executable(skipcleanuptestcase
+    NO_INSTALL
+    OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    SOURCES
+        tst_skipcleanuptestcase.cpp
+    PUBLIC_LIBRARIES
+        Qt::Test
+)
+
+## Scopes:
+#####################################################################
+
+qt_internal_apply_testlib_coverage_options(skipcleanuptestcase)
diff --git a/tests/auto/testlib/selftests/skipcleanuptestcase/tst_skipcleanuptestcase.cpp b/tests/auto/testlib/selftests/skipcleanuptestcase/tst_skipcleanuptestcase.cpp
new file mode 100644
index 0000000000..89be7ac8bf
--- /dev/null
+++ b/tests/auto/testlib/selftests/skipcleanuptestcase/tst_skipcleanuptestcase.cpp
@@ -0,0 +1,25 @@
+// Copyright (C) 2016 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include <QTest>
+
+class tst_SkipCleanupTestCase: public QObject
+{
+Q_OBJECT
+private slots:
+    void aTestFunction() const;
+    void cleanupTestCase() const;
+};
+
+void tst_SkipCleanupTestCase::aTestFunction() const
+{
+    QVERIFY(true);
+}
+
+void tst_SkipCleanupTestCase::cleanupTestCase() const
+{
+    QSKIP("Skip inside cleanupTestCase.");
+}
+
+QTEST_APPLESS_MAIN(tst_SkipCleanupTestCase)
+#include "tst_skipcleanuptestcase.moc"