Skip early return from test loops during cleanup()

The QTRY_* macros and QTestEventLoop exit early if the test has
resolved; however, in the cleanup phase of a test, even if the test
has failed, these loops should continue as normal.

[ChangeLog][QtTest] During the cleanup() phase of a test, the QTRY_*
macros and QTestEventLoop now ignore the test resolution, in contrast
to when they are used from the test itself, which (since 6.3.0) exits
the loops early if the test has failed.

Pick-to: 6.4 6.3
Fixes: QTBUG-104441
Change-Id: I2673161967cbbc57815155af698a9338ab98a686
Reviewed-by: Jason McDonald <macadder1@gmail.com>

6 files changed, 17 insertions, 113 deletions

diff --git a/tests/auto/testlib/selftests/expected_eventloop.junitxml b/tests/auto/testlib/selftests/expected_eventloop.junitxml
index 15a21427c8..566bc1605e 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.junitxml
+++ b/tests/auto/testlib/selftests/expected_eventloop.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_EventLoop" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="5" failures="3" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_EventLoop" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="5" failures="1" errors="0" skipped="1" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -10,10 +10,8 @@
     <failure type="fail" message="Failing test should still clean up"/>
   </testcase>
   <testcase name="skip" classname="tst_EventLoop" time="@TEST_DURATION@">
-    <failure type="fail" message="&apos;!std::exchange(m_inTestFunction, true)&apos; returned FALSE. (Earlier test failed to clean up)"/>
-  </testcase>
-  <testcase name="pass" classname="tst_EventLoop" time="@TEST_DURATION@">
-    <failure type="fail" message="&apos;!std::exchange(m_inTestFunction, true)&apos; returned FALSE. (Earlier test failed to clean up)"/>
+    <skipped message="Skipping test should still clean up"/>
   </testcase>
+  <testcase name="pass" classname="tst_EventLoop" time="@TEST_DURATION@"/>
   <testcase name="cleanupTestCase" classname="tst_EventLoop" time="@TEST_DURATION@"/>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/expected_eventloop.lightxml b/tests/auto/testlib/selftests/expected_eventloop.lightxml
index 0d67d73f75..40880fde01 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.lightxml
+++ b/tests/auto/testlib/selftests/expected_eventloop.lightxml
@@ -11,27 +11,16 @@
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
       <Description><![CDATA[Failing test should still clean up]]></Description>
     </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
-    </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="skip">
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
-    </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA[Skipping test should still clean up]]></Description>
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="pass">
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
-    </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
-    </Incident>
+    <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="cleanupTestCase">
diff --git a/tests/auto/testlib/selftests/expected_eventloop.tap b/tests/auto/testlib/selftests/expected_eventloop.tap
index 84036e6dec..496a6e636a 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.tap
+++ b/tests/auto/testlib/selftests/expected_eventloop.tap
@@ -8,68 +8,10 @@ not ok 2 - fail()
   file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
   line: 0
   ...
-not ok 2 - fail()
-  ---
-  type: QVERIFY
-  message: QTestEventLoop exited prematurely in cleanup()
-  wanted: true (loop.timeout())
-  found: false (loop.timeout())
-  expected: true (loop.timeout())
-  actual: false (loop.timeout())
-  at: tst_EventLoop::fail() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
-  line: 0
-  ...
-not ok 3 - skip()
-  ---
-  type: QVERIFY
-  message: Earlier test failed to clean up
-  wanted: true (!std::exchange(m_inTestFunction, true))
-  found: false (!std::exchange(m_inTestFunction, true))
-  expected: true (!std::exchange(m_inTestFunction, true))
-  actual: false (!std::exchange(m_inTestFunction, true))
-  at: tst_EventLoop::skip() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
-  line: 0
-  ...
-not ok 3 - skip()
-  ---
-  type: QVERIFY
-  message: QTestEventLoop exited prematurely in cleanup()
-  wanted: true (loop.timeout())
-  found: false (loop.timeout())
-  expected: true (loop.timeout())
-  actual: false (loop.timeout())
-  at: tst_EventLoop::skip() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
-  line: 0
-  ...
-not ok 4 - pass()
-  ---
-  type: QVERIFY
-  message: Earlier test failed to clean up
-  wanted: true (!std::exchange(m_inTestFunction, true))
-  found: false (!std::exchange(m_inTestFunction, true))
-  expected: true (!std::exchange(m_inTestFunction, true))
-  actual: false (!std::exchange(m_inTestFunction, true))
-  at: tst_EventLoop::pass() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
-  line: 0
-  ...
-not ok 4 - pass()
-  ---
-  type: QVERIFY
-  message: QTestEventLoop exited prematurely in cleanup()
-  wanted: true (loop.timeout())
-  found: false (loop.timeout())
-  expected: true (loop.timeout())
-  actual: false (loop.timeout())
-  at: tst_EventLoop::pass() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
-  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
-  line: 0
-  ...
+ok 3 - skip() # SKIP Skipping test should still clean up
+ok 4 - pass()
 ok 5 - cleanupTestCase()
 1..5
 # tests 5
-# pass 2
-# fail 3
+# pass 3
+# fail 1
diff --git a/tests/auto/testlib/selftests/expected_eventloop.teamcity b/tests/auto/testlib/selftests/expected_eventloop.teamcity
index 4c832be52f..a293a20135 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.teamcity
+++ b/tests/auto/testlib/selftests/expected_eventloop.teamcity
@@ -4,17 +4,10 @@
 ##teamcity[testStarted name='fail()' flowId='tst_EventLoop']
 ##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='Failing test should still clean up' flowId='tst_EventLoop']
 ##teamcity[testFinished name='fail()' flowId='tst_EventLoop']
-##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
-##teamcity[testFinished name='fail()' flowId='tst_EventLoop']
 ##teamcity[testStarted name='skip()' flowId='tst_EventLoop']
-##teamcity[testFailed name='skip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'!std::exchange(m_inTestFunction, true)|' returned FALSE. (Earlier test failed to clean up)' flowId='tst_EventLoop']
-##teamcity[testFinished name='skip()' flowId='tst_EventLoop']
-##teamcity[testFailed name='skip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
+##teamcity[testIgnored name='skip()' message='Skipping test should still clean up |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' flowId='tst_EventLoop']
 ##teamcity[testFinished name='skip()' flowId='tst_EventLoop']
 ##teamcity[testStarted name='pass()' flowId='tst_EventLoop']
-##teamcity[testFailed name='pass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'!std::exchange(m_inTestFunction, true)|' returned FALSE. (Earlier test failed to clean up)' flowId='tst_EventLoop']
-##teamcity[testFinished name='pass()' flowId='tst_EventLoop']
-##teamcity[testFailed name='pass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
 ##teamcity[testFinished name='pass()' flowId='tst_EventLoop']
 ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_EventLoop']
 ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_EventLoop']
diff --git a/tests/auto/testlib/selftests/expected_eventloop.txt b/tests/auto/testlib/selftests/expected_eventloop.txt
index b614c876c6..548ef393f6 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.txt
+++ b/tests/auto/testlib/selftests/expected_eventloop.txt
@@ -3,16 +3,9 @@ Config: Using QtTest library
 PASS   : tst_EventLoop::initTestCase()
 FAIL!  : tst_EventLoop::fail() Failing test should still clean up
    Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
-FAIL!  : tst_EventLoop::fail() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
-   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
-FAIL!  : tst_EventLoop::skip() '!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)
-   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
-FAIL!  : tst_EventLoop::skip() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
-   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
-FAIL!  : tst_EventLoop::pass() '!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)
-   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
-FAIL!  : tst_EventLoop::pass() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
+SKIP   : tst_EventLoop::skip() Skipping test should still clean up
    Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
+PASS   : tst_EventLoop::pass()
 PASS   : tst_EventLoop::cleanupTestCase()
-Totals: 2 passed, 3 failed, 0 skipped, 0 blacklisted, 0ms
+Totals: 3 passed, 1 failed, 1 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_EventLoop *********
diff --git a/tests/auto/testlib/selftests/expected_eventloop.xml b/tests/auto/testlib/selftests/expected_eventloop.xml
index 2613a73449..f9d9475666 100644
--- a/tests/auto/testlib/selftests/expected_eventloop.xml
+++ b/tests/auto/testlib/selftests/expected_eventloop.xml
@@ -13,27 +13,16 @@
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
       <Description><![CDATA[Failing test should still clean up]]></Description>
     </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
-    </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="skip">
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
-    </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
+    <Incident type="skip" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA[Skipping test should still clean up]]></Description>
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="pass">
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
-    </Incident>
-    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
-      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
-    </Incident>
+    <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="cleanupTestCase">