Add some testing of QTestEventLoop

Fairly minimal for now, just enough to verify a bug and serve as the
sign of success when it's fixed. Tests fail in ways they shouldn't,
for now; see expected_eventloop.* for details, notably "Earlier test
failed to clean up" messages.

Pick-to: 6.4 6.3
Task-number: QTBUG-104441
Change-Id: I59be4aa5f21fed23b19a0593a8c2f6c9956507df
Reviewed-by: Jason McDonald <macadder1@gmail.com>

10 files changed, 333 insertions, 2 deletions

diff --git a/tests/auto/testlib/selftests/CMakeLists.txt b/tests/auto/testlib/selftests/CMakeLists.txt
index f98d654e33..a38465484c 100644
--- a/tests/auto/testlib/selftests/CMakeLists.txt
+++ b/tests/auto/testlib/selftests/CMakeLists.txt
@@ -59,6 +59,7 @@ set(subprograms
     deleteLater
     deleteLater_noApp
     differentexec
+    eventloop
     exceptionthrow
     expectfail
     extendedcompare
diff --git a/tests/auto/testlib/selftests/eventloop/CMakeLists.txt b/tests/auto/testlib/selftests/eventloop/CMakeLists.txt
new file mode 100644
index 0000000000..f1fa9a788c
--- /dev/null
+++ b/tests/auto/testlib/selftests/eventloop/CMakeLists.txt
@@ -0,0 +1,16 @@
+#####################################################################
+## eventloop Binary:
+#####################################################################
+
+qt_internal_add_executable(eventloop
+    NO_INSTALL
+    OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    SOURCES
+        tst_eventloop.cpp
+    PUBLIC_LIBRARIES
+        Qt::Test
+)
+
+## Scopes:
+#####################################################################
+qt_internal_apply_testlib_coverage_options(eventloop)
diff --git a/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp b/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
new file mode 100644
index 0000000000..b5e8efce82
--- /dev/null
+++ b/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
@@ -0,0 +1,96 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include <QTest>
+#include <QTestEventLoop>
+#include <QtCore/QTimer>
+
+// Tests for QTestEventLoop (and some QTRY_* details)
+class tst_EventLoop: public QObject
+{
+Q_OBJECT
+
+    bool m_inTestFunction = false;
+private slots:
+    void cleanup();
+    void fail();
+    void skip();
+    void pass();
+};
+
+class DeferredFlag
+{
+    bool m_flag;
+public:
+    explicit DeferredFlag(bool initial = false) : m_flag(initial)
+    {
+        if (!initial)
+            QTimer::singleShot(50, [this] { m_flag = true; });
+    }
+    explicit operator bool() const { return m_flag; }
+    bool operator!() const { return !m_flag; }
+    friend bool operator==(DeferredFlag a, DeferredFlag b) { return bool(a) == bool(b); }
+};
+
+char *toString(DeferredFlag val)
+{
+    return qstrdup(bool(val) ? "DeferredFlag(true)" : "DeferredFlag(false)");
+}
+
+void tst_EventLoop::cleanup()
+{
+    // QTBUG-104441: looping didn't happen in cleanup() if test failed or skipped.
+    {
+        DeferredFlag flag;
+        auto &loop = QTestEventLoop::instance();
+        loop.enterLoopMSecs(100);
+        QVERIFY2(loop.timeout(), "QTestEventLoop exited prematurely in cleanup()");
+        QVERIFY(flag);
+    }
+    {
+        DeferredFlag flag;
+        QTRY_VERIFY2(flag, "QTRY_* loop exited prematurely in cleanup()");
+    }
+
+    m_inTestFunction = false;
+}
+
+void tst_EventLoop::fail()
+{
+    QVERIFY2(!std::exchange(m_inTestFunction, true), "Earlier test failed to clean up");
+    QFAIL("Failing test should still clean up");
+}
+
+void tst_EventLoop::skip()
+{
+    QVERIFY2(!std::exchange(m_inTestFunction, true), "Earlier test failed to clean up");
+    QSKIP("Skipping test should still clean up");
+}
+
+void tst_EventLoop::pass()
+{
+    QVERIFY2(!std::exchange(m_inTestFunction, true), "Earlier test failed to clean up");
+    {
+        DeferredFlag flag;
+        auto &loop = QTestEventLoop::instance();
+        loop.enterLoopMSecs(100);
+        QVERIFY(loop.timeout());
+        QVERIFY(flag);
+    }
+    {
+        DeferredFlag flag;
+        QTRY_VERIFY(flag);
+    }
+    DeferredFlag flag;
+    QTestEventLoop loop(this);
+    QVERIFY(!flag);
+    loop.enterLoopMSecs(1);
+    QVERIFY(loop.timeout());
+    QVERIFY(!flag);
+    loop.enterLoopMSecs(100);
+    QVERIFY(loop.timeout());
+    QVERIFY(flag);
+}
+
+QTEST_MAIN(tst_EventLoop)
+#include "tst_eventloop.moc"
diff --git a/tests/auto/testlib/selftests/expected_eventloop.junitxml b/tests/auto/testlib/selftests/expected_eventloop.junitxml
new file mode 100644
index 0000000000..15a21427c8
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_eventloop.junitxml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuite name="tst_EventLoop" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="5" failures="3" errors="0" skipped="0" time="@TEST_DURATION@">
+  <properties>
+    <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
+    <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
+    <property name="QtBuild" value=""/>
+  </properties>
+  <testcase name="initTestCase" classname="tst_EventLoop" time="@TEST_DURATION@"/>
+  <testcase name="fail" classname="tst_EventLoop" time="@TEST_DURATION@">
+    <failure type="fail" message="Failing test should still clean up"/>
+  </testcase>
+  <testcase name="skip" classname="tst_EventLoop" time="@TEST_DURATION@">
+    <failure type="fail" message="&apos;!std::exchange(m_inTestFunction, true)&apos; returned FALSE. (Earlier test failed to clean up)"/>
+  </testcase>
+  <testcase name="pass" classname="tst_EventLoop" time="@TEST_DURATION@">
+    <failure type="fail" message="&apos;!std::exchange(m_inTestFunction, true)&apos; returned FALSE. (Earlier test failed to clean up)"/>
+  </testcase>
+  <testcase name="cleanupTestCase" classname="tst_EventLoop" time="@TEST_DURATION@"/>
+</testsuite>
diff --git a/tests/auto/testlib/selftests/expected_eventloop.lightxml b/tests/auto/testlib/selftests/expected_eventloop.lightxml
new file mode 100644
index 0000000000..0d67d73f75
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_eventloop.lightxml
@@ -0,0 +1,41 @@
+  <Environment>
+    <QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
+    <QtBuild/>
+    <QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
+  </Environment>
+  <TestFunction name="initTestCase">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="fail">
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA[Failing test should still clean up]]></Description>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="skip">
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="pass">
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="cleanupTestCase">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_eventloop.tap b/tests/auto/testlib/selftests/expected_eventloop.tap
new file mode 100644
index 0000000000..85f48a13a1
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_eventloop.tap
@@ -0,0 +1,75 @@
+TAP version 13
+# tst_EventLoop
+ok 1 - initTestCase()
+not ok 2 - fail()
+  ---
+  # Failing test should still clean up
+  at: tst_EventLoop::fail() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
+  line: 0
+  ...
+not ok 2 - fail()
+  ---
+  type: QVERIFY
+  message: )
+  wanted: true (loop.timeout())
+  found: false (loop.timeout())
+  expected: true (loop.timeout())
+  actual: false (loop.timeout())
+  at: tst_EventLoop::fail() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
+  line: 0
+  ...
+not ok 3 - skip()
+  ---
+  type: QVERIFY
+  message: Earlier test failed to clean up
+  wanted: true (!std::exchange(m_inTestFunction, true))
+  found: false (!std::exchange(m_inTestFunction, true))
+  expected: true (!std::exchange(m_inTestFunction, true))
+  actual: false (!std::exchange(m_inTestFunction, true))
+  at: tst_EventLoop::skip() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
+  line: 0
+  ...
+not ok 3 - skip()
+  ---
+  type: QVERIFY
+  message: )
+  wanted: true (loop.timeout())
+  found: false (loop.timeout())
+  expected: true (loop.timeout())
+  actual: false (loop.timeout())
+  at: tst_EventLoop::skip() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
+  line: 0
+  ...
+not ok 4 - pass()
+  ---
+  type: QVERIFY
+  message: Earlier test failed to clean up
+  wanted: true (!std::exchange(m_inTestFunction, true))
+  found: false (!std::exchange(m_inTestFunction, true))
+  expected: true (!std::exchange(m_inTestFunction, true))
+  actual: false (!std::exchange(m_inTestFunction, true))
+  at: tst_EventLoop::pass() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
+  line: 0
+  ...
+not ok 4 - pass()
+  ---
+  type: QVERIFY
+  message: )
+  wanted: true (loop.timeout())
+  found: false (loop.timeout())
+  expected: true (loop.timeout())
+  actual: false (loop.timeout())
+  at: tst_EventLoop::pass() (qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp
+  line: 0
+  ...
+ok 5 - cleanupTestCase()
+1..5
+# tests 5
+# pass 2
+# fail 3
diff --git a/tests/auto/testlib/selftests/expected_eventloop.teamcity b/tests/auto/testlib/selftests/expected_eventloop.teamcity
new file mode 100644
index 0000000000..4c832be52f
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_eventloop.teamcity
@@ -0,0 +1,21 @@
+##teamcity[testSuiteStarted name='tst_EventLoop' flowId='tst_EventLoop']
+##teamcity[testStarted name='initTestCase()' flowId='tst_EventLoop']
+##teamcity[testFinished name='initTestCase()' flowId='tst_EventLoop']
+##teamcity[testStarted name='fail()' flowId='tst_EventLoop']
+##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='Failing test should still clean up' flowId='tst_EventLoop']
+##teamcity[testFinished name='fail()' flowId='tst_EventLoop']
+##teamcity[testFailed name='fail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
+##teamcity[testFinished name='fail()' flowId='tst_EventLoop']
+##teamcity[testStarted name='skip()' flowId='tst_EventLoop']
+##teamcity[testFailed name='skip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'!std::exchange(m_inTestFunction, true)|' returned FALSE. (Earlier test failed to clean up)' flowId='tst_EventLoop']
+##teamcity[testFinished name='skip()' flowId='tst_EventLoop']
+##teamcity[testFailed name='skip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
+##teamcity[testFinished name='skip()' flowId='tst_EventLoop']
+##teamcity[testStarted name='pass()' flowId='tst_EventLoop']
+##teamcity[testFailed name='pass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'!std::exchange(m_inTestFunction, true)|' returned FALSE. (Earlier test failed to clean up)' flowId='tst_EventLoop']
+##teamcity[testFinished name='pass()' flowId='tst_EventLoop']
+##teamcity[testFailed name='pass()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)|]' details='|'loop.timeout()|' returned FALSE. (QTestEventLoop exited prematurely in cleanup())' flowId='tst_EventLoop']
+##teamcity[testFinished name='pass()' flowId='tst_EventLoop']
+##teamcity[testStarted name='cleanupTestCase()' flowId='tst_EventLoop']
+##teamcity[testFinished name='cleanupTestCase()' flowId='tst_EventLoop']
+##teamcity[testSuiteFinished name='tst_EventLoop' flowId='tst_EventLoop']
diff --git a/tests/auto/testlib/selftests/expected_eventloop.txt b/tests/auto/testlib/selftests/expected_eventloop.txt
new file mode 100644
index 0000000000..b614c876c6
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_eventloop.txt
@@ -0,0 +1,18 @@
+********* Start testing of tst_EventLoop *********
+Config: Using QtTest library
+PASS   : tst_EventLoop::initTestCase()
+FAIL!  : tst_EventLoop::fail() Failing test should still clean up
+   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
+FAIL!  : tst_EventLoop::fail() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
+   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
+FAIL!  : tst_EventLoop::skip() '!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)
+   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
+FAIL!  : tst_EventLoop::skip() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
+   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
+FAIL!  : tst_EventLoop::pass() '!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)
+   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
+FAIL!  : tst_EventLoop::pass() 'loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())
+   Loc: [qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp(0)]
+PASS   : tst_EventLoop::cleanupTestCase()
+Totals: 2 passed, 3 failed, 0 skipped, 0 blacklisted, 0ms
+********* Finished testing of tst_EventLoop *********
diff --git a/tests/auto/testlib/selftests/expected_eventloop.xml b/tests/auto/testlib/selftests/expected_eventloop.xml
new file mode 100644
index 0000000000..2613a73449
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_eventloop.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<TestCase name="tst_EventLoop">
+  <Environment>
+    <QtVersion>@INSERT_QT_VERSION_HERE@</QtVersion>
+    <QtBuild/>
+    <QTestVersion>@INSERT_QT_VERSION_HERE@</QTestVersion>
+  </Environment>
+  <TestFunction name="initTestCase">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="fail">
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA[Failing test should still clean up]]></Description>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="skip">
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="pass">
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA['!std::exchange(m_inTestFunction, true)' returned FALSE. (Earlier test failed to clean up)]]></Description>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/eventloop/tst_eventloop.cpp" line="0">
+      <Description><![CDATA['loop.timeout()' returned FALSE. (QTestEventLoop exited prematurely in cleanup())]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
+  <TestFunction name="cleanupTestCase">
+    <Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+  </TestFunction>
+  <Duration msecs="0"/>
+</TestCase>
diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py
index 987a08b2c1..8f6134725b 100755
--- a/tests/auto/testlib/selftests/generate_expected_output.py
+++ b/tests/auto/testlib/selftests/generate_expected_output.py
@@ -31,8 +31,8 @@ TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting',
          'benchlibeventcounter', 'benchliboptions', 'benchlibtickcounter',
          'benchlibwalltime', 'blacklisted', 'cmptest', 'commandlinedata',
          'counting', 'crashes', 'datatable', 'datetime', 'deleteLater',
-         'deleteLater_noApp', 'differentexec', 'exceptionthrow', 'expectfail',
-         "extendedcompare", 'failcleanup', 'failcleanuptestcase',
+         'deleteLater_noApp', 'differentexec', 'eventloop', 'exceptionthrow',
+         'expectfail', "extendedcompare", 'failcleanup', 'failcleanuptestcase',
          'faildatatype', 'failfetchtype', 'failinit', 'failinitdata',
          'fetchbogus', 'findtestdata', 'float', 'globaldata', 'longstring',
          'maxwarnings', 'mouse', 'multiexec', 'pairdiagnostics', 'pass',