QmlTest: Prevent interleaved execution of TestCases

Run all TestCase instances on a timer in a singleton. This way, even if
one TestCase manages to trigger a different one while it's still
running, the new TestCase won't be executed until the old one is
finished.

Fixes: QTBUG-98350
Change-Id: I1797b5487f2c70fd2edfdbf8cf4c43a6353b12c8
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

4 files changed, 55 insertions, 11 deletions

diff --git a/src/qmltest/CMakeLists.txt b/src/qmltest/CMakeLists.txt
index 69698ad72d..8fa3ebbc54 100644
--- a/src/qmltest/CMakeLists.txt
+++ b/src/qmltest/CMakeLists.txt
@@ -9,6 +9,10 @@ set_source_files_properties(testlogger.js PROPERTIES
     QT_QML_SKIP_QMLDIR_ENTRY TRUE
 )
 
+set_source_files_properties(TestSchedule.qml PROPERTIES
+    QT_QML_SINGLETON_TYPE TRUE
+)
+
 qt_internal_add_qml_module(QuickTest
     URI "QtTest"
     VERSION "${PROJECT_VERSION}"
@@ -25,8 +29,9 @@ qt_internal_add_qml_module(QuickTest
         quicktestresult.cpp quicktestresult_p.h
         quicktestutil.cpp quicktestutil_p.h
     QML_FILES
-        TestCase.qml
         SignalSpy.qml
+        TestCase.qml
+        TestSchedule.qml
         testlogger.js
     DEFINES
         QT_NO_FOREACH
diff --git a/src/qmltest/TestCase.qml b/src/qmltest/TestCase.qml
index 2fcf9611c0..d86478eaf9 100644
--- a/src/qmltest/TestCase.qml
+++ b/src/qmltest/TestCase.qml
@@ -1849,7 +1849,11 @@ Item {
 
     /*! \internal */
     function qtest_run() {
-        if (TestLogger.log_start_test()) {
+        if (!when || completed || running || !qtest_componentCompleted)
+            return;
+
+        verify(TestLogger.log_can_start_test(qtest_testId))
+        if (TestLogger.log_start_test(qtest_testId)) {
             qtest_results.reset()
             qtest_results.testCaseName = name
             qtest_results.startLogging()
@@ -2020,8 +2024,8 @@ Item {
     onWhenChanged: {
         if (when != qtest_prevWhen) {
             qtest_prevWhen = when
-            if (when && !completed && !running && qtest_componentCompleted)
-                qtest_run()
+            if (when)
+                TestSchedule.testCases.push(testCase)
         }
     }
 
@@ -2041,7 +2045,7 @@ Item {
         if (optional)
             TestLogger.log_optional_test(qtest_testId)
         qtest_prevWhen = when
-        if (when && !completed && !running)
-            qtest_run()
+        if (when)
+            TestSchedule.testCases.push(testCase)
     }
 }
diff --git a/src/qmltest/TestSchedule.qml b/src/qmltest/TestSchedule.qml
new file mode 100644
index 0000000000..a2d291f80d
--- /dev/null
+++ b/src/qmltest/TestSchedule.qml
@@ -0,0 +1,29 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+pragma Singleton
+import QtQml
+
+Timer {
+    property list<QtObject> testCases
+    property QtObject currentTest: null
+
+    running: testCases.length > 0 && !currentTest
+    interval: 1
+    repeat: true
+
+    onTriggered: {
+        if (currentTest) {
+            console.error("Interleaved test execution detected. This shouldn't happen")
+            return;
+        }
+
+        try {
+            currentTest = testCases.shift()
+            currentTest.qtest_run()
+        } finally {
+            currentTest = null
+        }
+    }
+
+}
diff --git a/src/qmltest/testlogger.js b/src/qmltest/testlogger.js
index b001df3c58..dc28bf6c73 100644
--- a/src/qmltest/testlogger.js
+++ b/src/qmltest/testlogger.js
@@ -9,7 +9,7 @@ function log_init_results()
 {
     if (!testResults) {
         testResults = {
-            reportedStart: false,
+            runningTest: -1,
             nextId: 0,
             testCases: []
         }
@@ -36,16 +36,21 @@ function log_mandatory_test(testId)
 {
     log_init_results()
     var index = testResults.testCases.indexOf(testId)
-    if (index == -1)
+    if (index === -1)
         testResults.testCases.push(testId)
 }
 
-function log_start_test()
+function log_can_start_test(testId)
+{
+    return !testResults || testResults.runningTest === -1 || testResults.runningTest === testId;
+}
+
+function log_start_test(testId)
 {
     log_init_results()
-    if (testResults.reportedStart)
+    if (testResults.runningTest === testId)
         return false
-    testResults.reportedStart = true
+    testResults.runningTest = testId
     return true
 }
 
@@ -54,5 +59,6 @@ function log_complete_test(testId)
     var index = testResults.testCases.indexOf(testId)
     if (index >= 0)
         testResults.testCases.splice(index, 1)
+    testResults.runningTest = -1
     return testResults.testCases.length > 0
 }