Add QTRY_COMPARE_{EQ,NE,LT,LE,GT,GE}_WITH_TIMEOUT()

[ChangeLog][QTestLib] Add QTRY_COMPARE_{EQ,NE,LT,LE,GT,GE}_WITH_TIMEOUT
macros that repeatedly execute QCOMPARE_{EQ,NE,LT,LE,GT,GE} until either
the comparison returns true or the timeout expires. Also add
QTRY_COMPARE_{EQ,NE,LT,LE,GT,GE} macros that simply invoke the
*_WITH_TIMEOUT versions with the usual timeout of five seconds.

Task-number: QTBUG-98873
Change-Id: Ib0d7d1c8c997f442b46acd85da738a8f512cc875
Reviewed-by: Marc Mutz <marc.mutz@qt.io>

7 files changed, 191 insertions, 7 deletions

diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.junitxml b/tests/auto/testlib/selftests/expected_extendedcompare.junitxml
index 8e0959f67e..cbde2e3101 100644
--- a/tests/auto/testlib/selftests/expected_extendedcompare.junitxml
+++ b/tests/auto/testlib/selftests/expected_extendedcompare.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_ExtendedCompare" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="147" failures="73" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_ExtendedCompare" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="148" failures="74" errors="0" skipped="0" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -516,5 +516,11 @@
    Right  (getClassForValue(1).getValuePointer()): MyClass(1) on memory address with index 1]]>
     </failure>
   </testcase>
+  <testcase name="checkComparisonWithTimeout" classname="tst_ExtendedCompare" time="@TEST_DURATION@">
+    <failure type="fail" message="Left value is expected to be less than right value, but is not">
+      <![CDATA[   Left   (c)                         : ClassWithDeferredSetter(1)
+   Right  (ClassWithDeferredSetter(0)): ClassWithDeferredSetter(0)]]>
+    </failure>
+  </testcase>
   <testcase name="cleanupTestCase" classname="tst_ExtendedCompare" time="@TEST_DURATION@"/>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.lightxml b/tests/auto/testlib/selftests/expected_extendedcompare.lightxml
index bdc5d088d2..e59dd6edc7 100644
--- a/tests/auto/testlib/selftests/expected_extendedcompare.lightxml
+++ b/tests/auto/testlib/selftests/expected_extendedcompare.lightxml
@@ -709,6 +709,36 @@
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
+  <TestFunction name="checkComparisonWithTimeout">
+    <Incident type="pass" file="" line="0">
+      <DataTag><![CDATA[EQ]]></DataTag>
+    </Incident>
+    <Incident type="pass" file="" line="0">
+      <DataTag><![CDATA[NE]]></DataTag>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp" line="0">
+      <DataTag><![CDATA[LT]]></DataTag>
+      <Description><![CDATA[Left value is expected to be less than right value, but is not
+   Left   (c)                         : ClassWithDeferredSetter(1)
+   Right  (ClassWithDeferredSetter(0)): ClassWithDeferredSetter(0)]]></Description>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp" line="0">
+      <DataTag><![CDATA[LE]]></DataTag>
+      <Description><![CDATA[Left value is expected to be less than or equal to right value, but is not
+   Left   (c)                          : ClassWithDeferredSetter(1)
+   Right  (ClassWithDeferredSetter(-1)): ClassWithDeferredSetter(-1)]]></Description>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp" line="0">
+      <DataTag><![CDATA[GT]]></DataTag>
+      <Description><![CDATA[Left value is expected to be greater than right value, but is not
+   Left   (c)                         : ClassWithDeferredSetter(1)
+   Right  (ClassWithDeferredSetter(1)): ClassWithDeferredSetter(1)]]></Description>
+    </Incident>
+    <Incident type="pass" file="" line="0">
+      <DataTag><![CDATA[GE]]></DataTag>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
   <TestFunction name="cleanupTestCase">
     <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.tap b/tests/auto/testlib/selftests/expected_extendedcompare.tap
index 66166718a2..31562636fb 100644
--- a/tests/auto/testlib/selftests/expected_extendedcompare.tap
+++ b/tests/auto/testlib/selftests/expected_extendedcompare.tap
@@ -976,8 +976,47 @@ not ok 151 - checkComparisonForTemporaryObjects(GE)
   file: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp
   line: 0
   ...
-ok 152 - cleanupTestCase()
-1..152
-# tests 152
-# pass 77
-# fail 75
+ok 152 - checkComparisonWithTimeout(EQ)
+ok 153 - checkComparisonWithTimeout(NE)
+not ok 154 - checkComparisonWithTimeout(LT)
+  ---
+  type: QCOMPARE_LT
+  message: Left value is expected to be less than right value, but is not
+  wanted: < ClassWithDeferredSetter(0) (ClassWithDeferredSetter(0))
+  found: ClassWithDeferredSetter(1) (c)
+  expected: < ClassWithDeferredSetter(0) (ClassWithDeferredSetter(0))
+  actual: ClassWithDeferredSetter(1) (c)
+  at: tst_ExtendedCompare::checkComparisonWithTimeout() (qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp
+  line: 0
+  ...
+not ok 155 - checkComparisonWithTimeout(LE)
+  ---
+  type: QCOMPARE_LE
+  message: Left value is expected to be less than or equal to right value, but is not
+  wanted: <= ClassWithDeferredSetter(-1) (ClassWithDeferredSetter(-1))
+  found: ClassWithDeferredSetter(1) (c)
+  expected: <= ClassWithDeferredSetter(-1) (ClassWithDeferredSetter(-1))
+  actual: ClassWithDeferredSetter(1) (c)
+  at: tst_ExtendedCompare::checkComparisonWithTimeout() (qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp
+  line: 0
+  ...
+not ok 156 - checkComparisonWithTimeout(GT)
+  ---
+  type: QCOMPARE_GT
+  message: Left value is expected to be greater than right value, but is not
+  wanted: > ClassWithDeferredSetter(1) (ClassWithDeferredSetter(1))
+  found: ClassWithDeferredSetter(1) (c)
+  expected: > ClassWithDeferredSetter(1) (ClassWithDeferredSetter(1))
+  actual: ClassWithDeferredSetter(1) (c)
+  at: tst_ExtendedCompare::checkComparisonWithTimeout() (qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp
+  line: 0
+  ...
+ok 157 - checkComparisonWithTimeout(GE)
+ok 158 - cleanupTestCase()
+1..158
+# tests 158
+# pass 80
+# fail 78
diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.teamcity b/tests/auto/testlib/selftests/expected_extendedcompare.teamcity
index ef8189be33..0f365cd3d0 100644
--- a/tests/auto/testlib/selftests/expected_extendedcompare.teamcity
+++ b/tests/auto/testlib/selftests/expected_extendedcompare.teamcity
@@ -371,6 +371,16 @@
 ##teamcity[testFinished name='checkComparisonForTemporaryObjects()' flowId='tst_ExtendedCompare']
 ##teamcity[testFailed name='checkComparisonForTemporaryObjects()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)|]' details='Left value is expected to be greater than or equal to right value, but is not|n Left (getClassForValue(0).getValuePointer()): MyClass(2) on memory address with index 0|n Right (getClassForValue(1).getValuePointer()): MyClass(1) on memory address with index 1' flowId='tst_ExtendedCompare']
 ##teamcity[testFinished name='checkComparisonForTemporaryObjects()' flowId='tst_ExtendedCompare']
+##teamcity[testStarted name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare']
+##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare']
+##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare']
+##teamcity[testFailed name='checkComparisonWithTimeout()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)|]' details='Left value is expected to be less than right value, but is not|n Left (c) : ClassWithDeferredSetter(1)|n Right (ClassWithDeferredSetter(0)): ClassWithDeferredSetter(0)' flowId='tst_ExtendedCompare']
+##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare']
+##teamcity[testFailed name='checkComparisonWithTimeout()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)|]' details='Left value is expected to be less than or equal to right value, but is not|n Left (c) : ClassWithDeferredSetter(1)|n Right (ClassWithDeferredSetter(-1)): ClassWithDeferredSetter(-1)' flowId='tst_ExtendedCompare']
+##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare']
+##teamcity[testFailed name='checkComparisonWithTimeout()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)|]' details='Left value is expected to be greater than right value, but is not|n Left (c) : ClassWithDeferredSetter(1)|n Right (ClassWithDeferredSetter(1)): ClassWithDeferredSetter(1)' flowId='tst_ExtendedCompare']
+##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare']
+##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare']
 ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_ExtendedCompare']
 ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_ExtendedCompare']
 ##teamcity[testSuiteFinished name='tst_ExtendedCompare' flowId='tst_ExtendedCompare']
diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.txt b/tests/auto/testlib/selftests/expected_extendedcompare.txt
index ac240d4f85..15f3275844 100644
--- a/tests/auto/testlib/selftests/expected_extendedcompare.txt
+++ b/tests/auto/testlib/selftests/expected_extendedcompare.txt
@@ -376,6 +376,21 @@ FAIL!  : tst_ExtendedCompare::checkComparisonForTemporaryObjects(GE) Left value
    Left   (getClassForValue(0).getValuePointer()): MyClass(2) on memory address with index 0
    Right  (getClassForValue(1).getValuePointer()): MyClass(1) on memory address with index 1
    Loc: [qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)]
+PASS   : tst_ExtendedCompare::checkComparisonWithTimeout(EQ)
+PASS   : tst_ExtendedCompare::checkComparisonWithTimeout(NE)
+FAIL!  : tst_ExtendedCompare::checkComparisonWithTimeout(LT) Left value is expected to be less than right value, but is not
+   Left   (c)                         : ClassWithDeferredSetter(1)
+   Right  (ClassWithDeferredSetter(0)): ClassWithDeferredSetter(0)
+   Loc: [qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)]
+FAIL!  : tst_ExtendedCompare::checkComparisonWithTimeout(LE) Left value is expected to be less than or equal to right value, but is not
+   Left   (c)                          : ClassWithDeferredSetter(1)
+   Right  (ClassWithDeferredSetter(-1)): ClassWithDeferredSetter(-1)
+   Loc: [qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)]
+FAIL!  : tst_ExtendedCompare::checkComparisonWithTimeout(GT) Left value is expected to be greater than right value, but is not
+   Left   (c)                         : ClassWithDeferredSetter(1)
+   Right  (ClassWithDeferredSetter(1)): ClassWithDeferredSetter(1)
+   Loc: [qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)]
+PASS   : tst_ExtendedCompare::checkComparisonWithTimeout(GE)
 PASS   : tst_ExtendedCompare::cleanupTestCase()
-Totals: 77 passed, 75 failed, 0 skipped, 0 blacklisted, 0ms
+Totals: 80 passed, 78 failed, 0 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_ExtendedCompare *********
diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.xml b/tests/auto/testlib/selftests/expected_extendedcompare.xml
index 427651a235..7bfca23b02 100644
--- a/tests/auto/testlib/selftests/expected_extendedcompare.xml
+++ b/tests/auto/testlib/selftests/expected_extendedcompare.xml
@@ -711,6 +711,36 @@
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
+  <TestFunction name="checkComparisonWithTimeout">
+    <Incident type="pass" file="" line="0">
+      <DataTag><![CDATA[EQ]]></DataTag>
+    </Incident>
+    <Incident type="pass" file="" line="0">
+      <DataTag><![CDATA[NE]]></DataTag>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp" line="0">
+      <DataTag><![CDATA[LT]]></DataTag>
+      <Description><![CDATA[Left value is expected to be less than right value, but is not
+   Left   (c)                         : ClassWithDeferredSetter(1)
+   Right  (ClassWithDeferredSetter(0)): ClassWithDeferredSetter(0)]]></Description>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp" line="0">
+      <DataTag><![CDATA[LE]]></DataTag>
+      <Description><![CDATA[Left value is expected to be less than or equal to right value, but is not
+   Left   (c)                          : ClassWithDeferredSetter(1)
+   Right  (ClassWithDeferredSetter(-1)): ClassWithDeferredSetter(-1)]]></Description>
+    </Incident>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp" line="0">
+      <DataTag><![CDATA[GT]]></DataTag>
+      <Description><![CDATA[Left value is expected to be greater than right value, but is not
+   Left   (c)                         : ClassWithDeferredSetter(1)
+   Right  (ClassWithDeferredSetter(1)): ClassWithDeferredSetter(1)]]></Description>
+    </Incident>
+    <Incident type="pass" file="" line="0">
+      <DataTag><![CDATA[GE]]></DataTag>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
   <TestFunction name="cleanupTestCase">
     <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp b/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp
index 503729365f..d198d621e1 100644
--- a/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp
+++ b/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
 
 #include <QTest>
+#include <QtCore/qtimer.h>
 
 QT_BEGIN_NAMESPACE
 
@@ -143,6 +144,7 @@ private slots:
     void compareCustomTypes_data();
     void compareCustomTypes();
     void checkComparisonForTemporaryObjects();
+    void checkComparisonWithTimeout();
 };
 
 void tst_ExtendedCompare::initTestCase_data()
@@ -272,6 +274,58 @@ void tst_ExtendedCompare::checkComparisonForTemporaryObjects()
                       getClassForValue(1).getValuePointer());
 }
 
+class ClassWithDeferredSetter : public MyClass
+{
+public:
+    ClassWithDeferredSetter(int value) : MyClass(value) {}
+
+    void setValueDeferred(int value)
+    {
+        QTimer::singleShot(100, [this, value] { setValue(value); });
+    }
+};
+
+namespace QTest {
+
+char *toString(const ClassWithDeferredSetter &val)
+{
+    char *msg = new char[128];
+    qsnprintf(msg, 128, "ClassWithDeferredSetter(%d)", val.value());
+    return msg;
+}
+
+} // namespace QTest
+
+void tst_ExtendedCompare::checkComparisonWithTimeout()
+{
+    QFETCH_GLOBAL(QTest::ComparisonOperation, operation);
+    ClassWithDeferredSetter c(0);
+    c.setValueDeferred(1);
+    switch (operation) {
+    case QTest::ComparisonOperation::Equal:
+        QTRY_COMPARE_EQ_WITH_TIMEOUT(c, ClassWithDeferredSetter(1), 300);
+        break;
+    case QTest::ComparisonOperation::NotEqual:
+        QTRY_COMPARE_NE_WITH_TIMEOUT(c, ClassWithDeferredSetter(0), 300);
+        break;
+    case QTest::ComparisonOperation::LessThan:
+        QTRY_COMPARE_LT_WITH_TIMEOUT(c, ClassWithDeferredSetter(0), 300);
+        break;
+    case QTest::ComparisonOperation::LessThanOrEqual:
+        QTRY_COMPARE_LE_WITH_TIMEOUT(c, ClassWithDeferredSetter(-1), 300);
+        break;
+    case QTest::ComparisonOperation::GreaterThan:
+        QTRY_COMPARE_GT_WITH_TIMEOUT(c, ClassWithDeferredSetter(1), 300);
+        break;
+    case QTest::ComparisonOperation::GreaterThanOrEqual:
+        QTRY_COMPARE_GE_WITH_TIMEOUT(c, ClassWithDeferredSetter(1), 300);
+        break;
+    case QTest::ComparisonOperation::CustomCompare:
+        QFAIL("Unexpected comparison operation");
+        break;
+    }
+}
+
 QT_END_NAMESPACE
 
 QTEST_MAIN(tst_ExtendedCompare)