Test QTRY_COMPARE() and expand testing of QTRY_VERIFY*()

In the process, simplify the latter while adding some actual
time-variation for the QTRY_* loop to navigate round - based on the
extendedcompare test's ClassWithDeferredSetter. Testing remains
primitive, but is at least a bit more thorough.

Pick-to: 6.4
Change-Id: I40be8fb485f3f18f0a4f4bc62ad36cccac691979
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Mitch Curtis <mitch.curtis@qt.io>
Reviewed-by: Jason McDonald <macadder1@gmail.com>

7 files changed, 180 insertions, 36 deletions

diff --git a/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp b/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
index 94c04157f2..072324880e 100644
--- a/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
+++ b/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
@@ -1,9 +1,9 @@
 // Copyright (C) 2016 The Qt Company Ltd.
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
 
-
-#include <QtCore/QCoreApplication>
 #include <QTest>
+#include <QtCore/QCoreApplication>
+#include <QtCore/QTimer>
 #ifdef QT_GUI_LIB
 #include <QtGui/QColor>
 #include <QtGui/QImage>
@@ -139,6 +139,7 @@ private slots:
     void compareQVector3D();
     void compareQVector4D();
 #endif
+    void tryCompare();
     void verify();
     void verify2();
     void tryVerify();
@@ -649,16 +650,88 @@ void tst_Cmptest::verify2()
     QVERIFY2(opaqueFunc() < 2, QByteArray::number(opaqueFunc()).constData());
 }
 
+class DeferredFlag : public QObject // Can't be const.
+{
+    Q_OBJECT
+    bool m_flag;
+public:
+    // A boolean that either starts out true or decays to true after 50 ms.
+    // However, that decay will only happen when the event loop is run.
+    explicit DeferredFlag(bool initial = false) : m_flag(initial)
+    {
+        if (!initial)
+            QTimer::singleShot(50, this, &DeferredFlag::onTimeOut);
+    }
+    explicit operator bool() const { return m_flag; }
+    bool operator!() const { return !m_flag; }
+    friend bool operator==(const DeferredFlag &a, const DeferredFlag &b)
+    {
+        return bool(a) == bool(b);
+    }
+public slots:
+    void onTimeOut() { m_flag = true; }
+};
+
+char *toString(const DeferredFlag &val)
+{
+    return qstrdup(bool(val) ? "DeferredFlag(true)" : "DeferredFlag(false)");
+}
+
+void tst_Cmptest::tryCompare()
+{
+    /* Note that expected values given as DeferredFlag() shall be re-evaluated
+       each time the comparison is checked, hence supply a fresh false instance,
+       that'll be discarded before it has a chance to decay, hence only compare
+       equal to a false instance. Do not replace them with a local variable
+       initialized to false, as it would (of course) decay.
+    */
+    DeferredFlag trueAlready(true);
+    {
+        DeferredFlag c;
+        // QTRY should check before looping, so be equal to the fresh false immediately.
+        QTRY_COMPARE(c, DeferredFlag());
+        // Given time, it'll end up equal to a true one.
+        QTRY_COMPARE(c, trueAlready);
+    }
+    {
+        DeferredFlag c;
+        QTRY_COMPARE_WITH_TIMEOUT(c, DeferredFlag(), 300);
+        QVERIFY(!c); // Instantly equal, so succeeded without delay.
+        QTRY_COMPARE_WITH_TIMEOUT(c, trueAlready, 200);
+        qInfo("Should now time out and fail");
+        QTRY_COMPARE_WITH_TIMEOUT(c, DeferredFlag(), 200);
+    }
+}
+
 void tst_Cmptest::tryVerify()
 {
-    QTRY_VERIFY(opaqueFunc() > 2);
-    QTRY_VERIFY_WITH_TIMEOUT(opaqueFunc() < 2, 1);
+    {
+        DeferredFlag c;
+        QTRY_VERIFY(!c);
+        QTRY_VERIFY(c);
+    }
+    {
+        DeferredFlag c;
+        QTRY_VERIFY_WITH_TIMEOUT(!c, 300);
+        QTRY_VERIFY_WITH_TIMEOUT(c, 200);
+        qInfo("Should now time out and fail");
+        QTRY_VERIFY_WITH_TIMEOUT(!c, 200);
+    }
 }
 
 void tst_Cmptest::tryVerify2()
 {
-    QTRY_VERIFY2(opaqueFunc() > 2, QByteArray::number(opaqueFunc()).constData());
-    QTRY_VERIFY2_WITH_TIMEOUT(opaqueFunc() < 2, QByteArray::number(opaqueFunc()).constData(), 1);
+    {
+        DeferredFlag c;
+        QTRY_VERIFY2(!c, "Failed to check before looping");
+        QTRY_VERIFY2(c, "Failed to trigger single-shot");
+    }
+    {
+        DeferredFlag c;
+        QTRY_VERIFY2_WITH_TIMEOUT(!c, "Failed to check before looping", 300);
+        QTRY_VERIFY2_WITH_TIMEOUT(c, "Failed to trigger single-shot", 200);
+        QTRY_VERIFY2_WITH_TIMEOUT(!c, "Should time out and fail", 200);
+    }
 }
 
 void tst_Cmptest::verifyExplicitOperatorBool()
diff --git a/tests/auto/testlib/selftests/expected_cmptest.junitxml b/tests/auto/testlib/selftests/expected_cmptest.junitxml
index f71b8f0a19..75ec61ba99 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.junitxml
+++ b/tests/auto/testlib/selftests/expected_cmptest.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Cmptest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="69" failures="48" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_Cmptest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="70" failures="49" errors="0" skipped="0" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -282,6 +282,15 @@
    Expected (v4b): QVector4D(1, 3, 3, 4)]]>
     </failure>
   </testcase>
+  <testcase name="tryCompare" classname="tst_Cmptest" time="@TEST_DURATION@">
+    <failure type="fail" message="Compared values are not the same">
+      <![CDATA[   Actual   (c)             : DeferredFlag(true)
+   Expected (DeferredFlag()): DeferredFlag(false)]]>
+    </failure>
+    <system-out>
+      <![CDATA[Should now time out and fail]]>
+    </system-out>
+  </testcase>
   <testcase name="verify" classname="tst_Cmptest" time="@TEST_DURATION@">
     <failure type="fail" message="&apos;opaqueFunc() &lt; 2&apos; returned FALSE. ()"/>
   </testcase>
@@ -289,10 +298,13 @@
     <failure type="fail" message="&apos;opaqueFunc() &lt; 2&apos; returned FALSE. (42)"/>
   </testcase>
   <testcase name="tryVerify" classname="tst_Cmptest" time="@TEST_DURATION@">
-    <failure type="fail" message="&apos;opaqueFunc() &lt; 2&apos; returned FALSE. ()"/>
+    <failure type="fail" message="&apos;!c&apos; returned FALSE. ()"/>
+    <system-out>
+      <![CDATA[Should now time out and fail]]>
+    </system-out>
   </testcase>
   <testcase name="tryVerify2" classname="tst_Cmptest" time="@TEST_DURATION@">
-    <failure type="fail" message="&apos;opaqueFunc() &lt; 2&apos; returned FALSE. (42)"/>
+    <failure type="fail" message="&apos;!c&apos; returned FALSE. (Should time out and fail)"/>
   </testcase>
   <testcase name="verifyExplicitOperatorBool" classname="tst_Cmptest" time="@TEST_DURATION@"/>
   <testcase name="cleanupTestCase" classname="tst_Cmptest" time="@TEST_DURATION@"/>
diff --git a/tests/auto/testlib/selftests/expected_cmptest.lightxml b/tests/auto/testlib/selftests/expected_cmptest.lightxml
index 3cb492361b..351fad849c 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.lightxml
+++ b/tests/auto/testlib/selftests/expected_cmptest.lightxml
@@ -377,6 +377,17 @@
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
+  <TestFunction name="tryCompare">
+  <Message type="qinfo" file="" line="0">
+    <Description><![CDATA[Should now time out and fail]]></Description>
+  </Message>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
+      <Description><![CDATA[Compared values are not the same
+   Actual   (c)             : DeferredFlag(true)
+   Expected (DeferredFlag()): DeferredFlag(false)]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
   <TestFunction name="verify">
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
       <Description><![CDATA['opaqueFunc() < 2' returned FALSE. ()]]></Description>
@@ -390,14 +401,17 @@
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="tryVerify">
+  <Message type="qinfo" file="" line="0">
+    <Description><![CDATA[Should now time out and fail]]></Description>
+  </Message>
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
-      <Description><![CDATA['opaqueFunc() < 2' returned FALSE. ()]]></Description>
+      <Description><![CDATA['!c' returned FALSE. ()]]></Description>
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="tryVerify2">
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
-      <Description><![CDATA['opaqueFunc() < 2' returned FALSE. (42)]]></Description>
+      <Description><![CDATA['!c' returned FALSE. (Should time out and fail)]]></Description>
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_cmptest.tap b/tests/auto/testlib/selftests/expected_cmptest.tap
index da7f21ab70..19c90bcb93 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.tap
+++ b/tests/auto/testlib/selftests/expected_cmptest.tap
@@ -516,7 +516,23 @@ not ok 63 - compareQVector4D()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 64 - verify()
+not ok 64 - tryCompare()
+  ---
+  type: QCOMPARE
+  message: Compared values are not the same
+  wanted: DeferredFlag(false) (DeferredFlag())
+  found: DeferredFlag(true) (c)
+  expected: DeferredFlag(false) (DeferredFlag())
+  actual: DeferredFlag(true) (c)
+  at: tst_Cmptest::tryCompare() (qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
+  line: 0
+  extensions:
+    messages:
+    - severity: info
+      message: Should now time out and fail
+  ...
+not ok 65 - verify()
   ---
   type: QVERIFY
   message: Verification failed
@@ -528,7 +544,7 @@ not ok 64 - verify()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 65 - verify2()
+not ok 66 - verify2()
   ---
   type: QVERIFY
   message: 42
@@ -540,33 +556,37 @@ not ok 65 - verify2()
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-not ok 66 - tryVerify()
+not ok 67 - tryVerify()
   ---
   type: QVERIFY
   message: Verification failed
-  wanted: true (opaqueFunc() < 2)
-  found: false (opaqueFunc() < 2)
-  expected: true (opaqueFunc() < 2)
-  actual: false (opaqueFunc() < 2)
+  wanted: true (!c)
+  found: false (!c)
+  expected: true (!c)
+  actual: false (!c)
   at: tst_Cmptest::tryVerify() (qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
+  extensions:
+    messages:
+    - severity: info
+      message: Should now time out and fail
   ...
-not ok 67 - tryVerify2()
+not ok 68 - tryVerify2()
   ---
   type: QVERIFY
-  message: 42
-  wanted: true (opaqueFunc() < 2)
-  found: false (opaqueFunc() < 2)
-  expected: true (opaqueFunc() < 2)
-  actual: false (opaqueFunc() < 2)
+  message: Should time out and fail
+  wanted: true (!c)
+  found: false (!c)
+  expected: true (!c)
+  actual: false (!c)
   at: tst_Cmptest::tryVerify2() (qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp:0)
   file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp
   line: 0
   ...
-ok 68 - verifyExplicitOperatorBool()
-ok 69 - cleanupTestCase()
-1..69
-# tests 69
+ok 69 - verifyExplicitOperatorBool()
+ok 70 - cleanupTestCase()
+1..70
+# tests 70
 # pass 21
-# fail 48
+# fail 49
diff --git a/tests/auto/testlib/selftests/expected_cmptest.teamcity b/tests/auto/testlib/selftests/expected_cmptest.teamcity
index 83e007413e..5f1e8374b9 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.teamcity
+++ b/tests/auto/testlib/selftests/expected_cmptest.teamcity
@@ -169,6 +169,10 @@
 ##teamcity[testStarted name='compareQVector4D()' flowId='tst_Cmptest']
 ##teamcity[testFailed name='compareQVector4D()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='Compared values are not the same|n Actual (v4a): QVector4D(1, 2, 3, 4)|n Expected (v4b): QVector4D(1, 3, 3, 4)' flowId='tst_Cmptest']
 ##teamcity[testFinished name='compareQVector4D()' flowId='tst_Cmptest']
+##teamcity[testStarted name='tryCompare()' flowId='tst_Cmptest']
+##teamcity[testFailed name='tryCompare()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='Compared values are not the same|n Actual (c) : DeferredFlag(true)|n Expected (DeferredFlag()): DeferredFlag(false)' flowId='tst_Cmptest']
+##teamcity[testStdOut name='tryCompare()' out='QINFO: Should now time out and fail' flowId='tst_Cmptest']
+##teamcity[testFinished name='tryCompare()' flowId='tst_Cmptest']
 ##teamcity[testStarted name='verify()' flowId='tst_Cmptest']
 ##teamcity[testFailed name='verify()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'opaqueFunc() < 2|' returned FALSE. ()' flowId='tst_Cmptest']
 ##teamcity[testFinished name='verify()' flowId='tst_Cmptest']
@@ -176,10 +180,11 @@
 ##teamcity[testFailed name='verify2()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'opaqueFunc() < 2|' returned FALSE. (42)' flowId='tst_Cmptest']
 ##teamcity[testFinished name='verify2()' flowId='tst_Cmptest']
 ##teamcity[testStarted name='tryVerify()' flowId='tst_Cmptest']
-##teamcity[testFailed name='tryVerify()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'opaqueFunc() < 2|' returned FALSE. ()' flowId='tst_Cmptest']
+##teamcity[testFailed name='tryVerify()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'!c|' returned FALSE. ()' flowId='tst_Cmptest']
+##teamcity[testStdOut name='tryVerify()' out='QINFO: Should now time out and fail' flowId='tst_Cmptest']
 ##teamcity[testFinished name='tryVerify()' flowId='tst_Cmptest']
 ##teamcity[testStarted name='tryVerify2()' flowId='tst_Cmptest']
-##teamcity[testFailed name='tryVerify2()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'opaqueFunc() < 2|' returned FALSE. (42)' flowId='tst_Cmptest']
+##teamcity[testFailed name='tryVerify2()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'!c|' returned FALSE. (Should time out and fail)' flowId='tst_Cmptest']
 ##teamcity[testFinished name='tryVerify2()' flowId='tst_Cmptest']
 ##teamcity[testStarted name='verifyExplicitOperatorBool()' flowId='tst_Cmptest']
 ##teamcity[testFinished name='verifyExplicitOperatorBool()' flowId='tst_Cmptest']
diff --git a/tests/auto/testlib/selftests/expected_cmptest.txt b/tests/auto/testlib/selftests/expected_cmptest.txt
index 8547119988..f323729000 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.txt
+++ b/tests/auto/testlib/selftests/expected_cmptest.txt
@@ -191,15 +191,21 @@ FAIL!  : tst_Cmptest::compareQVector4D() Compared values are not the same
    Actual   (v4a): QVector4D(1, 2, 3, 4)
    Expected (v4b): QVector4D(1, 3, 3, 4)
    Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)]
+QINFO  : tst_Cmptest::tryCompare() Should now time out and fail
+FAIL!  : tst_Cmptest::tryCompare() Compared values are not the same
+   Actual   (c)             : DeferredFlag(true)
+   Expected (DeferredFlag()): DeferredFlag(false)
+   Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)]
 FAIL!  : tst_Cmptest::verify() 'opaqueFunc() < 2' returned FALSE. ()
    Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)]
 FAIL!  : tst_Cmptest::verify2() 'opaqueFunc() < 2' returned FALSE. (42)
    Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)]
-FAIL!  : tst_Cmptest::tryVerify() 'opaqueFunc() < 2' returned FALSE. ()
+QINFO  : tst_Cmptest::tryVerify() Should now time out and fail
+FAIL!  : tst_Cmptest::tryVerify() '!c' returned FALSE. ()
    Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)]
-FAIL!  : tst_Cmptest::tryVerify2() 'opaqueFunc() < 2' returned FALSE. (42)
+FAIL!  : tst_Cmptest::tryVerify2() '!c' returned FALSE. (Should time out and fail)
    Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)]
 PASS   : tst_Cmptest::verifyExplicitOperatorBool()
 PASS   : tst_Cmptest::cleanupTestCase()
-Totals: 21 passed, 48 failed, 0 skipped, 0 blacklisted, 0ms
+Totals: 21 passed, 49 failed, 0 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_Cmptest *********
diff --git a/tests/auto/testlib/selftests/expected_cmptest.xml b/tests/auto/testlib/selftests/expected_cmptest.xml
index f296a9e797..0bee84c786 100644
--- a/tests/auto/testlib/selftests/expected_cmptest.xml
+++ b/tests/auto/testlib/selftests/expected_cmptest.xml
@@ -379,6 +379,17 @@
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
+  <TestFunction name="tryCompare">
+  <Message type="qinfo" file="" line="0">
+    <Description><![CDATA[Should now time out and fail]]></Description>
+  </Message>
+    <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
+      <Description><![CDATA[Compared values are not the same
+   Actual   (c)             : DeferredFlag(true)
+   Expected (DeferredFlag()): DeferredFlag(false)]]></Description>
+    </Incident>
+    <Duration msecs="0"/>
+  </TestFunction>
   <TestFunction name="verify">
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
       <Description><![CDATA['opaqueFunc() < 2' returned FALSE. ()]]></Description>
@@ -392,14 +403,17 @@
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="tryVerify">
+  <Message type="qinfo" file="" line="0">
+    <Description><![CDATA[Should now time out and fail]]></Description>
+  </Message>
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
-      <Description><![CDATA['opaqueFunc() < 2' returned FALSE. ()]]></Description>
+      <Description><![CDATA['!c' returned FALSE. ()]]></Description>
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>
   <TestFunction name="tryVerify2">
     <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp" line="0">
-      <Description><![CDATA['opaqueFunc() < 2' returned FALSE. (42)]]></Description>
+      <Description><![CDATA['!c' returned FALSE. (Should time out and fail)]]></Description>
     </Incident>
     <Duration msecs="0"/>
   </TestFunction>