From 54bcefb25c8050b8a3ce9e5afb9fd70d9dff5bca Mon Sep 17 00:00:00 2001 From: Edward Welbourne Date: Mon, 11 Jul 2022 16:51:26 +0200 Subject: Test QTRY_COMPARE() and expand testing of QTRY_VERIFY*() In the process, simplify the latter while adding some actual time-variation for the QTRY_* loop to navigate round - based on the extendedcompare test's ClassWithDeferredSetter. Testing remains primitive, but is at least a bit more thorough. Pick-to: 6.4 Change-Id: I40be8fb485f3f18f0a4f4bc62ad36cccac691979 Reviewed-by: Qt CI Bot Reviewed-by: Mitch Curtis Reviewed-by: Jason McDonald --- .../auto/testlib/selftests/cmptest/tst_cmptest.cpp | 85 ++++++++++++++++++++-- .../testlib/selftests/expected_cmptest.junitxml | 18 ++++- .../testlib/selftests/expected_cmptest.lightxml | 18 ++++- tests/auto/testlib/selftests/expected_cmptest.tap | 56 +++++++++----- .../testlib/selftests/expected_cmptest.teamcity | 9 ++- tests/auto/testlib/selftests/expected_cmptest.txt | 12 ++- tests/auto/testlib/selftests/expected_cmptest.xml | 18 ++++- 7 files changed, 180 insertions(+), 36 deletions(-) (limited to 'tests/auto/testlib/selftests') diff --git a/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp b/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp index 94c04157f2..072324880e 100644 --- a/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp +++ b/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp @@ -1,9 +1,9 @@ // Copyright (C) 2016 The Qt Company Ltd. // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 - -#include #include +#include +#include #ifdef QT_GUI_LIB #include #include @@ -139,6 +139,7 @@ private slots: void compareQVector3D(); void compareQVector4D(); #endif + void tryCompare(); void verify(); void verify2(); void tryVerify(); @@ -649,16 +650,88 @@ void tst_Cmptest::verify2() QVERIFY2(opaqueFunc() < 2, QByteArray::number(opaqueFunc()).constData()); } +class DeferredFlag : public QObject // Can't be const. +{ + Q_OBJECT + bool m_flag; +public: + // A boolean that either starts out true or decays to true after 50 ms. + // However, that decay will only happen when the event loop is run. + explicit DeferredFlag(bool initial = false) : m_flag(initial) + { + if (!initial) + QTimer::singleShot(50, this, &DeferredFlag::onTimeOut); + } + explicit operator bool() const { return m_flag; } + bool operator!() const { return !m_flag; } + friend bool operator==(const DeferredFlag &a, const DeferredFlag &b) + { + return bool(a) == bool(b); + } +public slots: + void onTimeOut() { m_flag = true; } +}; + +char *toString(const DeferredFlag &val) +{ + return qstrdup(bool(val) ? "DeferredFlag(true)" : "DeferredFlag(false)"); +} + +void tst_Cmptest::tryCompare() +{ + /* Note that expected values given as DeferredFlag() shall be re-evaluated + each time the comparison is checked, hence supply a fresh false instance, + that'll be discarded before it has a chance to decay, hence only compare + equal to a false instance. Do not replace them with a local variable + initialized to false, as it would (of course) decay. + */ + DeferredFlag trueAlready(true); + { + DeferredFlag c; + // QTRY should check before looping, so be equal to the fresh false immediately. + QTRY_COMPARE(c, DeferredFlag()); + // Given time, it'll end up equal to a true one. + QTRY_COMPARE(c, trueAlready); + } + { + DeferredFlag c; + QTRY_COMPARE_WITH_TIMEOUT(c, DeferredFlag(), 300); + QVERIFY(!c); // Instantly equal, so succeeded without delay. + QTRY_COMPARE_WITH_TIMEOUT(c, trueAlready, 200); + qInfo("Should now time out and fail"); + QTRY_COMPARE_WITH_TIMEOUT(c, DeferredFlag(), 200); + } +} + void tst_Cmptest::tryVerify() { - QTRY_VERIFY(opaqueFunc() > 2); - QTRY_VERIFY_WITH_TIMEOUT(opaqueFunc() < 2, 1); + { + DeferredFlag c; + QTRY_VERIFY(!c); + QTRY_VERIFY(c); + } + { + DeferredFlag c; + QTRY_VERIFY_WITH_TIMEOUT(!c, 300); + QTRY_VERIFY_WITH_TIMEOUT(c, 200); + qInfo("Should now time out and fail"); + QTRY_VERIFY_WITH_TIMEOUT(!c, 200); + } } void tst_Cmptest::tryVerify2() { - QTRY_VERIFY2(opaqueFunc() > 2, QByteArray::number(opaqueFunc()).constData()); - QTRY_VERIFY2_WITH_TIMEOUT(opaqueFunc() < 2, QByteArray::number(opaqueFunc()).constData(), 1); + { + DeferredFlag c; + QTRY_VERIFY2(!c, "Failed to check before looping"); + QTRY_VERIFY2(c, "Failed to trigger single-shot"); + } + { + DeferredFlag c; + QTRY_VERIFY2_WITH_TIMEOUT(!c, "Failed to check before looping", 300); + QTRY_VERIFY2_WITH_TIMEOUT(c, "Failed to trigger single-shot", 200); + QTRY_VERIFY2_WITH_TIMEOUT(!c, "Should time out and fail", 200); + } } void tst_Cmptest::verifyExplicitOperatorBool() diff --git a/tests/auto/testlib/selftests/expected_cmptest.junitxml b/tests/auto/testlib/selftests/expected_cmptest.junitxml index f71b8f0a19..75ec61ba99 100644 --- a/tests/auto/testlib/selftests/expected_cmptest.junitxml +++ b/tests/auto/testlib/selftests/expected_cmptest.junitxml @@ -1,5 +1,5 @@ - + @@ -282,6 +282,15 @@ Expected (v4b): QVector4D(1, 3, 3, 4)]]> + + + + + + + + @@ -289,10 +298,13 @@ - + + + + - + diff --git a/tests/auto/testlib/selftests/expected_cmptest.lightxml b/tests/auto/testlib/selftests/expected_cmptest.lightxml index 3cb492361b..351fad849c 100644 --- a/tests/auto/testlib/selftests/expected_cmptest.lightxml +++ b/tests/auto/testlib/selftests/expected_cmptest.lightxml @@ -377,6 +377,17 @@ + + + + + + + + + @@ -390,14 +401,17 @@ + + + - + - + diff --git a/tests/auto/testlib/selftests/expected_cmptest.tap b/tests/auto/testlib/selftests/expected_cmptest.tap index da7f21ab70..19c90bcb93 100644 --- a/tests/auto/testlib/selftests/expected_cmptest.tap +++ b/tests/auto/testlib/selftests/expected_cmptest.tap @@ -516,7 +516,23 @@ not ok 63 - compareQVector4D() file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp line: 0 ... -not ok 64 - verify() +not ok 64 - tryCompare() + --- + type: QCOMPARE + message: Compared values are not the same + wanted: DeferredFlag(false) (DeferredFlag()) + found: DeferredFlag(true) (c) + expected: DeferredFlag(false) (DeferredFlag()) + actual: DeferredFlag(true) (c) + at: tst_Cmptest::tryCompare() (qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp:0) + file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp + line: 0 + extensions: + messages: + - severity: info + message: Should now time out and fail + ... +not ok 65 - verify() --- type: QVERIFY message: Verification failed @@ -528,7 +544,7 @@ not ok 64 - verify() file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp line: 0 ... -not ok 65 - verify2() +not ok 66 - verify2() --- type: QVERIFY message: 42 @@ -540,33 +556,37 @@ not ok 65 - verify2() file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp line: 0 ... -not ok 66 - tryVerify() +not ok 67 - tryVerify() --- type: QVERIFY message: Verification failed - wanted: true (opaqueFunc() < 2) - found: false (opaqueFunc() < 2) - expected: true (opaqueFunc() < 2) - actual: false (opaqueFunc() < 2) + wanted: true (!c) + found: false (!c) + expected: true (!c) + actual: false (!c) at: tst_Cmptest::tryVerify() (qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp:0) file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp line: 0 + extensions: + messages: + - severity: info + message: Should now time out and fail ... -not ok 67 - tryVerify2() +not ok 68 - tryVerify2() --- type: QVERIFY - message: 42 - wanted: true (opaqueFunc() < 2) - found: false (opaqueFunc() < 2) - expected: true (opaqueFunc() < 2) - actual: false (opaqueFunc() < 2) + message: Should time out and fail + wanted: true (!c) + found: false (!c) + expected: true (!c) + actual: false (!c) at: tst_Cmptest::tryVerify2() (qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp:0) file: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp line: 0 ... -ok 68 - verifyExplicitOperatorBool() -ok 69 - cleanupTestCase() -1..69 -# tests 69 +ok 69 - verifyExplicitOperatorBool() +ok 70 - cleanupTestCase() +1..70 +# tests 70 # pass 21 -# fail 48 +# fail 49 diff --git a/tests/auto/testlib/selftests/expected_cmptest.teamcity b/tests/auto/testlib/selftests/expected_cmptest.teamcity index 83e007413e..5f1e8374b9 100644 --- a/tests/auto/testlib/selftests/expected_cmptest.teamcity +++ b/tests/auto/testlib/selftests/expected_cmptest.teamcity @@ -169,6 +169,10 @@ ##teamcity[testStarted name='compareQVector4D()' flowId='tst_Cmptest'] ##teamcity[testFailed name='compareQVector4D()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='Compared values are not the same|n Actual (v4a): QVector4D(1, 2, 3, 4)|n Expected (v4b): QVector4D(1, 3, 3, 4)' flowId='tst_Cmptest'] ##teamcity[testFinished name='compareQVector4D()' flowId='tst_Cmptest'] +##teamcity[testStarted name='tryCompare()' flowId='tst_Cmptest'] +##teamcity[testFailed name='tryCompare()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='Compared values are not the same|n Actual (c) : DeferredFlag(true)|n Expected (DeferredFlag()): DeferredFlag(false)' flowId='tst_Cmptest'] +##teamcity[testStdOut name='tryCompare()' out='QINFO: Should now time out and fail' flowId='tst_Cmptest'] +##teamcity[testFinished name='tryCompare()' flowId='tst_Cmptest'] ##teamcity[testStarted name='verify()' flowId='tst_Cmptest'] ##teamcity[testFailed name='verify()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'opaqueFunc() < 2|' returned FALSE. ()' flowId='tst_Cmptest'] ##teamcity[testFinished name='verify()' flowId='tst_Cmptest'] @@ -176,10 +180,11 @@ ##teamcity[testFailed name='verify2()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'opaqueFunc() < 2|' returned FALSE. (42)' flowId='tst_Cmptest'] ##teamcity[testFinished name='verify2()' flowId='tst_Cmptest'] ##teamcity[testStarted name='tryVerify()' flowId='tst_Cmptest'] -##teamcity[testFailed name='tryVerify()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'opaqueFunc() < 2|' returned FALSE. ()' flowId='tst_Cmptest'] +##teamcity[testFailed name='tryVerify()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'!c|' returned FALSE. ()' flowId='tst_Cmptest'] +##teamcity[testStdOut name='tryVerify()' out='QINFO: Should now time out and fail' flowId='tst_Cmptest'] ##teamcity[testFinished name='tryVerify()' flowId='tst_Cmptest'] ##teamcity[testStarted name='tryVerify2()' flowId='tst_Cmptest'] -##teamcity[testFailed name='tryVerify2()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'opaqueFunc() < 2|' returned FALSE. (42)' flowId='tst_Cmptest'] +##teamcity[testFailed name='tryVerify2()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)|]' details='|'!c|' returned FALSE. (Should time out and fail)' flowId='tst_Cmptest'] ##teamcity[testFinished name='tryVerify2()' flowId='tst_Cmptest'] ##teamcity[testStarted name='verifyExplicitOperatorBool()' flowId='tst_Cmptest'] ##teamcity[testFinished name='verifyExplicitOperatorBool()' flowId='tst_Cmptest'] diff --git a/tests/auto/testlib/selftests/expected_cmptest.txt b/tests/auto/testlib/selftests/expected_cmptest.txt index 8547119988..f323729000 100644 --- a/tests/auto/testlib/selftests/expected_cmptest.txt +++ b/tests/auto/testlib/selftests/expected_cmptest.txt @@ -191,15 +191,21 @@ FAIL! : tst_Cmptest::compareQVector4D() Compared values are not the same Actual (v4a): QVector4D(1, 2, 3, 4) Expected (v4b): QVector4D(1, 3, 3, 4) Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)] +QINFO : tst_Cmptest::tryCompare() Should now time out and fail +FAIL! : tst_Cmptest::tryCompare() Compared values are not the same + Actual (c) : DeferredFlag(true) + Expected (DeferredFlag()): DeferredFlag(false) + Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)] FAIL! : tst_Cmptest::verify() 'opaqueFunc() < 2' returned FALSE. () Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)] FAIL! : tst_Cmptest::verify2() 'opaqueFunc() < 2' returned FALSE. (42) Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)] -FAIL! : tst_Cmptest::tryVerify() 'opaqueFunc() < 2' returned FALSE. () +QINFO : tst_Cmptest::tryVerify() Should now time out and fail +FAIL! : tst_Cmptest::tryVerify() '!c' returned FALSE. () Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)] -FAIL! : tst_Cmptest::tryVerify2() 'opaqueFunc() < 2' returned FALSE. (42) +FAIL! : tst_Cmptest::tryVerify2() '!c' returned FALSE. (Should time out and fail) Loc: [qtbase/tests/auto/testlib/selftests/cmptest/tst_cmptest.cpp(0)] PASS : tst_Cmptest::verifyExplicitOperatorBool() PASS : tst_Cmptest::cleanupTestCase() -Totals: 21 passed, 48 failed, 0 skipped, 0 blacklisted, 0ms +Totals: 21 passed, 49 failed, 0 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_Cmptest ********* diff --git a/tests/auto/testlib/selftests/expected_cmptest.xml b/tests/auto/testlib/selftests/expected_cmptest.xml index f296a9e797..0bee84c786 100644 --- a/tests/auto/testlib/selftests/expected_cmptest.xml +++ b/tests/auto/testlib/selftests/expected_cmptest.xml @@ -379,6 +379,17 @@ + + + + + + + + + @@ -392,14 +403,17 @@ + + + - + - + -- cgit v1.2.3