From cc6d984390dc937b9d8440b6ba7d4f578e22ac0d Mon Sep 17 00:00:00 2001 From: Ivan Solovev Date: Thu, 2 Jun 2022 11:51:49 +0200 Subject: Add QTRY_COMPARE_{EQ,NE,LT,LE,GT,GE}_WITH_TIMEOUT() [ChangeLog][QTestLib] Add QTRY_COMPARE_{EQ,NE,LT,LE,GT,GE}_WITH_TIMEOUT macros that repeatedly execute QCOMPARE_{EQ,NE,LT,LE,GT,GE} until either the comparison returns true or the timeout expires. Also add QTRY_COMPARE_{EQ,NE,LT,LE,GT,GE} macros that simply invoke the *_WITH_TIMEOUT versions with the usual timeout of five seconds. Task-number: QTBUG-98873 Change-Id: Ib0d7d1c8c997f442b46acd85da738a8f512cc875 Reviewed-by: Marc Mutz --- src/testlib/qtestcase.h | 36 +++++ src/testlib/qtestcase.qdoc | 150 +++++++++++++++++++++ .../selftests/expected_extendedcompare.junitxml | 8 +- .../selftests/expected_extendedcompare.lightxml | 30 +++++ .../testlib/selftests/expected_extendedcompare.tap | 49 ++++++- .../selftests/expected_extendedcompare.teamcity | 10 ++ .../testlib/selftests/expected_extendedcompare.txt | 17 ++- .../testlib/selftests/expected_extendedcompare.xml | 30 +++++ .../extendedcompare/tst_extendedcompare.cpp | 54 ++++++++ 9 files changed, 377 insertions(+), 7 deletions(-) diff --git a/src/testlib/qtestcase.h b/src/testlib/qtestcase.h index 555458e041..d4caf20e69 100644 --- a/src/testlib/qtestcase.h +++ b/src/testlib/qtestcase.h @@ -204,6 +204,42 @@ do { \ #define QTRY_COMPARE(expr, expected) QTRY_COMPARE_WITH_TIMEOUT((expr), expected, 5000) +#define QTRY_COMPARE_OP_WITH_TIMEOUT_IMPL(left, right, op, opId, timeout) \ +do { \ + QTRY_IMPL(((left) op (right)), timeout); \ + QCOMPARE_OP_IMPL(left, right, op, opId); \ +} while (false) + +#define QTRY_COMPARE_EQ_WITH_TIMEOUT(left, right, timeout) \ + QTRY_COMPARE_OP_WITH_TIMEOUT_IMPL(left, right, ==, Equal, timeout) + +#define QTRY_COMPARE_EQ(left, right) QTRY_COMPARE_EQ_WITH_TIMEOUT(left, right, 5000) + +#define QTRY_COMPARE_NE_WITH_TIMEOUT(left, right, timeout) \ + QTRY_COMPARE_OP_WITH_TIMEOUT_IMPL(left, right, !=, NotEqual, timeout) + +#define QTRY_COMPARE_NE(left, right) QTRY_COMPARE_NE_WITH_TIMEOUT(left, right, 5000) + +#define QTRY_COMPARE_LT_WITH_TIMEOUT(left, right, timeout) \ + QTRY_COMPARE_OP_WITH_TIMEOUT_IMPL(left, right, <, LessThan, timeout) + +#define QTRY_COMPARE_LT(left, right) QTRY_COMPARE_LT_WITH_TIMEOUT(left, right, 5000) + +#define QTRY_COMPARE_LE_WITH_TIMEOUT(left, right, timeout) \ + QTRY_COMPARE_OP_WITH_TIMEOUT_IMPL(left, right, <=, LessThanOrEqual, timeout) + +#define QTRY_COMPARE_LE(left, right) QTRY_COMPARE_LE_WITH_TIMEOUT(left, right, 5000) + +#define QTRY_COMPARE_GT_WITH_TIMEOUT(left, right, timeout) \ + QTRY_COMPARE_OP_WITH_TIMEOUT_IMPL(left, right, >, GreaterThan, timeout) + +#define QTRY_COMPARE_GT(left, right) QTRY_COMPARE_GT_WITH_TIMEOUT(left, right, 5000) + +#define QTRY_COMPARE_GE_WITH_TIMEOUT(left, right, timeout) \ + QTRY_COMPARE_OP_WITH_TIMEOUT_IMPL(left, right, >=, GreaterThanOrEqual, timeout) + +#define QTRY_COMPARE_GE(left, right) QTRY_COMPARE_GE_WITH_TIMEOUT(left, right, 5000) + #define QSKIP_INTERNAL(statement) \ do {\ QTest::qSkip(static_cast(statement), __FILE__, __LINE__);\ diff --git a/src/testlib/qtestcase.qdoc b/src/testlib/qtestcase.qdoc index 171a0b906c..d735647124 100644 --- a/src/testlib/qtestcase.qdoc +++ b/src/testlib/qtestcase.qdoc @@ -455,6 +455,156 @@ QEXPECT_FAIL() */ +/*! \macro QTRY_COMPARE_EQ_WITH_TIMEOUT(left, right, timeout) + \relates QTest + + This macro is similar to QCOMPARE_EQ(), but performs the comparison of the + \a left and \a right values repeatedly, until either the comparison returns + \c true or the \a timeout (in milliseconds) is reached. Between each + comparison, events will be processed. If the timeout is reached, a failure + is recorded in the test log and the test won't be executed further. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_EQ(), QTRY_COMPARE_EQ() +*/ + +/*! \macro QTRY_COMPARE_EQ(left, right) + \relates QTest + + Performs comparison of \a left and \a right values by invoking + QTRY_COMPARE_EQ_WITH_TIMEOUT with a timeout of five seconds. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_EQ(), QTRY_COMPARE_EQ_WITH_TIMEOUT() +*/ + +/*! \macro QTRY_COMPARE_NE_WITH_TIMEOUT(left, right, timeout) + \relates QTest + + This macro is similar to QCOMPARE_NE(), but performs the comparison of the + \a left and \a right values repeatedly, until either the comparison returns + \c true or the \a timeout (in milliseconds) is reached. Between each + comparison, events will be processed. If the timeout is reached, a failure + is recorded in the test log and the test won't be executed further. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_NE(), QTRY_COMPARE_NE() +*/ + +/*! \macro QTRY_COMPARE_NE(left, right) + \relates QTest + + Performs comparison of \a left and \a right values by invoking + QTRY_COMPARE_NE_WITH_TIMEOUT with a timeout of five seconds. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_NE(), QTRY_COMPARE_NE_WITH_TIMEOUT() +*/ + +/*! \macro QTRY_COMPARE_LT_WITH_TIMEOUT(left, right, timeout) + \relates QTest + + This macro is similar to QCOMPARE_LT(), but performs the comparison of the + \a left and \a right values repeatedly, until either the comparison returns + \c true or the \a timeout (in milliseconds) is reached. Between each + comparison, events will be processed. If the timeout is reached, a failure + is recorded in the test log and the test won't be executed further. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_LT(), QTRY_COMPARE_LT() +*/ + +/*! \macro QTRY_COMPARE_LT(left, right) + \relates QTest + + Performs comparison of \a left and \a right values by invoking + QTRY_COMPARE_LT_WITH_TIMEOUT with a timeout of five seconds. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_LT(), QTRY_COMPARE_LT_WITH_TIMEOUT() +*/ + +/*! \macro QTRY_COMPARE_LE_WITH_TIMEOUT(left, right, timeout) + \relates QTest + + This macro is similar to QCOMPARE_LE(), but performs the comparison of the + \a left and \a right values repeatedly, until either the comparison returns + \c true or the \a timeout (in milliseconds) is reached. Between each + comparison, events will be processed. If the timeout is reached, a failure + is recorded in the test log and the test won't be executed further. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_LE(), QTRY_COMPARE_LE() +*/ + +/*! \macro QTRY_COMPARE_LE(left, right) + \relates QTest + + Performs comparison of \a left and \a right values by invoking + QTRY_COMPARE_LE_WITH_TIMEOUT with a timeout of five seconds. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_LE(), QTRY_COMPARE_LE_WITH_TIMEOUT() +*/ + +/*! \macro QTRY_COMPARE_GT_WITH_TIMEOUT(left, right, timeout) + \relates QTest + + This macro is similar to QCOMPARE_GT(), but performs the comparison of the + \a left and \a right values repeatedly, until either the comparison returns + \c true or the \a timeout (in milliseconds) is reached. Between each + comparison, events will be processed. If the timeout is reached, a failure + is recorded in the test log and the test won't be executed further. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_GT(), QTRY_COMPARE_GT() +*/ + +/*! \macro QTRY_COMPARE_GT(left, right) + \relates QTest + + Performs comparison of \a left and \a right values by invoking + QTRY_COMPARE_GT_WITH_TIMEOUT with a timeout of five seconds. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_GT(), QTRY_COMPARE_GT_WITH_TIMEOUT() +*/ + +/*! \macro QTRY_COMPARE_GE_WITH_TIMEOUT(left, right, timeout) + \relates QTest + + This macro is similar to QCOMPARE_GE(), but performs the comparison of the + \a left and \a right values repeatedly, until either the comparison returns + \c true or the \a timeout (in milliseconds) is reached. Between each + comparison, events will be processed. If the timeout is reached, a failure + is recorded in the test log and the test won't be executed further. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_GE(), QTRY_COMPARE_GE() +*/ + +/*! \macro QTRY_COMPARE_GE(left, right) + \relates QTest + + Performs comparison of \a left and \a right values by invoking + QTRY_COMPARE_GE_WITH_TIMEOUT with a timeout of five seconds. + + \include qtestcase.qdoc macro-usage-limitation + + \sa QCOMPARE_GE(), QTRY_COMPARE_GE_WITH_TIMEOUT() +*/ + /*! \macro QFETCH(type, name) \relates QTest diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.junitxml b/tests/auto/testlib/selftests/expected_extendedcompare.junitxml index 8e0959f67e..cbde2e3101 100644 --- a/tests/auto/testlib/selftests/expected_extendedcompare.junitxml +++ b/tests/auto/testlib/selftests/expected_extendedcompare.junitxml @@ -1,5 +1,5 @@ - + @@ -516,5 +516,11 @@ Right (getClassForValue(1).getValuePointer()): MyClass(1) on memory address with index 1]]> + + + + + diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.lightxml b/tests/auto/testlib/selftests/expected_extendedcompare.lightxml index bdc5d088d2..e59dd6edc7 100644 --- a/tests/auto/testlib/selftests/expected_extendedcompare.lightxml +++ b/tests/auto/testlib/selftests/expected_extendedcompare.lightxml @@ -709,6 +709,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.tap b/tests/auto/testlib/selftests/expected_extendedcompare.tap index 66166718a2..31562636fb 100644 --- a/tests/auto/testlib/selftests/expected_extendedcompare.tap +++ b/tests/auto/testlib/selftests/expected_extendedcompare.tap @@ -976,8 +976,47 @@ not ok 151 - checkComparisonForTemporaryObjects(GE) file: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp line: 0 ... -ok 152 - cleanupTestCase() -1..152 -# tests 152 -# pass 77 -# fail 75 +ok 152 - checkComparisonWithTimeout(EQ) +ok 153 - checkComparisonWithTimeout(NE) +not ok 154 - checkComparisonWithTimeout(LT) + --- + type: QCOMPARE_LT + message: Left value is expected to be less than right value, but is not + wanted: < ClassWithDeferredSetter(0) (ClassWithDeferredSetter(0)) + found: ClassWithDeferredSetter(1) (c) + expected: < ClassWithDeferredSetter(0) (ClassWithDeferredSetter(0)) + actual: ClassWithDeferredSetter(1) (c) + at: tst_ExtendedCompare::checkComparisonWithTimeout() (qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp:0) + file: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp + line: 0 + ... +not ok 155 - checkComparisonWithTimeout(LE) + --- + type: QCOMPARE_LE + message: Left value is expected to be less than or equal to right value, but is not + wanted: <= ClassWithDeferredSetter(-1) (ClassWithDeferredSetter(-1)) + found: ClassWithDeferredSetter(1) (c) + expected: <= ClassWithDeferredSetter(-1) (ClassWithDeferredSetter(-1)) + actual: ClassWithDeferredSetter(1) (c) + at: tst_ExtendedCompare::checkComparisonWithTimeout() (qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp:0) + file: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp + line: 0 + ... +not ok 156 - checkComparisonWithTimeout(GT) + --- + type: QCOMPARE_GT + message: Left value is expected to be greater than right value, but is not + wanted: > ClassWithDeferredSetter(1) (ClassWithDeferredSetter(1)) + found: ClassWithDeferredSetter(1) (c) + expected: > ClassWithDeferredSetter(1) (ClassWithDeferredSetter(1)) + actual: ClassWithDeferredSetter(1) (c) + at: tst_ExtendedCompare::checkComparisonWithTimeout() (qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp:0) + file: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp + line: 0 + ... +ok 157 - checkComparisonWithTimeout(GE) +ok 158 - cleanupTestCase() +1..158 +# tests 158 +# pass 80 +# fail 78 diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.teamcity b/tests/auto/testlib/selftests/expected_extendedcompare.teamcity index ef8189be33..0f365cd3d0 100644 --- a/tests/auto/testlib/selftests/expected_extendedcompare.teamcity +++ b/tests/auto/testlib/selftests/expected_extendedcompare.teamcity @@ -371,6 +371,16 @@ ##teamcity[testFinished name='checkComparisonForTemporaryObjects()' flowId='tst_ExtendedCompare'] ##teamcity[testFailed name='checkComparisonForTemporaryObjects()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)|]' details='Left value is expected to be greater than or equal to right value, but is not|n Left (getClassForValue(0).getValuePointer()): MyClass(2) on memory address with index 0|n Right (getClassForValue(1).getValuePointer()): MyClass(1) on memory address with index 1' flowId='tst_ExtendedCompare'] ##teamcity[testFinished name='checkComparisonForTemporaryObjects()' flowId='tst_ExtendedCompare'] +##teamcity[testStarted name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare'] +##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare'] +##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare'] +##teamcity[testFailed name='checkComparisonWithTimeout()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)|]' details='Left value is expected to be less than right value, but is not|n Left (c) : ClassWithDeferredSetter(1)|n Right (ClassWithDeferredSetter(0)): ClassWithDeferredSetter(0)' flowId='tst_ExtendedCompare'] +##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare'] +##teamcity[testFailed name='checkComparisonWithTimeout()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)|]' details='Left value is expected to be less than or equal to right value, but is not|n Left (c) : ClassWithDeferredSetter(1)|n Right (ClassWithDeferredSetter(-1)): ClassWithDeferredSetter(-1)' flowId='tst_ExtendedCompare'] +##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare'] +##teamcity[testFailed name='checkComparisonWithTimeout()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)|]' details='Left value is expected to be greater than right value, but is not|n Left (c) : ClassWithDeferredSetter(1)|n Right (ClassWithDeferredSetter(1)): ClassWithDeferredSetter(1)' flowId='tst_ExtendedCompare'] +##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare'] +##teamcity[testFinished name='checkComparisonWithTimeout()' flowId='tst_ExtendedCompare'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_ExtendedCompare'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_ExtendedCompare'] ##teamcity[testSuiteFinished name='tst_ExtendedCompare' flowId='tst_ExtendedCompare'] diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.txt b/tests/auto/testlib/selftests/expected_extendedcompare.txt index ac240d4f85..15f3275844 100644 --- a/tests/auto/testlib/selftests/expected_extendedcompare.txt +++ b/tests/auto/testlib/selftests/expected_extendedcompare.txt @@ -376,6 +376,21 @@ FAIL! : tst_ExtendedCompare::checkComparisonForTemporaryObjects(GE) Left value Left (getClassForValue(0).getValuePointer()): MyClass(2) on memory address with index 0 Right (getClassForValue(1).getValuePointer()): MyClass(1) on memory address with index 1 Loc: [qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)] +PASS : tst_ExtendedCompare::checkComparisonWithTimeout(EQ) +PASS : tst_ExtendedCompare::checkComparisonWithTimeout(NE) +FAIL! : tst_ExtendedCompare::checkComparisonWithTimeout(LT) Left value is expected to be less than right value, but is not + Left (c) : ClassWithDeferredSetter(1) + Right (ClassWithDeferredSetter(0)): ClassWithDeferredSetter(0) + Loc: [qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)] +FAIL! : tst_ExtendedCompare::checkComparisonWithTimeout(LE) Left value is expected to be less than or equal to right value, but is not + Left (c) : ClassWithDeferredSetter(1) + Right (ClassWithDeferredSetter(-1)): ClassWithDeferredSetter(-1) + Loc: [qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)] +FAIL! : tst_ExtendedCompare::checkComparisonWithTimeout(GT) Left value is expected to be greater than right value, but is not + Left (c) : ClassWithDeferredSetter(1) + Right (ClassWithDeferredSetter(1)): ClassWithDeferredSetter(1) + Loc: [qtbase/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp(0)] +PASS : tst_ExtendedCompare::checkComparisonWithTimeout(GE) PASS : tst_ExtendedCompare::cleanupTestCase() -Totals: 77 passed, 75 failed, 0 skipped, 0 blacklisted, 0ms +Totals: 80 passed, 78 failed, 0 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_ExtendedCompare ********* diff --git a/tests/auto/testlib/selftests/expected_extendedcompare.xml b/tests/auto/testlib/selftests/expected_extendedcompare.xml index 427651a235..7bfca23b02 100644 --- a/tests/auto/testlib/selftests/expected_extendedcompare.xml +++ b/tests/auto/testlib/selftests/expected_extendedcompare.xml @@ -711,6 +711,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp b/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp index 503729365f..d198d621e1 100644 --- a/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp +++ b/tests/auto/testlib/selftests/extendedcompare/tst_extendedcompare.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 #include +#include QT_BEGIN_NAMESPACE @@ -143,6 +144,7 @@ private slots: void compareCustomTypes_data(); void compareCustomTypes(); void checkComparisonForTemporaryObjects(); + void checkComparisonWithTimeout(); }; void tst_ExtendedCompare::initTestCase_data() @@ -272,6 +274,58 @@ void tst_ExtendedCompare::checkComparisonForTemporaryObjects() getClassForValue(1).getValuePointer()); } +class ClassWithDeferredSetter : public MyClass +{ +public: + ClassWithDeferredSetter(int value) : MyClass(value) {} + + void setValueDeferred(int value) + { + QTimer::singleShot(100, [this, value] { setValue(value); }); + } +}; + +namespace QTest { + +char *toString(const ClassWithDeferredSetter &val) +{ + char *msg = new char[128]; + qsnprintf(msg, 128, "ClassWithDeferredSetter(%d)", val.value()); + return msg; +} + +} // namespace QTest + +void tst_ExtendedCompare::checkComparisonWithTimeout() +{ + QFETCH_GLOBAL(QTest::ComparisonOperation, operation); + ClassWithDeferredSetter c(0); + c.setValueDeferred(1); + switch (operation) { + case QTest::ComparisonOperation::Equal: + QTRY_COMPARE_EQ_WITH_TIMEOUT(c, ClassWithDeferredSetter(1), 300); + break; + case QTest::ComparisonOperation::NotEqual: + QTRY_COMPARE_NE_WITH_TIMEOUT(c, ClassWithDeferredSetter(0), 300); + break; + case QTest::ComparisonOperation::LessThan: + QTRY_COMPARE_LT_WITH_TIMEOUT(c, ClassWithDeferredSetter(0), 300); + break; + case QTest::ComparisonOperation::LessThanOrEqual: + QTRY_COMPARE_LE_WITH_TIMEOUT(c, ClassWithDeferredSetter(-1), 300); + break; + case QTest::ComparisonOperation::GreaterThan: + QTRY_COMPARE_GT_WITH_TIMEOUT(c, ClassWithDeferredSetter(1), 300); + break; + case QTest::ComparisonOperation::GreaterThanOrEqual: + QTRY_COMPARE_GE_WITH_TIMEOUT(c, ClassWithDeferredSetter(1), 300); + break; + case QTest::ComparisonOperation::CustomCompare: + QFAIL("Unexpected comparison operation"); + break; + } +} + QT_END_NAMESPACE QTEST_MAIN(tst_ExtendedCompare) -- cgit v1.2.3