From efb283fb7f72e950c8ecf755b960a3c1b36b5507 Mon Sep 17 00:00:00 2001 From: Mitch Curtis Date: Fri, 8 Oct 2021 13:58:27 +0200 Subject: Add QTest::failOnWarning This solves the long-standing problem of not being able to easily fail a test when a certain warning is output. [ChangeLog][QtTest] Added QTest::failOnWarning. When called in a test function, any warning that matches the given pattern will cause a test failure. The test will continue execution when a failure is added. All patterns are cleared at the end of each test function. Fixes: QTBUG-70029 Change-Id: I5763f8d4acf1cee8178be43a503619fbfb0f4f36 Reviewed-by: Edward Welbourne --- .../testlib/selftests/expected_warnings.junitxml | 60 ++++++++++- .../testlib/selftests/expected_warnings.lightxml | 112 +++++++++++++++++++++ tests/auto/testlib/selftests/expected_warnings.tap | 105 ++++++++++++++++++- .../testlib/selftests/expected_warnings.teamcity | 36 +++++++ tests/auto/testlib/selftests/expected_warnings.txt | 48 ++++++++- tests/auto/testlib/selftests/expected_warnings.xml | 112 +++++++++++++++++++++ .../testlib/selftests/warnings/tst_warnings.cpp | 103 +++++++++++++++++++ 7 files changed, 569 insertions(+), 7 deletions(-) (limited to 'tests/auto') diff --git a/tests/auto/testlib/selftests/expected_warnings.junitxml b/tests/auto/testlib/selftests/expected_warnings.junitxml index 691bb32a80..92285a4937 100644 --- a/tests/auto/testlib/selftests/expected_warnings.junitxml +++ b/tests/auto/testlib/selftests/expected_warnings.junitxml @@ -1,5 +1,5 @@ - + @@ -49,5 +49,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_warnings.lightxml b/tests/auto/testlib/selftests/expected_warnings.lightxml index 9d5a106500..9a76de420a 100644 --- a/tests/auto/testlib/selftests/expected_warnings.lightxml +++ b/tests/auto/testlib/selftests/expected_warnings.lightxml @@ -89,6 +89,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_warnings.tap b/tests/auto/testlib/selftests/expected_warnings.tap index 0e357cb770..4dbebb45b3 100644 --- a/tests/auto/testlib/selftests/expected_warnings.tap +++ b/tests/auto/testlib/selftests/expected_warnings.tap @@ -35,8 +35,103 @@ not ok 6 - testMissingWarningsWithData(second row) --- # Not all expected messages were received ... -ok 7 - cleanupTestCase() -1..7 -# tests 7 -# pass 3 -# fail 4 +# Ran out of space! +not ok 7 - testFailOnWarnings() + --- + # Received a warning that resulted in a failure: +Ran out of cabbage! + at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) + file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp + line: 0 + ... +# Ran out of tortillas! +# Ran out of oil! +# nope +not ok 8 - testFailOnWarnings() + --- + # Received a warning that resulted in a failure: +Ran out of biscuits! + at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) + file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp + line: 0 + ... +not ok 9 - testFailOnWarnings() + --- + # Received a warning that resulted in a failure: +Running low on toothpaste! + at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) + file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp + line: 0 + ... +# Running low on flour! +# Running low on toothpaste! +# Running low on toothpaste! +# Ran out of muffins! +ok 10 - testFailOnWarningsCleared() +not ok 11 - testFailOnWarningsWithData(warning1) + --- + # Received a warning that resulted in a failure: +warning1 + at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) + file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp + line: 0 + ... +# warning2 +# warning3 +# warning1 +not ok 12 - testFailOnWarningsWithData(warning2) + --- + # Received a warning that resulted in a failure: +warning2 + at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) + file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp + line: 0 + ... +# warning3 +# warning1 +# warning2 +not ok 13 - testFailOnWarningsWithData(warning3) + --- + # Received a warning that resulted in a failure: +warning3 + at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) + file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp + line: 0 + ... +not ok 14 - testFailOnWarningsFailInHelper() + --- + # This failure message should be printed but not cause the test to abort + at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) + file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp + line: 0 + ... +not ok 15 - testFailOnWarningsFailInHelper() + --- + # Received a warning that resulted in a failure: +Ran out of cabbage! + at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) + file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp + line: 0 + ... +not ok 16 - testFailOnWarningsFailInHelper() + --- + # My cabbage! :( + at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) + file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp + line: 0 + ... +not ok 17 - testFailOnWarningsThenSkip() + --- + # Received a warning that resulted in a failure: +Ran out of cabbage! + at: tst_Warnings::testFailOnWarningsThenSkip() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0) + file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp + line: 0 + ... +ok 18 - testFailOnWarningsThenSkip() # SKIP My cabbage! :( +ok 19 - testFailOnWarningsAndIgnoreWarnings() +ok 20 - cleanupTestCase() +1..20 +# tests 20 +# pass 5 +# fail 14 diff --git a/tests/auto/testlib/selftests/expected_warnings.teamcity b/tests/auto/testlib/selftests/expected_warnings.teamcity index 5811beae4c..e488ce7b30 100644 --- a/tests/auto/testlib/selftests/expected_warnings.teamcity +++ b/tests/auto/testlib/selftests/expected_warnings.teamcity @@ -20,6 +20,42 @@ ##teamcity[testFailed name='testMissingWarningsWithData(second row)' message='Failure!' details='Not all expected messages were received' flowId='tst_Warnings'] ##teamcity[testStdOut name='testMissingWarningsWithData(second row)' out='INFO: Did not receive message: "Warning0"|nINFO: Did not receive message: "Warning1"' flowId='tst_Warnings'] ##teamcity[testFinished name='testMissingWarningsWithData(second row)' flowId='tst_Warnings'] +##teamcity[testStarted name='testFailOnWarnings()' flowId='tst_Warnings'] +##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings'] +##teamcity[testStdOut name='testFailOnWarnings()' out='QWARN: Ran out of space!' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings'] +##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of biscuits!' flowId='tst_Warnings'] +##teamcity[testStdOut name='testFailOnWarnings()' out='QDEBUG: Ran out of tortillas!|nQINFO: Ran out of oil!|nQWARN: nope' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings'] +##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRunning low on toothpaste!' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings'] +##teamcity[testStarted name='testFailOnWarningsCleared()' flowId='tst_Warnings'] +##teamcity[testStdOut name='testFailOnWarningsCleared()' out='QWARN: Running low on flour!|nQDEBUG: Running low on toothpaste!|nQINFO: Running low on toothpaste!|nQWARN: Ran out of muffins!' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarningsCleared()' flowId='tst_Warnings'] +##teamcity[testStarted name='testFailOnWarningsWithData(warning1)' flowId='tst_Warnings'] +##teamcity[testFailed name='testFailOnWarningsWithData(warning1)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning1' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarningsWithData(warning1)' flowId='tst_Warnings'] +##teamcity[testStarted name='testFailOnWarningsWithData(warning2)' flowId='tst_Warnings'] +##teamcity[testFailed name='testFailOnWarningsWithData(warning2)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning2' flowId='tst_Warnings'] +##teamcity[testStdOut name='testFailOnWarningsWithData(warning2)' out='QWARN: warning2|nQWARN: warning3|nQWARN: warning1' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarningsWithData(warning2)' flowId='tst_Warnings'] +##teamcity[testStarted name='testFailOnWarningsWithData(warning3)' flowId='tst_Warnings'] +##teamcity[testFailed name='testFailOnWarningsWithData(warning3)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning3' flowId='tst_Warnings'] +##teamcity[testStdOut name='testFailOnWarningsWithData(warning3)' out='QWARN: warning3|nQWARN: warning1|nQWARN: warning2' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarningsWithData(warning3)' flowId='tst_Warnings'] +##teamcity[testStarted name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings'] +##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='This failure message should be printed but not cause the test to abort' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings'] +##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings'] +##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='My cabbage! :(' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings'] +##teamcity[testStarted name='testFailOnWarningsThenSkip()' flowId='tst_Warnings'] +##teamcity[testFailed name='testFailOnWarningsThenSkip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarningsThenSkip()' flowId='tst_Warnings'] +##teamcity[testIgnored name='testFailOnWarningsThenSkip()' message='My cabbage! :( |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' flowId='tst_Warnings'] +##teamcity[testStarted name='testFailOnWarningsAndIgnoreWarnings()' flowId='tst_Warnings'] +##teamcity[testFinished name='testFailOnWarningsAndIgnoreWarnings()' flowId='tst_Warnings'] ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Warnings'] ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Warnings'] ##teamcity[testSuiteFinished name='tst_Warnings' flowId='tst_Warnings'] diff --git a/tests/auto/testlib/selftests/expected_warnings.txt b/tests/auto/testlib/selftests/expected_warnings.txt index 0a65213b79..02361eb39d 100644 --- a/tests/auto/testlib/selftests/expected_warnings.txt +++ b/tests/auto/testlib/selftests/expected_warnings.txt @@ -23,6 +23,52 @@ FAIL! : tst_Warnings::testMissingWarningsWithData(first row) Not all expected m INFO : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning0" INFO : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning1" FAIL! : tst_Warnings::testMissingWarningsWithData(second row) Not all expected messages were received +QWARN : tst_Warnings::testFailOnWarnings() Ran out of space! +FAIL! : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure: +Ran out of cabbage! + Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] +QDEBUG : tst_Warnings::testFailOnWarnings() Ran out of tortillas! +QINFO : tst_Warnings::testFailOnWarnings() Ran out of oil! +QWARN : tst_Warnings::testFailOnWarnings() nope +FAIL! : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure: +Ran out of biscuits! + Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] +FAIL! : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure: +Running low on toothpaste! + Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] +QWARN : tst_Warnings::testFailOnWarnings() Running low on flour! +QDEBUG : tst_Warnings::testFailOnWarnings() Running low on toothpaste! +QINFO : tst_Warnings::testFailOnWarnings() Running low on toothpaste! +QWARN : tst_Warnings::testFailOnWarningsCleared() Ran out of muffins! +PASS : tst_Warnings::testFailOnWarningsCleared() +FAIL! : tst_Warnings::testFailOnWarningsWithData(warning1) Received a warning that resulted in a failure: +warning1 + Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] +QWARN : tst_Warnings::testFailOnWarningsWithData(warning1) warning2 +QWARN : tst_Warnings::testFailOnWarningsWithData(warning1) warning3 +QWARN : tst_Warnings::testFailOnWarningsWithData(warning2) warning1 +FAIL! : tst_Warnings::testFailOnWarningsWithData(warning2) Received a warning that resulted in a failure: +warning2 + Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] +QWARN : tst_Warnings::testFailOnWarningsWithData(warning2) warning3 +QWARN : tst_Warnings::testFailOnWarningsWithData(warning3) warning1 +QWARN : tst_Warnings::testFailOnWarningsWithData(warning3) warning2 +FAIL! : tst_Warnings::testFailOnWarningsWithData(warning3) Received a warning that resulted in a failure: +warning3 + Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] +FAIL! : tst_Warnings::testFailOnWarningsFailInHelper() This failure message should be printed but not cause the test to abort + Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] +FAIL! : tst_Warnings::testFailOnWarningsFailInHelper() Received a warning that resulted in a failure: +Ran out of cabbage! + Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] +FAIL! : tst_Warnings::testFailOnWarningsFailInHelper() My cabbage! :( + Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] +FAIL! : tst_Warnings::testFailOnWarningsThenSkip() Received a warning that resulted in a failure: +Ran out of cabbage! + Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] +SKIP : tst_Warnings::testFailOnWarningsThenSkip() My cabbage! :( + Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)] +PASS : tst_Warnings::testFailOnWarningsAndIgnoreWarnings() PASS : tst_Warnings::cleanupTestCase() -Totals: 3 passed, 4 failed, 0 skipped, 0 blacklisted, 0ms +Totals: 5 passed, 14 failed, 1 skipped, 0 blacklisted, 0ms ********* Finished testing of tst_Warnings ********* diff --git a/tests/auto/testlib/selftests/expected_warnings.xml b/tests/auto/testlib/selftests/expected_warnings.xml index 1b17dbbf5d..aab6a99f74 100644 --- a/tests/auto/testlib/selftests/expected_warnings.xml +++ b/tests/auto/testlib/selftests/expected_warnings.xml @@ -91,6 +91,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/warnings/tst_warnings.cpp b/tests/auto/testlib/selftests/warnings/tst_warnings.cpp index 61f090c2c3..ee73bd4fe4 100644 --- a/tests/auto/testlib/selftests/warnings/tst_warnings.cpp +++ b/tests/auto/testlib/selftests/warnings/tst_warnings.cpp @@ -40,6 +40,16 @@ private slots: void testMissingWarningsRegularExpression(); void testMissingWarningsWithData_data(); void testMissingWarningsWithData(); + + void testFailOnWarnings(); + void testFailOnWarningsCleared(); +#if QT_CONFIG(regularexpression) + void testFailOnWarningsWithData_data(); + void testFailOnWarningsWithData(); + void testFailOnWarningsFailInHelper(); + void testFailOnWarningsThenSkip(); +#endif + void testFailOnWarningsAndIgnoreWarnings(); }; void tst_Warnings::testWarnings() @@ -124,6 +134,99 @@ void tst_Warnings::testMissingWarningsWithData() qWarning("Warning2"); } +void tst_Warnings::testFailOnWarnings() +{ + // failOnWarnings() wasn't called yet; shouldn't fail; + qWarning("Ran out of space!"); + +#if QT_CONFIG(regularexpression) + const auto warnRegex = QRegularExpression("Ran out of .*!"); + QTest::failOnWarning(warnRegex); + // Should now fail. + qWarning("Ran out of cabbage!"); + + // Should not fail; none of these are warnings. + qDebug("Ran out of tortillas!"); + qInfo("Ran out of oil!"); + + // Should not fail; regex doesn't match. + qWarning("nope"); + + // Should fail; matches regex. + qWarning("Ran out of biscuits!"); +#endif // QT_CONFIG(regularexpression) + + QTest::failOnWarning("Running low on toothpaste!"); + + // Should fail; strings match. + qWarning("Running low on toothpaste!"); + + // Shouldn't fail; strings don't match. + qWarning("Running low on flour!"); + + // Should not fail; none of these are warnings. + qDebug("Running low on toothpaste!"); + qInfo("Running low on toothpaste!"); +} + +void tst_Warnings::testFailOnWarningsCleared() +{ + // The patterns passed to failOnWarnings() should be cleared at the end of + // each test function, so this shouldn't fail because of the failOnWarning() call in the previous function. + // Note that this test always needs to come after testFailOnWarnings for it to work. + qWarning("Ran out of muffins!"); +} + +#if QT_CONFIG(regularexpression) +void tst_Warnings::testFailOnWarningsWithData_data() +{ + // The warning message that should cause a failure. + QTest::addColumn("warningMessage"); + + QTest::newRow("warning1") << "warning1"; + QTest::newRow("warning2") << "warning2"; + QTest::newRow("warning3") << "warning3"; +} + +void tst_Warnings::testFailOnWarningsWithData() +{ + QFETCH(QString, warningMessage); + + QTest::failOnWarning(QRegularExpression(warningMessage)); + + // Only one of these should fail, depending on warningMessage. + qWarning("warning1"); + qWarning("warning2"); + qWarning("warning3"); +} + +void tst_Warnings::testFailOnWarningsFailInHelper() +{ + [](){ QFAIL("This failure message should be printed but not cause the test to abort"); }(); + const auto warnRegex = QRegularExpression("Ran out of .*!"); + QTest::failOnWarning(warnRegex); + qWarning("Ran out of cabbage!"); + QFAIL("My cabbage! :("); +} + +void tst_Warnings::testFailOnWarningsThenSkip() +{ + const auto warnRegex = QRegularExpression("Ran out of .*!"); + QTest::failOnWarning(warnRegex); + qWarning("Ran out of cabbage!"); + QSKIP("My cabbage! :("); +} +#endif // QT_CONFIG(regularexpression) + +void tst_Warnings::testFailOnWarningsAndIgnoreWarnings() +{ + const auto warningStr = "Running low on toothpaste!"; + QTest::failOnWarning(warningStr); + QTest::ignoreMessage(QtWarningMsg, warningStr); + // Shouldn't fail; we ignored it. + qWarning(warningStr); +} + QTEST_MAIN(tst_Warnings) #include "tst_warnings.moc" -- cgit v1.2.3