Add QTest::failOnWarning

This solves the long-standing problem of not being able to easily
fail a test when a certain warning is output.

[ChangeLog][QtTest] Added QTest::failOnWarning. When called in a test
function, any warning that matches the given pattern will cause a test
failure. The test will continue execution when a failure is added.
All patterns are cleared at the end of each test function.

Fixes: QTBUG-70029
Change-Id: I5763f8d4acf1cee8178be43a503619fbfb0f4f36
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

1 files changed, 112 insertions, 0 deletions

diff --git a/tests/auto/testlib/selftests/expected_warnings.xml b/tests/auto/testlib/selftests/expected_warnings.xml
index 1b17dbbf5d..aab6a99f74 100644
--- a/tests/auto/testlib/selftests/expected_warnings.xml
+++ b/tests/auto/testlib/selftests/expected_warnings.xml
@@ -91,6 +91,118 @@
 </Incident>
     <Duration msecs="0"/>
 </TestFunction>
+<TestFunction name="testFailOnWarnings">
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[Ran out of space!]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of cabbage!]]></Description>
+</Incident>
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[Ran out of tortillas!]]></Description>
+</Message>
+<Message type="qinfo" file="" line="0">
+    <Description><![CDATA[Ran out of oil!]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[nope]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of biscuits!]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Running low on toothpaste!]]></Description>
+</Incident>
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[Running low on flour!]]></Description>
+</Message>
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[Running low on toothpaste!]]></Description>
+</Message>
+<Message type="qinfo" file="" line="0">
+    <Description><![CDATA[Running low on toothpaste!]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsCleared">
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[Ran out of muffins!]]></Description>
+</Message>
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsWithData">
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <DataTag><![CDATA[warning1]]></DataTag>
+    <Description><![CDATA[Received a warning that resulted in a failure:
+warning1]]></Description>
+</Incident>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning1]]></DataTag>
+    <Description><![CDATA[warning2]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning1]]></DataTag>
+    <Description><![CDATA[warning3]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning2]]></DataTag>
+    <Description><![CDATA[warning1]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <DataTag><![CDATA[warning2]]></DataTag>
+    <Description><![CDATA[Received a warning that resulted in a failure:
+warning2]]></Description>
+</Incident>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning2]]></DataTag>
+    <Description><![CDATA[warning3]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning3]]></DataTag>
+    <Description><![CDATA[warning1]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning3]]></DataTag>
+    <Description><![CDATA[warning2]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <DataTag><![CDATA[warning3]]></DataTag>
+    <Description><![CDATA[Received a warning that resulted in a failure:
+warning3]]></Description>
+</Incident>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsFailInHelper">
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[This failure message should be printed but not cause the test to abort]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of cabbage!]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[My cabbage! :(]]></Description>
+</Incident>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsThenSkip">
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of cabbage!]]></Description>
+</Incident>
+<Message type="skip" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[My cabbage! :(]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsAndIgnoreWarnings">
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>