Add QTest::failOnWarning

This solves the long-standing problem of not being able to easily
fail a test when a certain warning is output.

[ChangeLog][QtTest] Added QTest::failOnWarning. When called in a test
function, any warning that matches the given pattern will cause a test
failure. The test will continue execution when a failure is added.
All patterns are cleared at the end of each test function.

Fixes: QTBUG-70029
Change-Id: I5763f8d4acf1cee8178be43a503619fbfb0f4f36
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

7 files changed, 569 insertions, 7 deletions

diff --git a/tests/auto/testlib/selftests/expected_warnings.junitxml b/tests/auto/testlib/selftests/expected_warnings.junitxml
index 691bb32a80..92285a4937 100644
--- a/tests/auto/testlib/selftests/expected_warnings.junitxml
+++ b/tests/auto/testlib/selftests/expected_warnings.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="7" failures="4" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_Warnings" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="15" failures="10" errors="0" skipped="1" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -49,5 +49,63 @@
       <![CDATA[Did not receive message: "Warning1"]]>
     </system-out>
   </testcase>
+  <testcase name="testFailOnWarnings" classname="tst_Warnings" time="@TEST_DURATION@">
+    <failure type="fail" message="Received a warning that resulted in a failure:">
+      <![CDATA[Ran out of cabbage!]]>
+    </failure>
+    <system-out>
+      <![CDATA[Ran out of tortillas!]]>
+      <![CDATA[Ran out of oil!]]>
+      <![CDATA[Running low on toothpaste!]]>
+      <![CDATA[Running low on toothpaste!]]>
+    </system-out>
+    <system-err>
+      <![CDATA[Ran out of space!]]>
+      <![CDATA[nope]]>
+      <![CDATA[Running low on flour!]]>
+    </system-err>
+  </testcase>
+  <testcase name="testFailOnWarningsCleared" classname="tst_Warnings" time="@TEST_DURATION@">
+    <system-err>
+      <![CDATA[Ran out of muffins!]]>
+    </system-err>
+  </testcase>
+  <testcase name="testFailOnWarningsWithData(warning1)" classname="tst_Warnings" time="@TEST_DURATION@">
+    <failure type="fail" message="Received a warning that resulted in a failure:">
+      <![CDATA[warning1]]>
+    </failure>
+    <system-err>
+      <![CDATA[warning2]]>
+      <![CDATA[warning3]]>
+    </system-err>
+  </testcase>
+  <testcase name="testFailOnWarningsWithData(warning2)" classname="tst_Warnings" time="@TEST_DURATION@">
+    <failure type="fail" message="Received a warning that resulted in a failure:">
+      <![CDATA[warning2]]>
+    </failure>
+    <system-err>
+      <![CDATA[warning1]]>
+      <![CDATA[warning3]]>
+    </system-err>
+  </testcase>
+  <testcase name="testFailOnWarningsWithData(warning3)" classname="tst_Warnings" time="@TEST_DURATION@">
+    <failure type="fail" message="Received a warning that resulted in a failure:">
+      <![CDATA[warning3]]>
+    </failure>
+    <system-err>
+      <![CDATA[warning1]]>
+      <![CDATA[warning2]]>
+    </system-err>
+  </testcase>
+  <testcase name="testFailOnWarningsFailInHelper" classname="tst_Warnings" time="@TEST_DURATION@">
+    <failure type="fail" message="This failure message should be printed but not cause the test to abort"/>
+  </testcase>
+  <testcase name="testFailOnWarningsThenSkip" classname="tst_Warnings" time="@TEST_DURATION@">
+    <failure type="fail" message="Received a warning that resulted in a failure:">
+      <![CDATA[Ran out of cabbage!]]>
+    </failure>
+    <skipped message="My cabbage! :("/>
+  </testcase>
+  <testcase name="testFailOnWarningsAndIgnoreWarnings" classname="tst_Warnings" time="@TEST_DURATION@"/>
   <testcase name="cleanupTestCase" classname="tst_Warnings" time="@TEST_DURATION@"/>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/expected_warnings.lightxml b/tests/auto/testlib/selftests/expected_warnings.lightxml
index 9d5a106500..9a76de420a 100644
--- a/tests/auto/testlib/selftests/expected_warnings.lightxml
+++ b/tests/auto/testlib/selftests/expected_warnings.lightxml
@@ -89,6 +89,118 @@
 </Incident>
     <Duration msecs="0"/>
 </TestFunction>
+<TestFunction name="testFailOnWarnings">
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[Ran out of space!]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of cabbage!]]></Description>
+</Incident>
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[Ran out of tortillas!]]></Description>
+</Message>
+<Message type="qinfo" file="" line="0">
+    <Description><![CDATA[Ran out of oil!]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[nope]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of biscuits!]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Running low on toothpaste!]]></Description>
+</Incident>
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[Running low on flour!]]></Description>
+</Message>
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[Running low on toothpaste!]]></Description>
+</Message>
+<Message type="qinfo" file="" line="0">
+    <Description><![CDATA[Running low on toothpaste!]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsCleared">
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[Ran out of muffins!]]></Description>
+</Message>
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsWithData">
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <DataTag><![CDATA[warning1]]></DataTag>
+    <Description><![CDATA[Received a warning that resulted in a failure:
+warning1]]></Description>
+</Incident>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning1]]></DataTag>
+    <Description><![CDATA[warning2]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning1]]></DataTag>
+    <Description><![CDATA[warning3]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning2]]></DataTag>
+    <Description><![CDATA[warning1]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <DataTag><![CDATA[warning2]]></DataTag>
+    <Description><![CDATA[Received a warning that resulted in a failure:
+warning2]]></Description>
+</Incident>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning2]]></DataTag>
+    <Description><![CDATA[warning3]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning3]]></DataTag>
+    <Description><![CDATA[warning1]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning3]]></DataTag>
+    <Description><![CDATA[warning2]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <DataTag><![CDATA[warning3]]></DataTag>
+    <Description><![CDATA[Received a warning that resulted in a failure:
+warning3]]></Description>
+</Incident>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsFailInHelper">
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[This failure message should be printed but not cause the test to abort]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of cabbage!]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[My cabbage! :(]]></Description>
+</Incident>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsThenSkip">
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of cabbage!]]></Description>
+</Incident>
+<Message type="skip" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[My cabbage! :(]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsAndIgnoreWarnings">
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/expected_warnings.tap b/tests/auto/testlib/selftests/expected_warnings.tap
index 0e357cb770..4dbebb45b3 100644
--- a/tests/auto/testlib/selftests/expected_warnings.tap
+++ b/tests/auto/testlib/selftests/expected_warnings.tap
@@ -35,8 +35,103 @@ not ok 6 - testMissingWarningsWithData(second row)
   ---
   # Not all expected messages were received
   ...
-ok 7 - cleanupTestCase()
-1..7
-# tests 7
-# pass 3
-# fail 4
+# Ran out of space!
+not ok 7 - testFailOnWarnings()
+  ---
+  # Received a warning that resulted in a failure:
+Ran out of cabbage!
+  at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+  line: 0
+  ...
+# Ran out of tortillas!
+# Ran out of oil!
+# nope
+not ok 8 - testFailOnWarnings()
+  ---
+  # Received a warning that resulted in a failure:
+Ran out of biscuits!
+  at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+  line: 0
+  ...
+not ok 9 - testFailOnWarnings()
+  ---
+  # Received a warning that resulted in a failure:
+Running low on toothpaste!
+  at: tst_Warnings::testFailOnWarnings() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+  line: 0
+  ...
+# Running low on flour!
+# Running low on toothpaste!
+# Running low on toothpaste!
+# Ran out of muffins!
+ok 10 - testFailOnWarningsCleared()
+not ok 11 - testFailOnWarningsWithData(warning1)
+  ---
+  # Received a warning that resulted in a failure:
+warning1
+  at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+  line: 0
+  ...
+# warning2
+# warning3
+# warning1
+not ok 12 - testFailOnWarningsWithData(warning2)
+  ---
+  # Received a warning that resulted in a failure:
+warning2
+  at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+  line: 0
+  ...
+# warning3
+# warning1
+# warning2
+not ok 13 - testFailOnWarningsWithData(warning3)
+  ---
+  # Received a warning that resulted in a failure:
+warning3
+  at: tst_Warnings::testFailOnWarningsWithData() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+  line: 0
+  ...
+not ok 14 - testFailOnWarningsFailInHelper()
+  ---
+  # This failure message should be printed but not cause the test to abort
+  at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+  line: 0
+  ...
+not ok 15 - testFailOnWarningsFailInHelper()
+  ---
+  # Received a warning that resulted in a failure:
+Ran out of cabbage!
+  at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+  line: 0
+  ...
+not ok 16 - testFailOnWarningsFailInHelper()
+  ---
+  # My cabbage! :(
+  at: tst_Warnings::testFailOnWarningsFailInHelper() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+  line: 0
+  ...
+not ok 17 - testFailOnWarningsThenSkip()
+  ---
+  # Received a warning that resulted in a failure:
+Ran out of cabbage!
+  at: tst_Warnings::testFailOnWarningsThenSkip() (qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+  line: 0
+  ...
+ok 18 - testFailOnWarningsThenSkip() # SKIP My cabbage! :(
+ok 19 - testFailOnWarningsAndIgnoreWarnings()
+ok 20 - cleanupTestCase()
+1..20
+# tests 20
+# pass 5
+# fail 14
diff --git a/tests/auto/testlib/selftests/expected_warnings.teamcity b/tests/auto/testlib/selftests/expected_warnings.teamcity
index 5811beae4c..e488ce7b30 100644
--- a/tests/auto/testlib/selftests/expected_warnings.teamcity
+++ b/tests/auto/testlib/selftests/expected_warnings.teamcity
@@ -20,6 +20,42 @@
 ##teamcity[testFailed name='testMissingWarningsWithData(second row)' message='Failure!' details='Not all expected messages were received' flowId='tst_Warnings']
 ##teamcity[testStdOut name='testMissingWarningsWithData(second row)' out='INFO: Did not receive message: "Warning0"|nINFO: Did not receive message: "Warning1"' flowId='tst_Warnings']
 ##teamcity[testFinished name='testMissingWarningsWithData(second row)' flowId='tst_Warnings']
+##teamcity[testStarted name='testFailOnWarnings()' flowId='tst_Warnings']
+##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings']
+##teamcity[testStdOut name='testFailOnWarnings()' out='QWARN: Ran out of space!' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings']
+##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of biscuits!' flowId='tst_Warnings']
+##teamcity[testStdOut name='testFailOnWarnings()' out='QDEBUG: Ran out of tortillas!|nQINFO: Ran out of oil!|nQWARN: nope' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings']
+##teamcity[testFailed name='testFailOnWarnings()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRunning low on toothpaste!' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarnings()' flowId='tst_Warnings']
+##teamcity[testStarted name='testFailOnWarningsCleared()' flowId='tst_Warnings']
+##teamcity[testStdOut name='testFailOnWarningsCleared()' out='QWARN: Running low on flour!|nQDEBUG: Running low on toothpaste!|nQINFO: Running low on toothpaste!|nQWARN: Ran out of muffins!' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarningsCleared()' flowId='tst_Warnings']
+##teamcity[testStarted name='testFailOnWarningsWithData(warning1)' flowId='tst_Warnings']
+##teamcity[testFailed name='testFailOnWarningsWithData(warning1)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning1' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarningsWithData(warning1)' flowId='tst_Warnings']
+##teamcity[testStarted name='testFailOnWarningsWithData(warning2)' flowId='tst_Warnings']
+##teamcity[testFailed name='testFailOnWarningsWithData(warning2)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning2' flowId='tst_Warnings']
+##teamcity[testStdOut name='testFailOnWarningsWithData(warning2)' out='QWARN: warning2|nQWARN: warning3|nQWARN: warning1' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarningsWithData(warning2)' flowId='tst_Warnings']
+##teamcity[testStarted name='testFailOnWarningsWithData(warning3)' flowId='tst_Warnings']
+##teamcity[testFailed name='testFailOnWarningsWithData(warning3)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nwarning3' flowId='tst_Warnings']
+##teamcity[testStdOut name='testFailOnWarningsWithData(warning3)' out='QWARN: warning3|nQWARN: warning1|nQWARN: warning2' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarningsWithData(warning3)' flowId='tst_Warnings']
+##teamcity[testStarted name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
+##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='This failure message should be printed but not cause the test to abort' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
+##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
+##teamcity[testFailed name='testFailOnWarningsFailInHelper()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='My cabbage! :(' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarningsFailInHelper()' flowId='tst_Warnings']
+##teamcity[testStarted name='testFailOnWarningsThenSkip()' flowId='tst_Warnings']
+##teamcity[testFailed name='testFailOnWarningsThenSkip()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' details='Received a warning that resulted in a failure:|nRan out of cabbage!' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarningsThenSkip()' flowId='tst_Warnings']
+##teamcity[testIgnored name='testFailOnWarningsThenSkip()' message='My cabbage! :( |[Loc: qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)|]' flowId='tst_Warnings']
+##teamcity[testStarted name='testFailOnWarningsAndIgnoreWarnings()' flowId='tst_Warnings']
+##teamcity[testFinished name='testFailOnWarningsAndIgnoreWarnings()' flowId='tst_Warnings']
 ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Warnings']
 ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Warnings']
 ##teamcity[testSuiteFinished name='tst_Warnings' flowId='tst_Warnings']
diff --git a/tests/auto/testlib/selftests/expected_warnings.txt b/tests/auto/testlib/selftests/expected_warnings.txt
index 0a65213b79..02361eb39d 100644
--- a/tests/auto/testlib/selftests/expected_warnings.txt
+++ b/tests/auto/testlib/selftests/expected_warnings.txt
@@ -23,6 +23,52 @@ FAIL!  : tst_Warnings::testMissingWarningsWithData(first row) Not all expected m
 INFO   : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning0"
 INFO   : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning1"
 FAIL!  : tst_Warnings::testMissingWarningsWithData(second row) Not all expected messages were received
+QWARN  : tst_Warnings::testFailOnWarnings() Ran out of space!
+FAIL!  : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure:
+Ran out of cabbage!
+   Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
+QDEBUG : tst_Warnings::testFailOnWarnings() Ran out of tortillas!
+QINFO  : tst_Warnings::testFailOnWarnings() Ran out of oil!
+QWARN  : tst_Warnings::testFailOnWarnings() nope
+FAIL!  : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure:
+Ran out of biscuits!
+   Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
+FAIL!  : tst_Warnings::testFailOnWarnings() Received a warning that resulted in a failure:
+Running low on toothpaste!
+   Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
+QWARN  : tst_Warnings::testFailOnWarnings() Running low on flour!
+QDEBUG : tst_Warnings::testFailOnWarnings() Running low on toothpaste!
+QINFO  : tst_Warnings::testFailOnWarnings() Running low on toothpaste!
+QWARN  : tst_Warnings::testFailOnWarningsCleared() Ran out of muffins!
+PASS   : tst_Warnings::testFailOnWarningsCleared()
+FAIL!  : tst_Warnings::testFailOnWarningsWithData(warning1) Received a warning that resulted in a failure:
+warning1
+   Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
+QWARN  : tst_Warnings::testFailOnWarningsWithData(warning1) warning2
+QWARN  : tst_Warnings::testFailOnWarningsWithData(warning1) warning3
+QWARN  : tst_Warnings::testFailOnWarningsWithData(warning2) warning1
+FAIL!  : tst_Warnings::testFailOnWarningsWithData(warning2) Received a warning that resulted in a failure:
+warning2
+   Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
+QWARN  : tst_Warnings::testFailOnWarningsWithData(warning2) warning3
+QWARN  : tst_Warnings::testFailOnWarningsWithData(warning3) warning1
+QWARN  : tst_Warnings::testFailOnWarningsWithData(warning3) warning2
+FAIL!  : tst_Warnings::testFailOnWarningsWithData(warning3) Received a warning that resulted in a failure:
+warning3
+   Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
+FAIL!  : tst_Warnings::testFailOnWarningsFailInHelper() This failure message should be printed but not cause the test to abort
+   Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
+FAIL!  : tst_Warnings::testFailOnWarningsFailInHelper() Received a warning that resulted in a failure:
+Ran out of cabbage!
+   Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
+FAIL!  : tst_Warnings::testFailOnWarningsFailInHelper() My cabbage! :(
+   Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
+FAIL!  : tst_Warnings::testFailOnWarningsThenSkip() Received a warning that resulted in a failure:
+Ran out of cabbage!
+   Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
+SKIP   : tst_Warnings::testFailOnWarningsThenSkip() My cabbage! :(
+   Loc: [qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp(0)]
+PASS   : tst_Warnings::testFailOnWarningsAndIgnoreWarnings()
 PASS   : tst_Warnings::cleanupTestCase()
-Totals: 3 passed, 4 failed, 0 skipped, 0 blacklisted, 0ms
+Totals: 5 passed, 14 failed, 1 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_Warnings *********
diff --git a/tests/auto/testlib/selftests/expected_warnings.xml b/tests/auto/testlib/selftests/expected_warnings.xml
index 1b17dbbf5d..aab6a99f74 100644
--- a/tests/auto/testlib/selftests/expected_warnings.xml
+++ b/tests/auto/testlib/selftests/expected_warnings.xml
@@ -91,6 +91,118 @@
 </Incident>
     <Duration msecs="0"/>
 </TestFunction>
+<TestFunction name="testFailOnWarnings">
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[Ran out of space!]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of cabbage!]]></Description>
+</Incident>
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[Ran out of tortillas!]]></Description>
+</Message>
+<Message type="qinfo" file="" line="0">
+    <Description><![CDATA[Ran out of oil!]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[nope]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of biscuits!]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Running low on toothpaste!]]></Description>
+</Incident>
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[Running low on flour!]]></Description>
+</Message>
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[Running low on toothpaste!]]></Description>
+</Message>
+<Message type="qinfo" file="" line="0">
+    <Description><![CDATA[Running low on toothpaste!]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsCleared">
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[Ran out of muffins!]]></Description>
+</Message>
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsWithData">
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <DataTag><![CDATA[warning1]]></DataTag>
+    <Description><![CDATA[Received a warning that resulted in a failure:
+warning1]]></Description>
+</Incident>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning1]]></DataTag>
+    <Description><![CDATA[warning2]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning1]]></DataTag>
+    <Description><![CDATA[warning3]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning2]]></DataTag>
+    <Description><![CDATA[warning1]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <DataTag><![CDATA[warning2]]></DataTag>
+    <Description><![CDATA[Received a warning that resulted in a failure:
+warning2]]></Description>
+</Incident>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning2]]></DataTag>
+    <Description><![CDATA[warning3]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning3]]></DataTag>
+    <Description><![CDATA[warning1]]></Description>
+</Message>
+<Message type="qwarn" file="" line="0">
+    <DataTag><![CDATA[warning3]]></DataTag>
+    <Description><![CDATA[warning2]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <DataTag><![CDATA[warning3]]></DataTag>
+    <Description><![CDATA[Received a warning that resulted in a failure:
+warning3]]></Description>
+</Incident>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsFailInHelper">
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[This failure message should be printed but not cause the test to abort]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of cabbage!]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[My cabbage! :(]]></Description>
+</Incident>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsThenSkip">
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[Received a warning that resulted in a failure:
+Ran out of cabbage!]]></Description>
+</Incident>
+<Message type="skip" file="qtbase/tests/auto/testlib/selftests/warnings/tst_warnings.cpp" line="0">
+    <Description><![CDATA[My cabbage! :(]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
+<TestFunction name="testFailOnWarningsAndIgnoreWarnings">
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0"/>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Incident type="pass" file="" line="0" />
     <Duration msecs="0"/>
diff --git a/tests/auto/testlib/selftests/warnings/tst_warnings.cpp b/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
index 61f090c2c3..ee73bd4fe4 100644
--- a/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+++ b/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
@@ -40,6 +40,16 @@ private slots:
     void testMissingWarningsRegularExpression();
     void testMissingWarningsWithData_data();
     void testMissingWarningsWithData();
+
+    void testFailOnWarnings();
+    void testFailOnWarningsCleared();
+#if QT_CONFIG(regularexpression)
+    void testFailOnWarningsWithData_data();
+    void testFailOnWarningsWithData();
+    void testFailOnWarningsFailInHelper();
+    void testFailOnWarningsThenSkip();
+#endif
+    void testFailOnWarningsAndIgnoreWarnings();
 };
 
 void tst_Warnings::testWarnings()
@@ -124,6 +134,99 @@ void tst_Warnings::testMissingWarningsWithData()
     qWarning("Warning2");
 }
 
+void tst_Warnings::testFailOnWarnings()
+{
+    // failOnWarnings() wasn't called yet; shouldn't fail;
+    qWarning("Ran out of space!");
+
+#if QT_CONFIG(regularexpression)
+    const auto warnRegex = QRegularExpression("Ran out of .*!");
+    QTest::failOnWarning(warnRegex);
+    // Should now fail.
+    qWarning("Ran out of cabbage!");
+
+    // Should not fail; none of these are warnings.
+    qDebug("Ran out of tortillas!");
+    qInfo("Ran out of oil!");
+
+    // Should not fail; regex doesn't match.
+    qWarning("nope");
+
+    // Should fail; matches regex.
+    qWarning("Ran out of biscuits!");
+#endif // QT_CONFIG(regularexpression)
+
+    QTest::failOnWarning("Running low on toothpaste!");
+
+    // Should fail; strings match.
+    qWarning("Running low on toothpaste!");
+
+    // Shouldn't fail; strings don't match.
+    qWarning("Running low on flour!");
+
+    // Should not fail; none of these are warnings.
+    qDebug("Running low on toothpaste!");
+    qInfo("Running low on toothpaste!");
+}
+
+void tst_Warnings::testFailOnWarningsCleared()
+{
+    // The patterns passed to failOnWarnings() should be cleared at the end of
+    // each test function, so this shouldn't fail because of the failOnWarning() call in the previous function.
+    // Note that this test always needs to come after testFailOnWarnings for it to work.
+    qWarning("Ran out of muffins!");
+}
+
+#if QT_CONFIG(regularexpression)
+void tst_Warnings::testFailOnWarningsWithData_data()
+{
+    // The warning message that should cause a failure.
+    QTest::addColumn<QString>("warningMessage");
+
+    QTest::newRow("warning1") << "warning1";
+    QTest::newRow("warning2") << "warning2";
+    QTest::newRow("warning3") << "warning3";
+}
+
+void tst_Warnings::testFailOnWarningsWithData()
+{
+    QFETCH(QString, warningMessage);
+
+    QTest::failOnWarning(QRegularExpression(warningMessage));
+
+    // Only one of these should fail, depending on warningMessage.
+    qWarning("warning1");
+    qWarning("warning2");
+    qWarning("warning3");
+}
+
+void tst_Warnings::testFailOnWarningsFailInHelper()
+{
+    [](){ QFAIL("This failure message should be printed but not cause the test to abort"); }();
+    const auto warnRegex = QRegularExpression("Ran out of .*!");
+    QTest::failOnWarning(warnRegex);
+    qWarning("Ran out of cabbage!");
+    QFAIL("My cabbage! :(");
+}
+
+void tst_Warnings::testFailOnWarningsThenSkip()
+{
+    const auto warnRegex = QRegularExpression("Ran out of .*!");
+    QTest::failOnWarning(warnRegex);
+    qWarning("Ran out of cabbage!");
+    QSKIP("My cabbage! :(");
+}
+#endif // QT_CONFIG(regularexpression)
+
+void tst_Warnings::testFailOnWarningsAndIgnoreWarnings()
+{
+    const auto warningStr = "Running low on toothpaste!";
+    QTest::failOnWarning(warningStr);
+    QTest::ignoreMessage(QtWarningMsg, warningStr);
+    // Shouldn't fail; we ignored it.
+    qWarning(warningStr);
+}
+
 QTEST_MAIN(tst_Warnings)
 
 #include "tst_warnings.moc"