Add a repeated-failure selftest for testlib

This mimics the effects of test-code delegating checks to other
functions, which can lead to repeated failures. The failing test
should only contribute one to the total of failed tsts.  Drive-by:
correct an existing test's reporting of its own name.

Task-number: QTBUG-95661
Change-Id: I370fd2aee378d9fdd35826f0d11677483c60423d
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>

7 files changed, 263 insertions, 33 deletions

diff --git a/tests/auto/testlib/selftests/expected_subtest.junitxml b/tests/auto/testlib/selftests/expected_subtest.junitxml
index 72b2c5c6b5..353f739985 100644
--- a/tests/auto/testlib/selftests/expected_subtest.junitxml
+++ b/tests/auto/testlib/selftests/expected_subtest.junitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite name="tst_Subtest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="9" failures="2" errors="0" skipped="0" time="@TEST_DURATION@">
+<testsuite name="tst_Subtest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="10" failures="3" errors="0" skipped="0" time="@TEST_DURATION@">
   <properties>
     <property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
     <property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
@@ -48,8 +48,8 @@
       <![CDATA[test3_data test3 (null)]]>
       <![CDATA[test3_data end]]>
       <![CDATA[init test3 data0]]>
-      <![CDATA[test2 test3 data0]]>
-      <![CDATA[test2 end]]>
+      <![CDATA[test3 test3 data0]]>
+      <![CDATA[test3 end]]>
       <![CDATA[cleanup test3 data0]]>
     </system-out>
   </testcase>
@@ -60,7 +60,7 @@
     </failure>
     <system-out>
       <![CDATA[init test3 data1]]>
-      <![CDATA[test2 test3 data1]]>
+      <![CDATA[test3 test3 data1]]>
       <![CDATA[cleanup test3 data1]]>
     </system-out>
   </testcase>
@@ -71,10 +71,17 @@
     </failure>
     <system-out>
       <![CDATA[init test3 data2]]>
-      <![CDATA[test2 test3 data2]]>
+      <![CDATA[test3 test3 data2]]>
       <![CDATA[cleanup test3 data2]]>
     </system-out>
   </testcase>
+  <testcase name="multiFail" classname="tst_Subtest" time="@TEST_DURATION@">
+    <failure type="fail" message="This failure message should be repeated ten times"/>
+    <system-out>
+      <![CDATA[init multiFail (null)]]>
+      <![CDATA[cleanup multiFail (null)]]>
+    </system-out>
+  </testcase>
   <testcase name="cleanupTestCase" classname="tst_Subtest" time="@TEST_DURATION@">
     <system-out>
       <![CDATA[cleanupTestCase cleanupTestCase (null)]]>
diff --git a/tests/auto/testlib/selftests/expected_subtest.lightxml b/tests/auto/testlib/selftests/expected_subtest.lightxml
index f35f6a6dc4..2c98c13cda 100644
--- a/tests/auto/testlib/selftests/expected_subtest.lightxml
+++ b/tests/auto/testlib/selftests/expected_subtest.lightxml
@@ -102,11 +102,11 @@
 </Message>
 <Message type="qdebug" file="" line="0">
     <DataTag><![CDATA[data0]]></DataTag>
-    <Description><![CDATA[test2 test3 data0]]></Description>
+    <Description><![CDATA[test3 test3 data0]]></Description>
 </Message>
 <Message type="qdebug" file="" line="0">
     <DataTag><![CDATA[data0]]></DataTag>
-    <Description><![CDATA[test2 end]]></Description>
+    <Description><![CDATA[test3 end]]></Description>
 </Message>
 <Message type="qdebug" file="" line="0">
     <DataTag><![CDATA[data0]]></DataTag>
@@ -121,7 +121,7 @@
 </Message>
 <Message type="qdebug" file="" line="0">
     <DataTag><![CDATA[data1]]></DataTag>
-    <Description><![CDATA[test2 test3 data1]]></Description>
+    <Description><![CDATA[test3 test3 data1]]></Description>
 </Message>
 <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
     <DataTag><![CDATA[data1]]></DataTag>
@@ -139,7 +139,7 @@
 </Message>
 <Message type="qdebug" file="" line="0">
     <DataTag><![CDATA[data2]]></DataTag>
-    <Description><![CDATA[test2 test3 data2]]></Description>
+    <Description><![CDATA[test3 test3 data2]]></Description>
 </Message>
 <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
     <DataTag><![CDATA[data2]]></DataTag>
@@ -153,6 +153,48 @@
 </Message>
     <Duration msecs="0"/>
 </TestFunction>
+<TestFunction name="multiFail">
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[init multiFail (null)]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[But this test should only contribute one to the failure count]]></Description>
+</Incident>
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[cleanup multiFail (null)]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Message type="qdebug" file="" line="0">
     <Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description>
diff --git a/tests/auto/testlib/selftests/expected_subtest.tap b/tests/auto/testlib/selftests/expected_subtest.tap
index 395eb2776c..ebbc1d1067 100644
--- a/tests/auto/testlib/selftests/expected_subtest.tap
+++ b/tests/auto/testlib/selftests/expected_subtest.tap
@@ -26,12 +26,12 @@ ok 5 - test2(data2)
 # test3_data test3 (null)
 # test3_data end
 # init test3 data0
-# test2 test3 data0
-# test2 end
+# test3 test3 data0
+# test3 end
 # cleanup test3 data0
 ok 6 - test3(data0)
 # init test3 data1
-# test2 test3 data1
+# test3 test3 data1
 not ok 7 - test3(data1)
   ---
   type: QCOMPARE
@@ -46,7 +46,7 @@ not ok 7 - test3(data1)
   ...
 # cleanup test3 data1
 # init test3 data2
-# test2 test3 data2
+# test3 test3 data2
 not ok 8 - test3(data2)
   ---
   type: QCOMPARE
@@ -60,9 +60,88 @@ not ok 8 - test3(data2)
   line: 0
   ...
 # cleanup test3 data2
+# init multiFail (null)
+not ok 9 - multiFail()
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+  line: 0
+  ...
+not ok 10 - multiFail()
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+  line: 0
+  ...
+not ok 11 - multiFail()
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+  line: 0
+  ...
+not ok 12 - multiFail()
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+  line: 0
+  ...
+not ok 13 - multiFail()
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+  line: 0
+  ...
+not ok 14 - multiFail()
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+  line: 0
+  ...
+not ok 15 - multiFail()
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+  line: 0
+  ...
+not ok 16 - multiFail()
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+  line: 0
+  ...
+not ok 17 - multiFail()
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+  line: 0
+  ...
+not ok 18 - multiFail()
+  ---
+  # This failure message should be repeated ten times
+  at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+  line: 0
+  ...
+not ok 19 - multiFail()
+  ---
+  # But this test should only contribute one to the failure count
+  at: tst_Subtest::multiFail() (qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp:0)
+  file: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+  line: 0
+  ...
+# cleanup multiFail (null)
 # cleanupTestCase cleanupTestCase (null)
-ok 9 - cleanupTestCase()
-1..9
-# tests 9
+ok 20 - cleanupTestCase()
+1..20
+# tests 20
 # pass 7
-# fail 2
+# fail 13
diff --git a/tests/auto/testlib/selftests/expected_subtest.teamcity b/tests/auto/testlib/selftests/expected_subtest.teamcity
index 3fcbcb7ea4..580e5695dd 100644
--- a/tests/auto/testlib/selftests/expected_subtest.teamcity
+++ b/tests/auto/testlib/selftests/expected_subtest.teamcity
@@ -15,17 +15,41 @@
 ##teamcity[testStdOut name='test2(data2)' out='QDEBUG: init test2 data2|nQDEBUG: test2 test2 data2|nQDEBUG: test2 end|nQDEBUG: cleanup test2 data2' flowId='tst_Subtest']
 ##teamcity[testFinished name='test2(data2)' flowId='tst_Subtest']
 ##teamcity[testStarted name='test3(data0)' flowId='tst_Subtest']
-##teamcity[testStdOut name='test3(data0)' out='QDEBUG: test3_data test3 (null)|nQDEBUG: test3_data end|nQDEBUG: init test3 data0|nQDEBUG: test2 test3 data0|nQDEBUG: test2 end|nQDEBUG: cleanup test3 data0' flowId='tst_Subtest']
+##teamcity[testStdOut name='test3(data0)' out='QDEBUG: test3_data test3 (null)|nQDEBUG: test3_data end|nQDEBUG: init test3 data0|nQDEBUG: test3 test3 data0|nQDEBUG: test3 end|nQDEBUG: cleanup test3 data0' flowId='tst_Subtest']
 ##teamcity[testFinished name='test3(data0)' flowId='tst_Subtest']
 ##teamcity[testStarted name='test3(data1)' flowId='tst_Subtest']
 ##teamcity[testFailed name='test3(data1)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='Compared values are not the same|n Actual (str) : "hello1"|n Expected (QString("hello0")): "hello0"' flowId='tst_Subtest']
-##teamcity[testStdOut name='test3(data1)' out='QDEBUG: init test3 data1|nQDEBUG: test2 test3 data1' flowId='tst_Subtest']
+##teamcity[testStdOut name='test3(data1)' out='QDEBUG: init test3 data1|nQDEBUG: test3 test3 data1' flowId='tst_Subtest']
 ##teamcity[testFinished name='test3(data1)' flowId='tst_Subtest']
 ##teamcity[testStarted name='test3(data2)' flowId='tst_Subtest']
 ##teamcity[testFailed name='test3(data2)' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='Compared values are not the same|n Actual (str) : "hello2"|n Expected (QString("hello0")): "hello0"' flowId='tst_Subtest']
-##teamcity[testStdOut name='test3(data2)' out='QDEBUG: cleanup test3 data1|nQDEBUG: init test3 data2|nQDEBUG: test2 test3 data2' flowId='tst_Subtest']
+##teamcity[testStdOut name='test3(data2)' out='QDEBUG: cleanup test3 data1|nQDEBUG: init test3 data2|nQDEBUG: test3 test3 data2' flowId='tst_Subtest']
 ##teamcity[testFinished name='test3(data2)' flowId='tst_Subtest']
+##teamcity[testStarted name='multiFail()' flowId='tst_Subtest']
+##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
+##teamcity[testStdOut name='multiFail()' out='QDEBUG: cleanup test3 data2|nQDEBUG: init multiFail (null)' flowId='tst_Subtest']
+##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
+##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
+##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
+##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
+##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
+##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
+##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
+##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
+##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
+##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
+##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
+##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
+##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
+##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
+##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
+##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
+##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
+##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='This failure message should be repeated ten times' flowId='tst_Subtest']
+##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
+##teamcity[testFailed name='multiFail()' message='Failure! |[Loc: qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)|]' details='But this test should only contribute one to the failure count' flowId='tst_Subtest']
+##teamcity[testFinished name='multiFail()' flowId='tst_Subtest']
 ##teamcity[testStarted name='cleanupTestCase()' flowId='tst_Subtest']
-##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup test3 data2|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest']
+##teamcity[testStdOut name='cleanupTestCase()' out='QDEBUG: cleanup multiFail (null)|nQDEBUG: cleanupTestCase cleanupTestCase (null)' flowId='tst_Subtest']
 ##teamcity[testFinished name='cleanupTestCase()' flowId='tst_Subtest']
 ##teamcity[testSuiteFinished name='tst_Subtest' flowId='tst_Subtest']
diff --git a/tests/auto/testlib/selftests/expected_subtest.txt b/tests/auto/testlib/selftests/expected_subtest.txt
index 8b0dbf63ee..d35d7b4d24 100644
--- a/tests/auto/testlib/selftests/expected_subtest.txt
+++ b/tests/auto/testlib/selftests/expected_subtest.txt
@@ -26,25 +26,49 @@ PASS   : tst_Subtest::test2(data2)
 QDEBUG : tst_Subtest::test3() test3_data test3 (null)
 QDEBUG : tst_Subtest::test3() test3_data end
 QDEBUG : tst_Subtest::test3(data0) init test3 data0
-QDEBUG : tst_Subtest::test3(data0) test2 test3 data0
-QDEBUG : tst_Subtest::test3(data0) test2 end
+QDEBUG : tst_Subtest::test3(data0) test3 test3 data0
+QDEBUG : tst_Subtest::test3(data0) test3 end
 QDEBUG : tst_Subtest::test3(data0) cleanup test3 data0
 PASS   : tst_Subtest::test3(data0)
 QDEBUG : tst_Subtest::test3(data1) init test3 data1
-QDEBUG : tst_Subtest::test3(data1) test2 test3 data1
+QDEBUG : tst_Subtest::test3(data1) test3 test3 data1
 FAIL!  : tst_Subtest::test3(data1) Compared values are not the same
    Actual   (str)              : "hello1"
    Expected (QString("hello0")): "hello0"
    Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
 QDEBUG : tst_Subtest::test3(data1) cleanup test3 data1
 QDEBUG : tst_Subtest::test3(data2) init test3 data2
-QDEBUG : tst_Subtest::test3(data2) test2 test3 data2
+QDEBUG : tst_Subtest::test3(data2) test3 test3 data2
 FAIL!  : tst_Subtest::test3(data2) Compared values are not the same
    Actual   (str)              : "hello2"
    Expected (QString("hello0")): "hello0"
    Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
 QDEBUG : tst_Subtest::test3(data2) cleanup test3 data2
+QDEBUG : tst_Subtest::multiFail() init multiFail (null)
+FAIL!  : tst_Subtest::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+FAIL!  : tst_Subtest::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+FAIL!  : tst_Subtest::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+FAIL!  : tst_Subtest::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+FAIL!  : tst_Subtest::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+FAIL!  : tst_Subtest::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+FAIL!  : tst_Subtest::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+FAIL!  : tst_Subtest::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+FAIL!  : tst_Subtest::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+FAIL!  : tst_Subtest::multiFail() This failure message should be repeated ten times
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+FAIL!  : tst_Subtest::multiFail() But this test should only contribute one to the failure count
+   Loc: [qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp(0)]
+QDEBUG : tst_Subtest::multiFail() cleanup multiFail (null)
 QDEBUG : tst_Subtest::cleanupTestCase() cleanupTestCase cleanupTestCase (null)
 PASS   : tst_Subtest::cleanupTestCase()
-Totals: 7 passed, 2 failed, 0 skipped, 0 blacklisted, 0ms
+Totals: 7 passed, 13 failed, 0 skipped, 0 blacklisted, 0ms
 ********* Finished testing of tst_Subtest *********
diff --git a/tests/auto/testlib/selftests/expected_subtest.xml b/tests/auto/testlib/selftests/expected_subtest.xml
index 793f5d4d2d..1f3867de3a 100644
--- a/tests/auto/testlib/selftests/expected_subtest.xml
+++ b/tests/auto/testlib/selftests/expected_subtest.xml
@@ -104,11 +104,11 @@
 </Message>
 <Message type="qdebug" file="" line="0">
     <DataTag><![CDATA[data0]]></DataTag>
-    <Description><![CDATA[test2 test3 data0]]></Description>
+    <Description><![CDATA[test3 test3 data0]]></Description>
 </Message>
 <Message type="qdebug" file="" line="0">
     <DataTag><![CDATA[data0]]></DataTag>
-    <Description><![CDATA[test2 end]]></Description>
+    <Description><![CDATA[test3 end]]></Description>
 </Message>
 <Message type="qdebug" file="" line="0">
     <DataTag><![CDATA[data0]]></DataTag>
@@ -123,7 +123,7 @@
 </Message>
 <Message type="qdebug" file="" line="0">
     <DataTag><![CDATA[data1]]></DataTag>
-    <Description><![CDATA[test2 test3 data1]]></Description>
+    <Description><![CDATA[test3 test3 data1]]></Description>
 </Message>
 <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
     <DataTag><![CDATA[data1]]></DataTag>
@@ -141,7 +141,7 @@
 </Message>
 <Message type="qdebug" file="" line="0">
     <DataTag><![CDATA[data2]]></DataTag>
-    <Description><![CDATA[test2 test3 data2]]></Description>
+    <Description><![CDATA[test3 test3 data2]]></Description>
 </Message>
 <Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
     <DataTag><![CDATA[data2]]></DataTag>
@@ -155,6 +155,48 @@
 </Message>
     <Duration msecs="0"/>
 </TestFunction>
+<TestFunction name="multiFail">
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[init multiFail (null)]]></Description>
+</Message>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[This failure message should be repeated ten times]]></Description>
+</Incident>
+<Incident type="fail" file="qtbase/tests/auto/testlib/selftests/subtest/tst_subtest.cpp" line="0">
+    <Description><![CDATA[But this test should only contribute one to the failure count]]></Description>
+</Incident>
+<Message type="qdebug" file="" line="0">
+    <Description><![CDATA[cleanup multiFail (null)]]></Description>
+</Message>
+    <Duration msecs="0"/>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Message type="qdebug" file="" line="0">
     <Description><![CDATA[cleanupTestCase cleanupTestCase (null)]]></Description>
diff --git a/tests/auto/testlib/selftests/subtest/tst_subtest.cpp b/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
index 0837e70233..416cf80aa8 100644
--- a/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
+++ b/tests/auto/testlib/selftests/subtest/tst_subtest.cpp
@@ -1,6 +1,6 @@
 /****************************************************************************
 **
-** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2021 The Qt Company Ltd.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the test suite of the Qt Toolkit.
@@ -47,6 +47,8 @@ private slots:
     void test2();
     void test3_data();
     void test3();
+
+    void multiFail();
 };
 
 
@@ -131,7 +133,7 @@ void tst_Subtest::test3_data()
 
 void tst_Subtest::test3()
 {
-    qDebug() << "test2"
+    qDebug() << "test3"
              << (QTest::currentTestFunction() ? QTest::currentTestFunction() : "(null)")
              << (QTest::currentDataTag() ? QTest::currentDataTag() : "(null)");
 
@@ -140,7 +142,17 @@ void tst_Subtest::test3()
     // second and third time we call this it should FAIL
     QCOMPARE(str, QString("hello0"));
 
-    qDebug() << "test2 end";
+    qDebug() << "test3 end";
+}
+
+void tst_Subtest::multiFail()
+{
+    // Simulates tests which call a shared function that does common checks, or
+    // that do checks in code run asynchronously from a messae loop.
+    for (int i = 0; i < 10; ++i)
+        []() { QFAIL("This failure message should be repeated ten times"); }();
+    // FIXME QTBUG-95661: it gets counted as eleven failures, of course.
+    QFAIL("But this test should only contribute one to the failure count");
 }
 
 QTEST_MAIN(tst_Subtest)