testlib: Clear ignored messages after every data row

Previously, ignored messages were only cleared at the end of each test
function, i.e. after all data rows were finished. This meant that if a
data row in a data-driven test function didn't cause all of the expected
messages to be generated, the remaining messages would be carried over
to the next data row.  This would result in errors about missing
messages being associated with the last data row rather than with the
correct data row.

This commit makes testlib check for missing ignored messages after
running each data row rather than only doing so after the last data
row.

This commit also adds a regression test to demonstrate that ignored
messages can no longer be carried over from one data row to another.

Change-Id: Ibee51aa6e96866fbcbcb4acee1a8340a86a6a4ba
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>

5 files changed, 91 insertions, 2 deletions

diff --git a/tests/auto/testlib/selftests/expected_warnings.lightxml b/tests/auto/testlib/selftests/expected_warnings.lightxml
index 116156ed8a..ad786832ca 100644
--- a/tests/auto/testlib/selftests/expected_warnings.lightxml
+++ b/tests/auto/testlib/selftests/expected_warnings.lightxml
@@ -37,6 +37,32 @@
     <Description><![CDATA[Not all expected messages were received]]></Description>
 </Incident>
 </TestFunction>
+<TestFunction name="testMissingWarningsWithData">
+<Message type="info" file="" line="0">
+    <DataTag><![CDATA[first row]]></DataTag>
+    <Description><![CDATA[Did not receive message: "Warning0"]]></Description>
+</Message>
+<Message type="info" file="" line="0">
+    <DataTag><![CDATA[first row]]></DataTag>
+    <Description><![CDATA[Did not receive message: "Warning1"]]></Description>
+</Message>
+<Incident type="fail" file="" line="0">
+    <DataTag><![CDATA[first row]]></DataTag>
+    <Description><![CDATA[Not all expected messages were received]]></Description>
+</Incident>
+<Message type="info" file="" line="0">
+    <DataTag><![CDATA[second row]]></DataTag>
+    <Description><![CDATA[Did not receive message: "Warning0"]]></Description>
+</Message>
+<Message type="info" file="" line="0">
+    <DataTag><![CDATA[second row]]></DataTag>
+    <Description><![CDATA[Did not receive message: "Warning1"]]></Description>
+</Message>
+<Incident type="fail" file="" line="0">
+    <DataTag><![CDATA[second row]]></DataTag>
+    <Description><![CDATA[Not all expected messages were received]]></Description>
+</Incident>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Incident type="pass" file="" line="0" />
 </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_warnings.txt b/tests/auto/testlib/selftests/expected_warnings.txt
index 6f7d0f491b..6196ec0261 100644
--- a/tests/auto/testlib/selftests/expected_warnings.txt
+++ b/tests/auto/testlib/selftests/expected_warnings.txt
@@ -11,6 +11,12 @@ PASS   : tst_Warnings::testWarnings()
 INFO   : tst_Warnings::testMissingWarnings() Did not receive message: "Warning0"
 INFO   : tst_Warnings::testMissingWarnings() Did not receive message: "Warning1"
 FAIL!  : tst_Warnings::testMissingWarnings() Not all expected messages were received
+INFO   : tst_Warnings::testMissingWarningsWithData(first row) Did not receive message: "Warning0"
+INFO   : tst_Warnings::testMissingWarningsWithData(first row) Did not receive message: "Warning1"
+FAIL!  : tst_Warnings::testMissingWarningsWithData(first row) Not all expected messages were received
+INFO   : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning0"
+INFO   : tst_Warnings::testMissingWarningsWithData(second row) Did not receive message: "Warning1"
+FAIL!  : tst_Warnings::testMissingWarningsWithData(second row) Not all expected messages were received
 PASS   : tst_Warnings::cleanupTestCase()
-Totals: 3 passed, 1 failed, 0 skipped
+Totals: 3 passed, 3 failed, 0 skipped
 ********* Finished testing of tst_Warnings *********
diff --git a/tests/auto/testlib/selftests/expected_warnings.xml b/tests/auto/testlib/selftests/expected_warnings.xml
index 13e6c1df79..12cff7c13d 100644
--- a/tests/auto/testlib/selftests/expected_warnings.xml
+++ b/tests/auto/testlib/selftests/expected_warnings.xml
@@ -39,6 +39,32 @@
     <Description><![CDATA[Not all expected messages were received]]></Description>
 </Incident>
 </TestFunction>
+<TestFunction name="testMissingWarningsWithData">
+<Message type="info" file="" line="0">
+    <DataTag><![CDATA[first row]]></DataTag>
+    <Description><![CDATA[Did not receive message: "Warning0"]]></Description>
+</Message>
+<Message type="info" file="" line="0">
+    <DataTag><![CDATA[first row]]></DataTag>
+    <Description><![CDATA[Did not receive message: "Warning1"]]></Description>
+</Message>
+<Incident type="fail" file="" line="0">
+    <DataTag><![CDATA[first row]]></DataTag>
+    <Description><![CDATA[Not all expected messages were received]]></Description>
+</Incident>
+<Message type="info" file="" line="0">
+    <DataTag><![CDATA[second row]]></DataTag>
+    <Description><![CDATA[Did not receive message: "Warning0"]]></Description>
+</Message>
+<Message type="info" file="" line="0">
+    <DataTag><![CDATA[second row]]></DataTag>
+    <Description><![CDATA[Did not receive message: "Warning1"]]></Description>
+</Message>
+<Incident type="fail" file="" line="0">
+    <DataTag><![CDATA[second row]]></DataTag>
+    <Description><![CDATA[Not all expected messages were received]]></Description>
+</Incident>
+</TestFunction>
 <TestFunction name="cleanupTestCase">
 <Incident type="pass" file="" line="0" />
 </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_warnings.xunitxml b/tests/auto/testlib/selftests/expected_warnings.xunitxml
index a96393d648..3e3b9ce18e 100644
--- a/tests/auto/testlib/selftests/expected_warnings.xunitxml
+++ b/tests/auto/testlib/selftests/expected_warnings.xunitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite errors="8" failures="1" tests="4" name="tst_Warnings">
+<testsuite errors="12" failures="3" tests="5" name="tst_Warnings">
   <properties>
     <property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
     <property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
@@ -18,6 +18,14 @@
     <!-- message="Did not receive message: &quot;Warning1&quot;" type="info" -->
     <failure message="Not all expected messages were received" result="fail"/>
   </testcase>
+  <testcase result="fail" name="testMissingWarningsWithData">
+    <!-- tag="first row" message="Did not receive message: &quot;Warning0&quot;" type="info" -->
+    <!-- tag="first row" message="Did not receive message: &quot;Warning1&quot;" type="info" -->
+    <failure tag="first row" message="Not all expected messages were received" result="fail"/>
+    <!-- tag="second row" message="Did not receive message: &quot;Warning0&quot;" type="info" -->
+    <!-- tag="second row" message="Did not receive message: &quot;Warning1&quot;" type="info" -->
+    <failure tag="second row" message="Not all expected messages were received" result="fail"/>
+  </testcase>
   <testcase result="pass" name="cleanupTestCase"/>
   <system-err>
 <![CDATA[Warning]]>
@@ -28,5 +36,9 @@
 <![CDATA[Baba]]>
 <![CDATA[Did not receive message: "Warning0"]]>
 <![CDATA[Did not receive message: "Warning1"]]>
+<![CDATA[Did not receive message: "Warning0"]]>
+<![CDATA[Did not receive message: "Warning1"]]>
+<![CDATA[Did not receive message: "Warning0"]]>
+<![CDATA[Did not receive message: "Warning1"]]>
   </system-err>
 </testsuite>
diff --git a/tests/auto/testlib/selftests/warnings/tst_warnings.cpp b/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
index 933d0cc6ce..e53869e18e 100644
--- a/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
+++ b/tests/auto/testlib/selftests/warnings/tst_warnings.cpp
@@ -49,6 +49,8 @@ class tst_Warnings: public QObject
 private slots:
     void testWarnings();
     void testMissingWarnings();
+    void testMissingWarningsWithData_data();
+    void testMissingWarningsWithData();
 };
 
 void tst_Warnings::testWarnings()
@@ -82,6 +84,23 @@ void tst_Warnings::testMissingWarnings()
     qWarning("Warning2");
 }
 
+void tst_Warnings::testMissingWarningsWithData_data()
+{
+    QTest::addColumn<int>("dummy");
+
+    QTest::newRow("first row") << 0;
+    QTest::newRow("second row") << 1;
+}
+
+void tst_Warnings::testMissingWarningsWithData()
+{
+    QTest::ignoreMessage(QtWarningMsg, "Warning0");
+    QTest::ignoreMessage(QtWarningMsg, "Warning1");
+    QTest::ignoreMessage(QtWarningMsg, "Warning2");
+
+    qWarning("Warning2");
+}
+
 QTEST_MAIN(tst_Warnings)
 
 #include "tst_warnings.moc"