Add selftest for xpass in a data-driven test.

Change-Id: I9491d1e17ad31453ec353a11c9efb31522a16459
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>

5 files changed, 71 insertions, 31 deletions

diff --git a/tests/auto/testlib/selftests/expected_expectfail.lightxml b/tests/auto/testlib/selftests/expected_expectfail.lightxml
index 76c8e31e4f..02e4dde7d8 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.lightxml
+++ b/tests/auto/testlib/selftests/expected_expectfail.lightxml
@@ -9,7 +9,7 @@
 <Message type="qdebug" file="" line="0">
     <Description><![CDATA[begin]]></Description>
 </Message>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="70">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="72">
     <Description><![CDATA[This should xfail]]></Description>
 </Incident>
 <Message type="qdebug" file="" line="0">
@@ -21,31 +21,31 @@
 <Message type="qdebug" file="" line="0">
     <Description><![CDATA[begin]]></Description>
 </Message>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="78">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="80">
     <Description><![CDATA[This should xfail]]></Description>
 </Incident>
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="xfailTwice">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="88">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="90">
     <Description><![CDATA[Already expecting a fail]]></Description>
 </Incident>
 </TestFunction>
 <TestFunction name="xfailWithQString">
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="97">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="99">
     <Description><![CDATA[A string]]></Description>
 </Incident>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="102">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="104">
     <Description><![CDATA[Bug 5 (The message)]]></Description>
 </Incident>
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="xfailDataDriven">
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="131">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="133">
     <DataTag><![CDATA[Abort]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="131">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="133">
     <DataTag><![CDATA[Continue]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>
@@ -55,18 +55,24 @@
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="xfailOnAnyRow">
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="166">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="168">
     <DataTag><![CDATA[first row]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="166">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="168">
     <DataTag><![CDATA[second row]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="xpass">
-<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="172">
+<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="174">
+    <Description><![CDATA['true' returned FALSE. ()]]></Description>
+</Incident>
+</TestFunction>
+<TestFunction name="xpassDataDriven">
+<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="196">
+    <DataTag><![CDATA[XPass]]></DataTag>
     <Description><![CDATA['true' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_expectfail.txt b/tests/auto/testlib/selftests/expected_expectfail.txt
index c3a38edbb0..1b98b6b561 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.txt
+++ b/tests/auto/testlib/selftests/expected_expectfail.txt
@@ -3,33 +3,35 @@ Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE
 PASS   : tst_ExpectFail::initTestCase()
 QDEBUG : tst_ExpectFail::xfailAndContinue() begin
 XFAIL  : tst_ExpectFail::xfailAndContinue() This should xfail
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(70)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(72)]
 QDEBUG : tst_ExpectFail::xfailAndContinue() after
 PASS   : tst_ExpectFail::xfailAndContinue()
 QDEBUG : tst_ExpectFail::xfailAndAbort() begin
 XFAIL  : tst_ExpectFail::xfailAndAbort() This should xfail
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(78)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(80)]
 PASS   : tst_ExpectFail::xfailAndAbort()
 FAIL!  : tst_ExpectFail::xfailTwice() Already expecting a fail
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(88)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(90)]
 XFAIL  : tst_ExpectFail::xfailWithQString() A string
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(97)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(99)]
 XFAIL  : tst_ExpectFail::xfailWithQString() Bug 5 (The message)
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(102)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(104)]
 PASS   : tst_ExpectFail::xfailWithQString()
 XFAIL  : tst_ExpectFail::xfailDataDriven(Abort) This test should xfail
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(131)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(133)]
 XFAIL  : tst_ExpectFail::xfailDataDriven(Continue) This test should xfail
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(131)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(133)]
 PASS   : tst_ExpectFail::xfailDataDriven()
 PASS   : tst_ExpectFail::xfailOnWrongRow()
 XFAIL  : tst_ExpectFail::xfailOnAnyRow(first row) This test should xfail
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(166)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(168)]
 XFAIL  : tst_ExpectFail::xfailOnAnyRow(second row) This test should xfail
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(166)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(168)]
 PASS   : tst_ExpectFail::xfailOnAnyRow()
 XPASS  : tst_ExpectFail::xpass() 'true' returned FALSE. ()
    Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(172)]
+XPASS  : tst_ExpectFail::xpassDataDriven(XPass) 'true' returned FALSE. ()
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(196)]
 PASS   : tst_ExpectFail::cleanupTestCase()
-Totals: 8 passed, 2 failed, 0 skipped
+Totals: 8 passed, 3 failed, 0 skipped
 ********* Finished testing of tst_ExpectFail *********
diff --git a/tests/auto/testlib/selftests/expected_expectfail.xml b/tests/auto/testlib/selftests/expected_expectfail.xml
index e195384727..74b11fb0e5 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.xml
+++ b/tests/auto/testlib/selftests/expected_expectfail.xml
@@ -11,7 +11,7 @@
 <Message type="qdebug" file="" line="0">
     <Description><![CDATA[begin]]></Description>
 </Message>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="70">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="72">
     <Description><![CDATA[This should xfail]]></Description>
 </Incident>
 <Message type="qdebug" file="" line="0">
@@ -23,31 +23,31 @@
 <Message type="qdebug" file="" line="0">
     <Description><![CDATA[begin]]></Description>
 </Message>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="78">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="80">
     <Description><![CDATA[This should xfail]]></Description>
 </Incident>
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="xfailTwice">
-<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="88">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="90">
     <Description><![CDATA[Already expecting a fail]]></Description>
 </Incident>
 </TestFunction>
 <TestFunction name="xfailWithQString">
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="97">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="99">
     <Description><![CDATA[A string]]></Description>
 </Incident>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="102">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="104">
     <Description><![CDATA[Bug 5 (The message)]]></Description>
 </Incident>
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="xfailDataDriven">
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="131">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="133">
     <DataTag><![CDATA[Abort]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="131">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="133">
     <DataTag><![CDATA[Continue]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>
@@ -57,18 +57,24 @@
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="xfailOnAnyRow">
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="166">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="168">
     <DataTag><![CDATA[first row]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="166">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="168">
     <DataTag><![CDATA[second row]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="xpass">
-<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="172">
+<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="174">
+    <Description><![CDATA['true' returned FALSE. ()]]></Description>
+</Incident>
+</TestFunction>
+<TestFunction name="xpassDataDriven">
+<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="196">
+    <DataTag><![CDATA[XPass]]></DataTag>
     <Description><![CDATA['true' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
diff --git a/tests/auto/testlib/selftests/expected_expectfail.xunitxml b/tests/auto/testlib/selftests/expected_expectfail.xunitxml
index a94e1eaf47..017e4501c1 100644
--- a/tests/auto/testlib/selftests/expected_expectfail.xunitxml
+++ b/tests/auto/testlib/selftests/expected_expectfail.xunitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite errors="11" failures="2" tests="10" name="tst_ExpectFail">
+<testsuite errors="11" failures="3" tests="11" name="tst_ExpectFail">
   <properties>
     <property value="5.0.0" name="QTestVersion"/>
     <property value="5.0.0" name="QtVersion"/>
@@ -33,6 +33,9 @@
   <testcase result="xpass" name="xpass">
     <failure message="&apos;true&apos; returned FALSE. ()" result="xpass"/>
   </testcase>
+  <testcase result="xpass" name="xpassDataDriven">
+    <failure tag="XPass" message="&apos;true&apos; returned FALSE. ()" result="xpass"/>
+  </testcase>
   <testcase result="pass" name="cleanupTestCase"/>
   <system-err>
 <![CDATA[begin]]>
diff --git a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
index 1fa3711990..3a000ba182 100644
--- a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+++ b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
@@ -61,6 +61,8 @@ private slots:
     void xfailOnAnyRow_data() const;
     void xfailOnAnyRow() const;
     void xpass() const;
+    void xpassDataDriven_data() const;
+    void xpassDataDriven() const;
 };
 
 void tst_ExpectFail::xfailAndContinue() const
@@ -176,5 +178,26 @@ void tst_ExpectFail::xpass() const
     QVERIFY2(false, "This should not be reached");
 }
 
+void tst_ExpectFail::xpassDataDriven_data() const
+{
+    QTest::addColumn<bool>("shouldXPass");
+
+    QTest::newRow("XPass")  << true;
+    QTest::newRow("Pass")   << false;
+}
+
+void tst_ExpectFail::xpassDataDriven() const
+{
+    QFETCH(bool, shouldXPass);
+
+    if (shouldXPass)
+        QEXPECT_FAIL(QTest::currentDataTag(), "This test should xpass", Abort);
+
+    QVERIFY(true);
+
+    // We should only get here if the test wasn't supposed to xpass.
+    QVERIFY2(!shouldXPass, "Test failed to terminate on XPASS");
+}
+
 QTEST_MAIN(tst_ExpectFail)
 #include "tst_expectfail.moc"