qmltest: Count passes, fails and skips consistently.

This commit changes qmltest to count one pass per data row, rather than
a single pass for an entire data-driven test function.  This makes
counting of passes consistent with counting of fails and skips which
were already counted once per data row.  This change also means that the
plain-text, xml and light-xml output formats will now show one result
per data row, so that every data row executed will be shown in the test
output with either a pass, fail or skip result.  Previously data rows
that passed were not shown in the output, preventing analysis of total
number of tests runs, pass-rates, and various other metrics.

This commit corresponds to (and depends on) a commit in the qtbase
module which changes the behaviour of qtestlib in the same way.

Task-number: QTBUG-21848
Task-number: QTBUG-22124
Change-Id: I3c1f0e68bdff4087b9ccfc80a3f96f4541335b6f
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>
Reviewed-by: Yunqiao Yin <charles.yin@nokia.com>

4 files changed, 19 insertions, 21 deletions

diff --git a/src/imports/testlib/TestCase.qml b/src/imports/testlib/TestCase.qml
index 02fa095da2..1e08686b61 100644
--- a/src/imports/testlib/TestCase.qml
+++ b/src/imports/testlib/TestCase.qml
@@ -477,7 +477,7 @@ Item {
                              e.fileName, e.lineNumber)
             }
         }
-        return !qtest_results.dataFailed
+        return !qtest_results.failed
     }
 
     function qtest_runFunction(prop, arg) {
@@ -486,6 +486,7 @@ Item {
             qtest_runInternal(prop, arg)
             qtest_results.finishTestData()
             qtest_runInternal("cleanup")
+            qtest_results.finishTestDataCleanup()
         }
     }
 
@@ -515,6 +516,7 @@ Item {
 
                 // Run the cleanup function.
                 qtest_runInternal("cleanup")
+                qtest_results.finishTestDataCleanup()
             } while (!qtest_results.measurementAccepted())
             qtest_results.endDataRun()
         } while (qtest_results.needsMoreMeasurements())
@@ -566,6 +568,8 @@ Item {
         var runTests = true
         if (!qtest_runInternal("initTestCase"))
             runTests = false
+        qtest_results.finishTestData()
+        qtest_results.finishTestDataCleanup()
         qtest_results.finishTestFunction()
 
         // Run the test methods.
@@ -636,6 +640,8 @@ Item {
         // Clean up and exit.
         running = false
         completed = true
+        qtest_results.finishTestData()
+        qtest_results.finishTestDataCleanup()
         qtest_results.finishTestFunction()
         qtest_results.functionName = ""
 
diff --git a/src/qmltest/quicktest.cpp b/src/qmltest/quicktest.cpp
index 551311cd28..7130f9848b 100644
--- a/src/qmltest/quicktest.cpp
+++ b/src/qmltest/quicktest.cpp
@@ -173,6 +173,7 @@ template <class View> void handleCompileErrors(const QFileInfo &fi, const View &
     results.fail(errors.at(0).description(),
                  errors.at(0).url(), errors.at(0).line());
     results.finishTestData();
+    results.finishTestDataCleanup();
     results.finishTestFunction();
     results.setFunctionName(QString());
     results.stopLogging();
diff --git a/src/qmltest/quicktestresult.cpp b/src/qmltest/quicktestresult.cpp
index ddfa928f96..5654c87ad9 100644
--- a/src/qmltest/quicktestresult.cpp
+++ b/src/qmltest/quicktestresult.cpp
@@ -192,28 +192,15 @@ void QuickTestResult::setDataTag(const QString &tag)
 /*!
     \qmlproperty bool TestResult::failed
 
-    This property returns true if the current test function has
-    failed; false otherwise.  The fail state is reset when
-    functionName is changed or finishTestFunction() is called.
+    This property returns true if the current test function (or
+    current test data row for a data-driven test) has failed;
+    false otherwise.  The fail state is reset when functionName
+    is changed or finishTestDataCleanup() is called.
 
-    \sa skipped, dataFailed
+    \sa skipped
 */
 bool QuickTestResult::isFailed() const
 {
-    return QTestResult::testFailed();
-}
-
-/*!
-    \qmlproperty bool TestResult::dataFailed
-
-    This property returns true if the current data function has
-    failed; false otherwise.  The fail state is reset when
-    functionName is changed or finishTestFunction() is called.
-
-    \sa failed
-*/
-bool QuickTestResult::isDataFailed() const
-{
     return QTestResult::currentTestFailed();
 }
 
@@ -349,6 +336,11 @@ void QuickTestResult::finishTestData()
     QTestResult::finishedCurrentTestData();
 }
 
+void QuickTestResult::finishTestDataCleanup()
+{
+    QTestResult::finishedCurrentTestDataCleanup();
+}
+
 void QuickTestResult::finishTestFunction()
 {
     QTestResult::finishedCurrentTestFunction();
diff --git a/src/qmltest/quicktestresult_p.h b/src/qmltest/quicktestresult_p.h
index f6f0c3a1b3..3bc6f9c7cb 100644
--- a/src/qmltest/quicktestresult_p.h
+++ b/src/qmltest/quicktestresult_p.h
@@ -61,7 +61,6 @@ class Q_QUICK_TEST_EXPORT QuickTestResult : public QObject
     Q_PROPERTY(QString functionName READ functionName WRITE setFunctionName NOTIFY functionNameChanged)
     Q_PROPERTY(QString dataTag READ dataTag WRITE setDataTag NOTIFY dataTagChanged)
     Q_PROPERTY(bool failed READ isFailed)
-    Q_PROPERTY(bool dataFailed READ isDataFailed)
     Q_PROPERTY(bool skipped READ isSkipped WRITE setSkipped NOTIFY skippedChanged)
     Q_PROPERTY(int passCount READ passCount)
     Q_PROPERTY(int failCount READ failCount)
@@ -88,7 +87,6 @@ public:
     void setDataTag(const QString &tag);
 
     bool isFailed() const;
-    bool isDataFailed() const;
 
     bool isSkipped() const;
     void setSkipped(bool skip);
@@ -109,6 +107,7 @@ public Q_SLOTS:
     void clearTestTable();
 
     void finishTestData();
+    void finishTestDataCleanup();
     void finishTestFunction();
 
     void fail(const QString &message, const QUrl &location, int line);