AutoTest: Enhance reporting passes and fails

Depending on the report and log level of Boost UTF
the number of test passes and fails might be wrong.
Circumvent by providing a way to report a summary.

Change-Id: I6d2cb7674550f10a0263e08d21cce42569f3d7a8
Reviewed-by: David Schulz <david.schulz@qt.io>

5 files changed, 33 insertions, 4 deletions

diff --git a/src/plugins/autotest/testoutputreader.h b/src/plugins/autotest/testoutputreader.h
index b78f9cbecb..87216a2570 100644
--- a/src/plugins/autotest/testoutputreader.h
+++ b/src/plugins/autotest/testoutputreader.h
@@ -48,6 +48,8 @@ public:
     void createAndReportResult(const QString &message, ResultType type);
     bool hadValidOutput() const { return m_hadValidOutput; }
     int disabledTests() const { return m_disabled; }
+    bool hasSummary() const { return !m_summary.isEmpty(); }
+    QHash<ResultType, int> summary() const { return m_summary; }
     void setId(const QString &id) { m_id = id; }
     QString id() const { return m_id; }
 
@@ -64,6 +66,7 @@ protected:
     QProcess *m_testApplication;  // not owned
     QString m_buildDir;
     QString m_id;
+    QHash<ResultType, int> m_summary;
     int m_disabled = -1;
 private:
     bool m_hadValidOutput = false;
diff --git a/src/plugins/autotest/testresultmodel.cpp b/src/plugins/autotest/testresultmodel.cpp
index 874e813660..f07bc5f008 100644
--- a/src/plugins/autotest/testresultmodel.cpp
+++ b/src/plugins/autotest/testresultmodel.cpp
@@ -23,10 +23,11 @@
 **
 ****************************************************************************/
 
+#include "testresultmodel.h"
 #include "autotesticons.h"
 #include "autotestplugin.h"
 #include "testresultdelegate.h"
-#include "testresultmodel.h"
+#include "testrunner.h"
 #include "testsettings.h"
 
 #include <projectexplorer/projectexplorericons.h>
@@ -220,6 +221,10 @@ QString TestResultItem::resultString() const
 TestResultModel::TestResultModel(QObject *parent)
     : Utils::TreeModel<TestResultItem>(new TestResultItem(TestResultPtr()), parent)
 {
+    connect(TestRunner::instance(), &TestRunner::reportSummary,
+            this, [this](const QString &id, const QHash<ResultType, int> &summary){
+        m_reportedSummary.insert(id, summary);
+    });
 }
 
 void TestResultModel::updateParent(const TestResultItem *item)
@@ -256,7 +261,7 @@ void TestResultModel::addTestResult(const TestResultPtr &testResult, bool autoEx
         return;
     }
 
-    m_testResultCount[testResult->result()]++;
+    m_testResultCount[testResult->id()][testResult->result()]++;
 
     TestResultItem *newItem = new TestResultItem(testResult);
     TestResultItem *root = nullptr;
@@ -314,6 +319,7 @@ void TestResultModel::clearTestResults()
 {
     clear();
     m_testResultCount.clear();
+    m_reportedSummary.clear();
     m_disabled = 0;
     m_fileNames.clear();
     m_maxWidthOfFileName = 0;
@@ -363,6 +369,21 @@ int TestResultModel::maxWidthOfLineNumber(const QFont &font)
     return m_widthOfLineNumber;
 }
 
+int TestResultModel::resultTypeCount(ResultType type) const
+{
+    int result = 0;
+
+    for (auto resultsForId : m_testResultCount.values())
+        result += resultsForId.value(type, 0);
+
+    for (auto id : m_reportedSummary.keys()) {
+        if (int counted = m_testResultCount.value(id).value(type))
+            result -= counted;
+        result += m_reportedSummary[id].value(type);
+    }
+    return result;
+}
+
 TestResultItem *TestResultModel::findParentItemFor(const TestResultItem *item,
                                                    const TestResultItem *startItem) const
 {
diff --git a/src/plugins/autotest/testresultmodel.h b/src/plugins/autotest/testresultmodel.h
index 38c880003b..6370296ac0 100644
--- a/src/plugins/autotest/testresultmodel.h
+++ b/src/plugins/autotest/testresultmodel.h
@@ -84,7 +84,7 @@ public:
     int maxWidthOfFileName(const QFont &font);
     int maxWidthOfLineNumber(const QFont &font);
 
-    int resultTypeCount(ResultType type) const { return m_testResultCount.value(type, 0); }
+    int resultTypeCount(ResultType type) const;
     int disabledTests() const { return m_disabled; }
     void raiseDisabledTests(int amount) { m_disabled += amount; }
 
@@ -94,7 +94,8 @@ private:
     TestResultItem *findParentItemFor(const TestResultItem *item,
                                       const TestResultItem *startItem = nullptr) const;
     void updateParent(const TestResultItem *item);
-    QMap<ResultType, int> m_testResultCount;
+    QHash<QString, QMap<ResultType, int>> m_testResultCount;
+    QHash<QString, QHash<ResultType, int>> m_reportedSummary;
     int m_widthOfLineNumber = 0;
     int m_maxWidthOfFileName = 0;
     int m_disabled = 0;
diff --git a/src/plugins/autotest/testrunner.cpp b/src/plugins/autotest/testrunner.cpp
index e1f4497580..cf432870e3 100644
--- a/src/plugins/autotest/testrunner.cpp
+++ b/src/plugins/autotest/testrunner.cpp
@@ -270,6 +270,9 @@ void TestRunner::onProcessFinished()
     const int disabled = m_currentOutputReader->disabledTests();
     if (disabled > 0)
         emit hadDisabledTests(disabled);
+    if (m_currentOutputReader->hasSummary())
+        emit reportSummary(m_currentOutputReader->id(), m_currentOutputReader->summary());
+
     resetInternalPointers();
 
     if (!m_fakeFutureInterface) {
diff --git a/src/plugins/autotest/testrunner.h b/src/plugins/autotest/testrunner.h
index 79e7a8c389..5628a6d29e 100644
--- a/src/plugins/autotest/testrunner.h
+++ b/src/plugins/autotest/testrunner.h
@@ -71,6 +71,7 @@ signals:
     void requestStopTestRun();
     void testResultReady(const TestResultPtr &result);
     void hadDisabledTests(int disabled);
+    void reportSummary(const QString &id, const QHash<ResultType, int> &summary);
 
 private:
     void buildProject(ProjectExplorer::Project *project);