CodeCoverage: Handle QTest based subtests.

Set QT_TESTCOCOON_ACTIVE environment variable when the coverage is installed
for a test and unset it when the coverage data is saved. Tests that run when
QT_TESTCOCOON_ACTIVE is set are subtests and will not be considered as
stand-alone tests for the coverage.

When a test is run as a subtest its coverage data will not be saved for
itself but for the main test it is merged with. Also its status will not be
reported since only the status of the main test is expected in the test report,
e.g. the test tests/auto/testlib/selftests.

Change-Id: Icfdf99300aae18040e1a3441a8af21f68df4c0db
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>
Reviewed-by: Jason McDonald <jason.mcdonald@nokia.com>

4 files changed, 45 insertions, 15 deletions

diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index 0440e26aca..d94abff23c 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -934,16 +934,24 @@ QT_BEGIN_NAMESPACE
     QTouchEventSequence is called (ie when the object returned runs out of scope).
 */
 
-static void installCoverageTool(const char * appname, const char * testname)
+static bool installCoverageTool(const char * appname, const char * testname)
 {
 #ifdef __COVERAGESCANNER__
+    if (!qgetenv("QT_TESTCOCOON_ACTIVE").isEmpty())
+        return false;
+    // Set environment variable QT_TESTCOCOON_ACTIVE to prevent an eventual subtest from
+    // being considered as a stand-alone test regarding the coverage analysis.
+    qputenv("QT_TESTCOCOON_ACTIVE", "1");
+
     // Install Coverage Tool
     __coveragescanner_install(appname);
     __coveragescanner_testname(testname);
     __coveragescanner_clear();
+    return true;
 #else
     Q_UNUSED(appname);
     Q_UNUSED(testname);
+    return false;
 #endif
 }
 
@@ -1962,7 +1970,8 @@ int QTest::qExec(QObject *testObject, int argc, char **argv)
 
     qtest_qParseArgs(argc, argv, false);
 
-    installCoverageTool(argv[0], metaObject->className());
+    bool installedTestCoverage = installCoverageTool(argv[0], metaObject->className());
+    QTestLog::setInstalledTestCoverage(installedTestCoverage);
 
 #ifdef QTESTLIB_USE_VALGRIND
     if (QBenchmarkGlobalData::current->mode() == QBenchmarkGlobalData::CallgrindParentProcess) {
diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp
index 29dfbc144e..c8487a2c54 100644
--- a/src/testlib/qtestlog.cpp
+++ b/src/testlib/qtestlog.cpp
@@ -56,9 +56,11 @@
 
 QT_BEGIN_NAMESPACE
 
-static void saveCoverageTool(const char * appname, bool testfailed)
+static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage)
 {
 #ifdef __COVERAGESCANNER__
+    if (!installedTestCoverage)
+        return;
     // install again to make sure the filename is correct.
     // without this, a plugin or similar may have changed the filename.
     __coveragescanner_install(appname);
@@ -66,9 +68,11 @@ static void saveCoverageTool(const char * appname, bool testfailed)
     __coveragescanner_save();
     __coveragescanner_testname("");
     __coveragescanner_clear();
+    unsetenv("QT_TESTCOCOON_ACTIVE");
 #else
     Q_UNUSED(appname);
     Q_UNUSED(testfailed);
+    Q_UNUSED(installedTestCoverage);
 #endif
 }
 
@@ -198,6 +202,7 @@ namespace QTest {
 
     static int verbosity = 0;
     static int maxWarnings = 2002;
+    static bool installedTestCoverage = true;
 
     static QtMsgHandler oldMessageHandler;
 
@@ -388,7 +393,7 @@ void QTestLog::stopLogging()
     QTest::TestLoggers::stopLogging();
     QTest::TestLoggers::destroyLoggers();
     QTest::loggerUsingStdout = false;
-    saveCoverageTool(QTestResult::currentAppname(), failCount() != 0);
+    saveCoverageTool(QTestResult::currentAppname(), failCount() != 0, QTestLog::installedTestCoverage());
 }
 
 void QTestLog::addLogger(LogMode mode, const char *filename)
@@ -502,4 +507,14 @@ void QTestLog::resetCounters()
     QTest::skips = 0;
 }
 
+void QTestLog::setInstalledTestCoverage(bool installed)
+{
+    QTest::installedTestCoverage = installed;
+}
+
+bool QTestLog::installedTestCoverage()
+{
+    return QTest::installedTestCoverage;
+}
+
 QT_END_NAMESPACE
diff --git a/src/testlib/qtestlog_p.h b/src/testlib/qtestlog_p.h
index 1fe52367ec..1a9754e460 100644
--- a/src/testlib/qtestlog_p.h
+++ b/src/testlib/qtestlog_p.h
@@ -103,6 +103,9 @@ public:
 
     static void resetCounters();
 
+    static void setInstalledTestCoverage(bool installed);
+    static bool installedTestCoverage();
+
 private:
     QTestLog();
     ~QTestLog();
diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp
index dc7966b82b..7e671a50f2 100644
--- a/tests/auto/testlib/selftests/tst_selftests.cpp
+++ b/tests/auto/testlib/selftests/tst_selftests.cpp
@@ -456,23 +456,26 @@ void tst_Selftests::runSubTest_data()
     }
 }
 
+static void insertEnvironmentVariable(QString const& name, QProcessEnvironment &result)
+{
+    const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment();
+    const QString value = systemEnvironment.value(name);
+    if (!value.isEmpty())
+        result.insert(name, value);
+}
+
 static inline QProcessEnvironment processEnvironment()
 {
     QProcessEnvironment result;
-    const QString path = QStringLiteral("PATH");
-    const QProcessEnvironment systemEnvironment = QProcessEnvironment::systemEnvironment();
-    result.insert(path, systemEnvironment.value(path));
+    insertEnvironmentVariable(QStringLiteral("PATH"), result);
     // Preserve DISPLAY for X11 as some tests use QtGui.
 #if defined(Q_OS_UNIX) && !defined(Q_OS_MAC)
-    const QString display = QStringLiteral("DISPLAY");
-    const QString displayValue = systemEnvironment.value(display);
-    if (!displayValue.isEmpty())
-        result.insert(display, displayValue);
+    insertEnvironmentVariable(QStringLiteral("DISPLAY"), result);
+#endif
+    insertEnvironmentVariable(QStringLiteral("QT_QPA_PLATFORM"), result);
+#ifdef __COVERAGESCANNER__
+    insertEnvironmentVariable(QStringLiteral("QT_TESTCOCOON_ACTIVE"), result);
 #endif
-    const QString platform = QStringLiteral("QT_QPA_PLATFORM");
-    const QString platformValue = systemEnvironment.value(platform);
-    if (!platformValue.isEmpty())
-        result.insert(platform, platformValue);
     return result;
 }