Add a feature to enable CodeCoverage analysis of testlib

Based on Asmo Saarela's advice (QTPM-686), adapted on advice from
FrogLogic support and converted to a feature so that the selftest and
testlib qmake config can be co-ordinated.

Task-number: QTPM-1385
Change-Id: Icd706f086009e1e08b3f8c5cd553f792402e28c0
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

6 files changed, 66 insertions, 1 deletions

diff --git a/src/testlib/configure.json b/src/testlib/configure.json
index c464037205..df6132cdc2 100644
--- a/src/testlib/configure.json
+++ b/src/testlib/configure.json
@@ -5,6 +5,12 @@
     ],
 
     "features": {
+        "testlib_selfcover": {
+            "label": "Coverage testing of testlib itself",
+            "purpose": "Gauges how thoroughly testlib's selftest exercises testlib's code",
+            "autoDetect": false,
+            "output": [ "publicFeature" ]
+        },
         "itemmodeltester": {
             "label": "Tester for item models",
             "purpose": "Provides a utility to test item models.",
diff --git a/src/testlib/qtest.h b/src/testlib/qtest.h
index 3906eddb79..ebd94939ce 100644
--- a/src/testlib/qtest.h
+++ b/src/testlib/qtest.h
@@ -374,9 +374,30 @@ QT_END_NAMESPACE
 #  define QTEST_SET_MAIN_SOURCE_PATH  QTest::setMainSourcePath(__FILE__);
 #endif
 
+// Hooks for coverage-testing of QTestLib itself:
+#if QT_CONFIG(testlib_selfcover) && defined(__COVERAGESCANNER__)
+struct QtCoverageScanner
+{
+    QtCoverageScanner(const char *name)
+    {
+        __coveragescanner_clear();
+        __coveragescanner_testname(name);
+    }
+    ~QtCoverageScanner()
+    {
+        __coveragescanner_save();
+        __coveragescanner_testname("");
+    }
+};
+#define TESTLIB_SELFCOVERAGE_START(name) QtCoverageScanner _qtCoverageScanner(name);
+#else
+#define TESTLIB_SELFCOVERAGE_START(name)
+#endif
+
 #define QTEST_APPLESS_MAIN(TestObject) \
 int main(int argc, char *argv[]) \
 { \
+    TESTLIB_SELFCOVERAGE_START(TestObject) \
     TestObject tc; \
     QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \
@@ -406,6 +427,7 @@ int main(int argc, char *argv[]) \
 #define QTEST_MAIN(TestObject) \
 int main(int argc, char *argv[]) \
 { \
+    TESTLIB_SELFCOVERAGE_START(#TestObject) \
     QApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     QTEST_DISABLE_KEYPAD_NAVIGATION \
@@ -421,6 +443,7 @@ int main(int argc, char *argv[]) \
 #define QTEST_MAIN(TestObject) \
 int main(int argc, char *argv[]) \
 { \
+    TESTLIB_SELFCOVERAGE_START(#TestObject) \
     QGuiApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     TestObject tc; \
@@ -433,6 +456,7 @@ int main(int argc, char *argv[]) \
 #define QTEST_MAIN(TestObject) \
 int main(int argc, char *argv[]) \
 { \
+    TESTLIB_SELFCOVERAGE_START(#TestObject) \
     QCoreApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     TestObject tc; \
@@ -445,6 +469,7 @@ int main(int argc, char *argv[]) \
 #define QTEST_GUILESS_MAIN(TestObject) \
 int main(int argc, char *argv[]) \
 { \
+    TESTLIB_SELFCOVERAGE_START(#TestObject) \
     QCoreApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     TestObject tc; \
diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index 9f0cdb0109..15e123bdb7 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -246,7 +246,7 @@ static void stackTrace()
 
 static bool installCoverageTool(const char * appname, const char * testname)
 {
-#ifdef __COVERAGESCANNER__
+#if defined(__COVERAGESCANNER__) && !QT_CONFIG(testlib_selfcover)
     if (!qEnvironmentVariableIsEmpty("QT_TESTCOCOON_ACTIVE"))
         return false;
     // Set environment variable QT_TESTCOCOON_ACTIVE to prevent an eventual subtest from
diff --git a/src/testlib/qtestlog.cpp b/src/testlib/qtestlog.cpp
index 1268730cc6..e7150260e5 100644
--- a/src/testlib/qtestlog.cpp
+++ b/src/testlib/qtestlog.cpp
@@ -73,6 +73,10 @@ QT_BEGIN_NAMESPACE
 static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage)
 {
 #ifdef __COVERAGESCANNER__
+#  if QT_CONFIG(testlib_selfcover)
+    __coveragescanner_teststate(QTestLog::failCount() > 0 ? "FAILED" :
+                                QTestLog::passCount() > 0 ? "PASSED" : "SKIPPED");
+#  else
     if (!installedTestCoverage)
         return;
     // install again to make sure the filename is correct.
@@ -83,6 +87,7 @@ static void saveCoverageTool(const char * appname, bool testfailed, bool install
     __coveragescanner_testname("");
     __coveragescanner_clear();
     unsetenv("QT_TESTCOCOON_ACTIVE");
+#  endif // testlib_selfcover
 #else
     Q_UNUSED(appname);
     Q_UNUSED(testfailed);
diff --git a/src/testlib/selfcover.pri b/src/testlib/selfcover.pri
new file mode 100644
index 0000000000..7de50ba6e6
--- /dev/null
+++ b/src/testlib/selfcover.pri
@@ -0,0 +1,28 @@
+# Configuration for testlib and its tests, to instrument with
+# FrogLogic's Squish CoCo (cf. testcocoon.prf, which handles similar
+# for general code; but testlib needs special handling).
+
+# Only for use when feature testlib_selfcover is enabled:
+!qtConfig(testlib_selfcover): return()
+
+# This enables verification that testlib itself is adequately tested,
+# as a grounds for trusting that testing with it is useful.
+# Exclude all non-testlib source from coverage instrumentation:
+COVERAGE_OPTIONS = --cs-exclude-file-abs-wildcard=$$QT_SOURCE_TREE/*
+COVERAGE_OPTIONS += --cs-include-file-abs-wildcard=*/src/testlib/*
+COVERAGE_OPTIONS += --cs-mcc # enable Multiple Condition Coverage
+COVERAGE_OPTIONS += --cs-mcdc # enable Multiple Condition / Decision Coverage
+# (recommended for ISO 26262 ASIL A, B and C -- highly recommended for ASIL D)
+# https://doc.froglogic.com/squish-coco/4.1/codecoverage.html#sec%3Amcdc
+
+QMAKE_CFLAGS += $$COVERAGE_OPTIONS
+QMAKE_CXXFLAGS += $$COVERAGE_OPTIONS
+QMAKE_LFLAGS += $$COVERAGE_OPTIONS
+
+# FIXME: relies on QMAKE_* being just the command-names, with no path prefix
+QMAKE_CC = cs$$QMAKE_CC
+QMAKE_CXX = cs$$QMAKE_CXX
+QMAKE_LINK = cs$$QMAKE_LINK
+QMAKE_LINK_SHLIB = cs$$QMAKE_LINK_SHLIB
+QMAKE_AR = cs$$QMAKE_AR
+QMAKE_LIB = cs$$QMAKE_LIB
diff --git a/src/testlib/testlib.pro b/src/testlib/testlib.pro
index 34bb581e02..f52a913a08 100644
--- a/src/testlib/testlib.pro
+++ b/src/testlib/testlib.pro
@@ -146,4 +146,5 @@ mac {
 
 !qtHaveModule(network): HEADERSCLEAN_EXCLUDE += qtest_network.h
 
+include(selfcover.pri)
 load(qt_module)