Dodge qFatal() so as to get coverage results despite it

The silent and blacklisted selftests of testlib end in a qFatal(), to
test its messaging is handled correctly.  However, this prevents hooks
in main() from saving coverage data when we're gathering that.  So use
a transient signal handler that longjmp()s back to a setjmp() just
before the qFatal() to let the test complete normally (but, since
qFatal() does something different on MS-Win, don't apply this to it).

Note that testlib's internal FatalSignalHandler handles all fatal
signals *except* SIGABRT, so this isn't over-riding it.  (In any case,
this restores the prior signal handler in setjmp()'s catch branch.)

Added missing expected_silent.tap test output while checking that this
change doesn't affect (the rest of) the test output.

Change-Id: I7e460581ad93e26639c066b3229438a66fd299de
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

3 files changed, 82 insertions, 2 deletions

diff --git a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
index 90520385ec..9c08af382a 100644
--- a/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
+++ b/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
@@ -75,6 +75,17 @@ void tst_Blacklisted::xpass()
     QVERIFY2(true, "This test should XPASS, blacklist ignored for XPASS");
 }
 
+#ifndef Q_OS_WIN
+#include <signal.h>
+#include <setjmp.h>
+
+static jmp_buf state;
+static void abort_handler(int signal)
+{
+    longjmp(state, 1);
+}
+#endif
+
 void tst_Blacklisted::messages()
 {
     qWarning("This is a warning that should not appear in silent test output");
@@ -83,7 +94,15 @@ void tst_Blacklisted::messages()
     qCritical("This is a critical message that should not appear in silent test output");
     qInfo("This is an info message that should not appear in silent test output");
     QTestLog::info("This is an internal testlib info message that should not appear in silent test output", __FILE__, __LINE__);
-    qFatal("This is a fatal error message that should still appear in silent test output");
+
+#ifndef Q_OS_WIN
+    // We're testing qFatal, but we don't want to actually std::abort() !
+    auto prior = signal(SIGABRT, abort_handler);
+    if (setjmp(state))
+        signal(SIGABRT, prior);
+    else
+#endif
+        qFatal("This is a fatal error message that should still appear in silent test output");
 }
 
 QTEST_MAIN(tst_Blacklisted)
diff --git a/tests/auto/testlib/selftests/expected_silent.tap b/tests/auto/testlib/selftests/expected_silent.tap
new file mode 100644
index 0000000000..c5f7716390
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_silent.tap
@@ -0,0 +1,42 @@
+TAP version 13
+# tst_Silent
+ok 1 - initTestCase()
+ok 2 - pass()
+ok 3 - skip() # SKIP This test should skip
+not ok 4 - fail()
+  ---
+  type: QVERIFY
+  message: This test should fail
+  wanted: true (false)
+  found: false (false)
+  expected: true (false)
+  actual: false (false)
+  at: tst_Silent::fail() (qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp:60)
+  file: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp
+  line: 60
+  ...
+not ok 5 - xfail() # TODO This test should XFAIL
+  ---
+  at: tst_Silent::xfail() (qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp:66)
+  file: qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp
+  line: 66
+  ...
+ok 6 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should XPASS)
+# This is a warning that should not appear in silent test output
+# This is an internal testlib warning that should not appear in silent test output
+# This is a debug message that should not appear in silent test output
+# This is a critical message that should not appear in silent test output
+# This is an info message that should not appear in silent test output
+# This is an internal testlib info message that should not appear in silent test output
+# This is a fatal error message that should still appear in silent test output
+not ok 7 - messages()
+  ---
+  # Received a fatal error.
+  at: tst_Silent::messages() (Unknown file:0)
+  file: Unknown file
+  line: 0
+  ...
+1..7
+# tests 7
+# pass 3
+# fail 3
diff --git a/tests/auto/testlib/selftests/silent/tst_silent.cpp b/tests/auto/testlib/selftests/silent/tst_silent.cpp
index 97fbbfd30a..cf3c35dec4 100644
--- a/tests/auto/testlib/selftests/silent/tst_silent.cpp
+++ b/tests/auto/testlib/selftests/silent/tst_silent.cpp
@@ -72,6 +72,17 @@ void tst_Silent::xpass()
     QVERIFY2(true, "This test should XPASS");
 }
 
+#ifndef Q_OS_WIN
+#include <signal.h>
+#include <setjmp.h>
+
+static jmp_buf state;
+static void abort_handler(int signal)
+{
+    longjmp(state, 1);
+}
+#endif
+
 void tst_Silent::messages()
 {
     qWarning("This is a warning that should not appear in silent test output");
@@ -80,7 +91,15 @@ void tst_Silent::messages()
     qCritical("This is a critical message that should not appear in silent test output");
     qInfo("This is an info message that should not appear in silent test output");
     QTestLog::info("This is an internal testlib info message that should not appear in silent test output", __FILE__, __LINE__);
-    qFatal("This is a fatal error message that should still appear in silent test output");
+
+#ifndef Q_OS_WIN
+    // We're testing qFatal, but we don't want to actually std::abort() !
+    auto prior = signal(SIGABRT, abort_handler);
+    if (setjmp(state))
+        signal(SIGABRT, prior);
+    else
+#endif
+        qFatal("This is a fatal error message that should still appear in silent test output");
 }
 
 QTEST_MAIN(tst_Silent)