Improve crash handler

- Terminate sub-processes with SIGTERM.
- Setup alternate signal stack only once and hence allocate the
  memory only once (there can be only one alternate signal stack)

This is an adjusted backport from 5.14: f02cbab and f43d30f

Task-number: AUTOSUITE-1454
Change-Id: I1e294d6f81a735be6d77890f09f6d50a80e2b970
Reviewed-by: Robert Griebl <robert.griebl@qt.io>

2 files changed, 28 insertions, 18 deletions

diff --git a/src/common-lib/crashhandler.cpp b/src/common-lib/crashhandler.cpp
index 6bcaf586..529f46ff 100644
--- a/src/common-lib/crashhandler.cpp
+++ b/src/common-lib/crashhandler.cpp
@@ -189,6 +189,9 @@ static void initBacktrace()
         // 4 means to remove 4 stack frames: this way the backtrace starts at std::terminate
         crashHandler(buffer, 4);
     });
+
+    // create a new process group, so that we are able to kill all children with ::kill(0, ...)
+    setpgid(0, 0);
 }
 
 static void printMsgToConsole(const char *format, ...) Q_ATTRIBUTE_FORMAT_PRINTF(1, 2);
@@ -419,6 +422,10 @@ static void crashHandler(const char *why, int stackFramesToIgnore)
         printCrashInfo(Dlt, why, stackFramesToIgnore);
     }
 
+    // make sure to terminate our sub-processes as well, but not ourselves
+    signal(SIGTERM, SIG_IGN);
+    kill(0, SIGTERM);
+
     if (dumpCore) {
         fprintf(stderr, "\n > the process will be aborted (core dumped)\n\n");
         abort();
diff --git a/src/common-lib/unixsignalhandler.cpp b/src/common-lib/unixsignalhandler.cpp
index 897a3103..a88c1cd5 100644
--- a/src/common-lib/unixsignalhandler.cpp
+++ b/src/common-lib/unixsignalhandler.cpp
@@ -56,15 +56,6 @@
 
 QT_BEGIN_NAMESPACE_AM
 
-#if defined(Q_OS_UNIX)
-
-// make it clear in the valgrind backtrace that this is a deliberate leak
-static void *malloc_valgrind_ignore(size_t size)
-{
-    return malloc(size);
-}
-#endif
-
 // sigmask() is not available on Windows
 UnixSignalHandler::am_sigmask_t UnixSignalHandler::am_sigmask(int sig)
 {
@@ -73,12 +64,31 @@ UnixSignalHandler::am_sigmask_t UnixSignalHandler::am_sigmask(int sig)
 
 UnixSignalHandler *UnixSignalHandler::s_instance = nullptr;
 
+#if defined(Q_OS_UNIX)
+// make it clear in the valgrind backtrace that this is a deliberate leak
+static void *malloc_valgrind_ignore(size_t size)
+{
+    return malloc(size);
+}
+
 UnixSignalHandler::UnixSignalHandler()
     : QObject()
-#if defined(Q_OS_UNIX)
     , m_pipe { -1, -1 }
-#endif
+{
+    // Setup alternate signal stack (to get backtrace for stack overflow)
+    // Canonical size might not be suffcient to get QML backtrace, so we double it
+    constexpr size_t stackSize = SIGSTKSZ * 2;
+    stack_t sigstack;
+    // valgrind will report this as leaked: nothing we can do about it
+    sigstack.ss_sp = malloc_valgrind_ignore(stackSize);
+    sigstack.ss_size = stackSize;
+    sigstack.ss_flags = 0;
+    sigaltstack(&sigstack, nullptr);
+}
+#else
+UnixSignalHandler::UnixSignalHandler() : QObject()
 { }
+#endif
 
 UnixSignalHandler *UnixSignalHandler::instance()
 {
@@ -208,13 +218,6 @@ bool UnixSignalHandler::install(Type handlerType, const std::initializer_list<in
         m_handlers.emplace_back(sig, handlerType == ForwardedToEventLoopHandler, handler);
 
 #if defined(Q_OS_UNIX)
-    // Use alternate signal stack to get backtrace for stack overflow
-    stack_t sigstack;
-    sigstack.ss_sp = malloc_valgrind_ignore(SIGSTKSZ); // valgrind will report this as leaked: nothing we can do about it
-    sigstack.ss_size = SIGSTKSZ;
-    sigstack.ss_flags = 0;
-    sigaltstack(&sigstack, nullptr);
-
     struct sigaction sigact;
     sigact.sa_flags = SA_ONSTACK;
     sigact.sa_handler = sigHandler;