QTest: add debugger detection for macOS

Which, in Mach speak, is an exception handler for the process. Also
check if the crash reporter is in a mode that will show a dialog for
any crashed process. If so, leave it to that crash reporter to do stack
traces.

This patch has the nice side-effect that a crashing test won't have both
a debugger and CrashReporter generate stack traces.

You can check the settings for CrashReporter on macOS with the command:

    defaults read com.apple.CrashReporter DialogType

If it is set to 'basic' or 'developer' or 'crashreport', CrashReporter
will show the dialog. If set to 'server' it won't. Any unattended system
should have it set to 'server' (which will have QTest invoke lldb to
generate stack traces):

    defaults write com.apple.CrashReporter DialogType server

Change-Id: I39153e44cff15c00341857f178b1dcfab154f8ee
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

1 files changed, 37 insertions, 0 deletions

diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index 1c13f8edc2..c175967054 100644
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -103,6 +103,9 @@
 
 #if defined(Q_OS_MACX)
 #include <IOKit/pwr_mgt/IOPMLib.h>
+#include <mach/task.h>
+#include <mach/mach_init.h>
+#include <CoreFoundation/CFPreferences.h>
 #endif
 
 #include <vector>
@@ -137,6 +140,40 @@ static bool debuggerPresent()
     return pid != 0;
 #elif defined(Q_OS_WIN)
     return IsDebuggerPresent();
+#elif defined(Q_OS_MACOS)
+    auto equals = [](CFStringRef str1, CFStringRef str2) -> bool {
+        return CFStringCompare(str1, str2, kCFCompareCaseInsensitive) == kCFCompareEqualTo;
+    };
+
+    // Check if there is an exception handler for the process:
+    mach_msg_type_number_t portCount = 0;
+    exception_mask_t masks[EXC_TYPES_COUNT];
+    mach_port_t ports[EXC_TYPES_COUNT];
+    exception_behavior_t behaviors[EXC_TYPES_COUNT];
+    thread_state_flavor_t flavors[EXC_TYPES_COUNT];
+    exception_mask_t mask = EXC_MASK_ALL & ~(EXC_MASK_RESOURCE | EXC_MASK_GUARD);
+    kern_return_t result = task_get_exception_ports(mach_task_self(), mask, masks, &portCount,
+                                                    ports, behaviors, flavors);
+    if (result == KERN_SUCCESS) {
+        for (mach_msg_type_number_t portIndex = 0; portIndex < portCount; ++portIndex) {
+            if (MACH_PORT_VALID(ports[portIndex])) {
+                return true;
+            }
+        }
+    }
+
+    // Ok, no debugger attached. So, let's see if CrashReporter will throw up a dialog. If so, we
+    // leave it to the OS to do the stack trace.
+    CFStringRef crashReporterType = static_cast<CFStringRef>(
+                CFPreferencesCopyAppValue(CFSTR("DialogType"), CFSTR("com.apple.CrashReporter")));
+    if (crashReporterType == nullptr)
+        return true;
+
+    const bool createsStackTrace =
+            !equals(crashReporterType, CFSTR("server")) &&
+            !equals(crashReporterType, CFSTR("none"));
+    CFRelease(crashReporterType);
+    return createsStackTrace;
 #else
     // TODO
     return false;