Add rudimentary support for address sanitizer builds

This change adds a new setup.py option called --sanitize-address
which will build all executables and shared libraries with address
sanitizer enabled. The builds will only succeed on Linux and macOS
machines that have new enough gcc / clang versions, so it is a
"use at your own risk" build configuration.

This change was necessitated by the random crashes that are sometimes
observed on the CI machines, and due to valgrind not working properly
on new macOS versions, using AddressSanitizer is the next best thing.

Note that when running tests with address sanitizer builds, you
might need to export a LD_PRELOAD / DYLD_INSERT_LIBRARIES environment
variable pointing to the address sanitizer runtime library path, which
will be provided by the crashed application.

Change-Id: I93014002e5c5e94bcc808ba2fb830d60724cfb69
Reviewed-by: Christian Tismer <tismer@stackless.com>

3 files changed, 41 insertions, 0 deletions

diff --git a/setup.py b/setup.py
index 8a8970c68..eb7b1646e 100644
--- a/setup.py
+++ b/setup.py
@@ -81,6 +81,7 @@ For development purposes the following options might be of use, when using "setu
     --skip-packaging will skip creation of the python package,
     --ignore-git will skip the fetching and checkout steps for supermodule and all submodules.
     --verbose-build will output the compiler invocation with command line arguments, etc.
+    --sanitize-address will build all targets with address sanitizer enabled.
 
 REQUIREMENTS:
 - Python: 2.6, 2.7, 3.3, 3.4, 3.5 and 3.6 are supported
@@ -296,6 +297,7 @@ OPTION_RPATH_VALUES = option_value("rpath")
 OPTION_QT_CONF_PREFIX = option_value("qt-conf-prefix")
 OPTION_ICULIB = option_value("iculib-url") # Deprecated
 OPTION_VERBOSE_BUILD = has_option("verbose-build")
+OPTION_SANITIZE_ADDRESS = has_option("sanitize-address")
 
 # This is used automatically by distutils.command.install object, to specify final installation
 # location.
@@ -1033,6 +1035,13 @@ class pyside_build(_build):
         if OPTION_VERBOSE_BUILD:
             cmake_cmd.append("-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON")
 
+        if OPTION_SANITIZE_ADDRESS:
+            # Some simple sanity checking. Only use at your own risk.
+            if sys.platform.startswith('linux') or sys.platform.startswith('darwin'):
+                cmake_cmd.append("-DSANITIZE_ADDRESS=ON")
+            else:
+                raise DistutilsSetupError("Address sanitizer can only be used on Linux and macOS.")
+
         if extension.lower() == "pyside2":
             pyside_qt_conf_prefix = ''
             if OPTION_QT_CONF_PREFIX:
diff --git a/sources/pyside2/CMakeLists.txt b/sources/pyside2/CMakeLists.txt
index 08ed52473..fac031fdb 100644
--- a/sources/pyside2/CMakeLists.txt
+++ b/sources/pyside2/CMakeLists.txt
@@ -398,6 +398,22 @@ else()
     message(STATUS "PySide will be generated using the protected hack!")
 endif()
 
+# Build with Address sanitizer enabled if requested. This may break things, so use at your own risk.
+if (SANITIZE_ADDRESS AND NOT MSVC)
+    # Currently this does not check that the clang / gcc version used supports Address sanitizer,
+    # so once again, use at your own risk.
+    add_compile_options("-fsanitize=address" "-g" "-fno-omit-frame-pointer")
+    # We need to add the sanitize address option to all linked executables / shared libraries
+    # so that proper sanitizer symbols are linked in.
+    #
+    # Note that when running tests, you may need to set an additional environment variable
+    # in set_tests_properties for shiboken2 / pyside tests, or exported in your shell. Address
+    # sanitizer will tell you what environment variable needs to be exported. For example:
+    # export DYLD_INSERT_LIBRARIES=/Applications/Xcode.app/Contents/Developer/Toolchains/
+    #   ./XcodeDefault.xctoolchain/usr/lib/clang/8.1.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib
+    set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_STANDARD_LIBRARIES} -fsanitize=address")
+endif()
+
 add_subdirectory(libpyside)
 find_package(Qt5Designer)
 if(Qt5UiTools_FOUND AND Qt5Designer_FOUND)
diff --git a/sources/shiboken2/CMakeLists.txt b/sources/shiboken2/CMakeLists.txt
index 238ee2d34..c5cca506e 100644
--- a/sources/shiboken2/CMakeLists.txt
+++ b/sources/shiboken2/CMakeLists.txt
@@ -144,6 +144,22 @@ if(CMAKE_HOST_APPLE)
     endif()
 endif()
 
+# Build with Address sanitizer enabled if requested. This may break things, so use at your own risk.
+if (SANITIZE_ADDRESS AND NOT MSVC)
+    # Currently this does not check that the clang / gcc version used supports Address sanitizer,
+    # so once again, use at your own risk.
+    add_compile_options("-fsanitize=address" "-g" "-fno-omit-frame-pointer")
+    # We need to add the sanitize address option to all linked executables / shared libraries
+    # so that proper sanitizer symbols are linked in.
+    #
+    # Note that when running tests, you may need to set an additional environment variable
+    # in set_tests_properties for shiboken2 / pyside tests, or exported in your shell. Address
+    # sanitizer will tell you what environment variable needs to be exported. For example:
+    # export DYLD_INSERT_LIBRARIES=/Applications/Xcode.app/Contents/Developer/Toolchains/
+    #   ./XcodeDefault.xctoolchain/usr/lib/clang/8.1.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib
+    set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_STANDARD_LIBRARIES} -fsanitize=address")
+endif()
+
 add_subdirectory(ApiExtractor)
 
 set(generator_plugin_DIR ${LIB_INSTALL_DIR}/generatorrunner${generator_SUFFIX})