Fix sanitizer build

Task-number: QTBUG-64726
Change-Id: Ic88eed62e85ad1d95afcbd154c6556509e42bd47
Reviewed-by: Michal Klocek <michal.klocek@qt.io>

5 files changed, 55 insertions, 18 deletions

diff --git a/configure.json b/configure.json
index 398df0ce9..7ef1f5003 100644
--- a/configure.json
+++ b/configure.json
@@ -534,7 +534,7 @@
             "output": [ "privateFeature" ]
         },
         "webengine-sanitizer" : {
-            "label": "Sanitizer ",
+            "label": "Sanitizer",
             "autoDetect": "config.sanitizer && tests.webengine-sanitizer",
             "condition": "config.sanitizer",
             "output": [ "privateFeature" ]
@@ -561,6 +561,11 @@
             "type": "warning",
             "condition": "!features.webengine-flex",
             "message": "flex is required to build QtWebEngine."
+        },
+        {
+            "type": "warning",
+            "condition": "config.sanitizer && !tests.webengine-sanitizer && !features.webengine-sanitizer",
+            "message": "Qt WebEngine cannot be built with the chosen sanitizer configuration. Check config.log for details or use -feature-webengine-sanitizer to force the build."
         }
     ],
 
@@ -588,6 +593,11 @@
                     "condition": "config.unix"
                 },
                 {
+                    "type": "feature",
+                    "args": "webengine-sanitizer",
+                    "condition": "config.sanitizer"
+                },
+                {
                    "section": "Optional system libraries used",
                    "condition": "config.unix",
                    "entries": [
diff --git a/configure.pri b/configure.pri
index e0e482685..497557262 100644
--- a/configure.pri
+++ b/configure.pri
@@ -159,6 +159,7 @@ defineTest(qtConfTest_detectIcuuc) {
 
 defineTest(qtConfTest_isSanitizerSupported) {
   sanitizer_combo_supported = true
+
   sanitize_address {
     asan_supported = false
     linux-clang-libc++:isSanitizerSupportedOnLinux() {
@@ -168,7 +169,7 @@ defineTest(qtConfTest_isSanitizerSupported) {
     }
     !$$asan_supported {
       sanitizer_combo_supported = false
-       qtLog("An address sanitizer-enabled Qt WebEngine build can only be built on Linux or macOS using Clang and libc++.")
+      qtLog("An address sanitizer-enabled Qt WebEngine build can only be built on Linux or macOS using Clang and libc++.")
     }
   }
 
@@ -179,12 +180,16 @@ defineTest(qtConfTest_isSanitizerSupported) {
 
   sanitize_undefined {
     ubsan_supported = false
-    linux-clang-libc++:isSanitizerSupportedOnLinux():CONFIG(release, debug|release):!debug_and_release {
-      ubsan_supported = true
+    CONFIG(release, debug|release):!debug_and_release {
+      linux-clang-libc++:isSanitizerSupportedOnLinux() {
+        ubsan_supported = true
+      } else:macos:isSanitizerSupportedOnMacOS() {
+        ubsan_supported = true
+      }
     }
     !$$ubsan_supported {
       sanitizer_combo_supported = false
-      qtLog("An undefined behavior sanitizer-enabled Qt WebEngine build can only be built on Linux using Clang and libc++ in release mode.")
+      qtLog("An undefined behavior sanitizer-enabled Qt WebEngine build can only be built on Linux or macOS using Clang and libc++ in release mode.")
     }
   }
 
@@ -209,11 +214,12 @@ defineTest(isSanitizerSupportedOnLinux) {
 }
 
 defineTest(isSanitizerSupportedOnMacOS) {
-  isEmpty(QT_APPLE_CLANG_MAJOR_VERSION) {
+  isEmpty(QMAKE_APPLE_CLANG_MAJOR_VERSION) {
     QTWEBENGINE_CLANG_IS_APPLE = false
   } else {
     QTWEBENGINE_CLANG_IS_APPLE = true
   }
+
   $$QTWEBENGINE_CLANG_IS_APPLE:isSanitizerMacOSAppleClangVersionSupported(): return(true)
   else:isSanitizerMacOSClangVersionSupported(): return(true)
   return(false)
@@ -221,28 +227,28 @@ defineTest(isSanitizerSupportedOnMacOS) {
 
 defineTest(isSanitizerMacOSAppleClangVersionSupported) {
   # Clang sanitizer suppression attributes work from Apple Clang version 7.3.0+.
-  greaterThan(QT_APPLE_CLANG_MAJOR_VERSION, 7): return(true)
-  greaterThan(QT_APPLE_CLANG_MINOR_VERSION, 2): return(true)
+  greaterThan(QMAKE_APPLE_CLANG_MAJOR_VERSION, 7): return(true)
+  greaterThan(QMAKE_APPLE_CLANG_MINOR_VERSION, 2): return(true)
 
-  qtLog("Using Apple Clang version $${QT_APPLE_CLANG_MAJOR_VERSION}.$${QT_APPLE_CLANG_MINOR_VERSION}.$${QT_APPLE_CLANG_PATCH_VERSION}, but at least Apple Clang version 7.3.0 is required to build a sanitizer-enabled Qt WebEngine.")
+  qtLog("Using Apple Clang version $${QMAKE_APPLE_CLANG_MAJOR_VERSION}.$${QMAKE_APPLE_CLANG_MINOR_VERSION}.$${QMAKE_APPLE_CLANG_PATCH_VERSION}, but at least Apple Clang version 7.3.0 is required to build a sanitizer-enabled Qt WebEngine.")
   return(false)
 }
 
 defineTest(isSanitizerMacOSClangVersionSupported) {
   # Clang sanitizer suppression attributes work from non-apple Clang version 3.7+.
-  greaterThan(QT_CLANG_MAJOR_VERSION, 3): return(true)
-  greaterThan(QT_CLANG_MINOR_VERSION, 6): return(true)
+  greaterThan(QMAKE_CLANG_MAJOR_VERSION, 3): return(true)
+  greaterThan(QMAKE_CLANG_MINOR_VERSION, 6): return(true)
 
-  qtLog("Using Clang version $${QT_CLANG_MAJOR_VERSION}.$${QT_CLANG_MINOR_VERSION}, but at least Clang version 3.7 is required to build a sanitizer-enabled Qt WebEngine.")
+  qtLog("Using Clang version $${QMAKE_CLANG_MAJOR_VERSION}.$${QMAKE_CLANG_MINOR_VERSION}, but at least Clang version 3.7 is required to build a sanitizer-enabled Qt WebEngine.")
   return(false)
 }
 
 defineTest(isSanitizerLinuxClangVersionSupported) {
   # Clang sanitizer suppression attributes work from Clang version 3.7+.
-  greaterThan(QT_CLANG_MAJOR_VERSION, 3): return(true)
-  greaterThan(QT_CLANG_MINOR_VERSION, 6): return(true)
+  greaterThan(QMAKE_CLANG_MAJOR_VERSION, 3): return(true)
+  greaterThan(QMAKE_CLANG_MINOR_VERSION, 6): return(true)
 
-  qtLog("Using Clang version $${QT_CLANG_MAJOR_VERSION}.$${QT_CLANG_MINOR_VERSION}, but at least Clang version 3.7 is required to build a sanitizer-enabled Qt WebEngine.")
+  qtLog("Using Clang version $${QMAKE_CLANG_MAJOR_VERSION}.$${QMAKE_CLANG_MINOR_VERSION}, but at least Clang version 3.7 is required to build a sanitizer-enabled Qt WebEngine.")
   return(false)
 }
 
diff --git a/mkspecs/features/configure.prf b/mkspecs/features/configure.prf
index 904e7b1c0..8b8fd3efe 100644
--- a/mkspecs/features/configure.prf
+++ b/mkspecs/features/configure.prf
@@ -48,7 +48,7 @@ defineTest(runConfigure) {
     }
 
     sanitizer: !qtConfig(webengine-sanitizer) {
-        skipBuild("Chosen sanitizer configuration is not supported. Use --feature-webengine-sanitizer=yes to force build with the chosen sanitizer configuration.")
+        skipBuild("Chosen sanitizer configuration is not supported. Check config.log for details or use -feature-webengine-sanitizer to force build with the chosen sanitizer configuration.")
         return(false);
     }
 
diff --git a/src/core/config/common.pri b/src/core/config/common.pri
index d8a4fa181..d9dd7d7fd 100644
--- a/src/core/config/common.pri
+++ b/src/core/config/common.pri
@@ -62,3 +62,15 @@ CONFIG(debug, debug|release) {
 
 # Compiling with -Os makes a huge difference in binary size
 optimize_size: gn_args += optimize_for_size=true
+
+# We don't want to apply sanitizer options to the build tools (GN, dict convert, etc).
+!host_build {
+    sanitizer: gn_args += sanitizer_keep_symbols=true
+    sanitize_address: gn_args += is_asan=true
+    sanitize_thread: gn_args += is_tsan=true
+    sanitize_memory: gn_args += is_msan=true
+    # rtti is required for a specific check of ubsan, -fsanitize=vptr, which uses the runtime
+    # type information to check that correct derived objects are assigned to base pointers. Without
+    # rtti, linking would fail at build time.
+    sanitize_undefined: gn_args += is_ubsan=true use_rtti=true
+}
diff --git a/src/core/core_module.pro b/src/core/core_module.pro
index 2409ccb12..d21985e60 100644
--- a/src/core/core_module.pro
+++ b/src/core/core_module.pro
@@ -40,8 +40,17 @@ else: LIBS_PRIVATE += $$NINJA_ARCHIVES
 LIBS_PRIVATE += $$NINJA_LIB_DIRS $$NINJA_LIBS
 # GN's LFLAGS doesn't always work across all the Linux configurations we support.
 # The Windows and macOS ones from GN does provide a few useful flags however
-linux: QMAKE_LFLAGS += -Wl,--gc-sections -Wl,-O1 -Wl,-z,now -Wl,-z,defs
-else: QMAKE_LFLAGS += $$NINJA_LFLAGS
+
+linux {
+    QMAKE_LFLAGS += -Wl,--gc-sections -Wl,-O1 -Wl,-z,now
+    # Embedded address sanitizer symbols are undefined and are picked up by the dynamic link loader
+    # at runtime. Thus we do not to pass the linker flag below, because the linker would complain
+    # about the undefined sanitizer symbols.
+    !sanitizer: QMAKE_LFLAGS += -Wl,-z,defs
+} else {
+    QMAKE_LFLAGS += $$NINJA_LFLAGS
+}
+
 POST_TARGETDEPS += $$NINJA_TARGETDEPS