CMake: Handle detection of linux-g++-32 mkspec

If Qt is configured with -platform linux-g++-32 , make sure
to add the -m32 compile options for all built targets.

On 64 bit host OSes that provide both 32 and 64 bit libraries
we need to exclude the 64 bit libraries from being picked up.
The locations of the libraries are distro-specific.

This change by default excludes the Ubuntu x86_64 libraries
paths.
Opt outs are provided, which when used, forces Qt builders to
specify their own ignore paths in a custom CMake toolchain file.

The compile option and default path exclusions are added to the
Qt-generated CMake toolchain file as well, so they are reused
when building other Qt repositories.

Note that there is no foolproof way to tell CMake to ignore all
x86_64 packages / libraries, even if CMake 3.23
CMAKE_IGNORE_PREFIX_PATH is used, because there might not be
a single sysroot to exclude.
Both x86 and x86_64 libraries can co-exist in the same sysroot,
e.g in /usr

One would have to list each package / library directory in
CMAKE_IGNORE_PATH manually.

Additionally, the PKG_CONFIG_LIBDIR environment variable is also set
to Ubuntu specific prefixes, to ensure that pkg_check_modules ->
pkg-config don't pickup x86_64 libraries.

Fixes: QTBUG-101963
Change-Id: Ib17c8d2cd0ba33b2cf748772245bcd558de9120c
Reviewed-by: Jörg Bornemann <joerg.bornemann@qt.io>

2 files changed, 137 insertions, 0 deletions

diff --git a/cmake/QtAutoDetect.cmake b/cmake/QtAutoDetect.cmake
index 14f220f69a..3731707f88 100644
--- a/cmake/QtAutoDetect.cmake
+++ b/cmake/QtAutoDetect.cmake
@@ -420,6 +420,91 @@ function(qt_auto_detect_win32_arm)
     endif()
 endfunction()
 
+function(qt_auto_detect_linux_x86)
+    if("${QT_QMAKE_TARGET_MKSPEC}" STREQUAL "linux-g++-32" AND NOT QT_NO_AUTO_DETECT_LINUX_X86)
+
+        # Add flag to ensure code is compiled for 32bit x86 ABI aka i386 or its flavors.
+        set(__qt_toolchain_common_flags_init "-m32")
+
+        if(NOT QT_NO_OVERRIDE_LANG_FLAGS_INIT)
+            set(CMAKE_C_FLAGS_INIT "${__qt_toolchain_common_flags_init}" PARENT_SCOPE)
+            set(CMAKE_CXX_FLAGS_INIT "${__qt_toolchain_common_flags_init}" PARENT_SCOPE)
+            set(CMAKE_ASM_FLAGS_INIT "${__qt_toolchain_common_flags_init}" PARENT_SCOPE)
+        endif()
+
+        # Each distro places arch-specific libraries according to its own file system layout.
+        #
+        # https://wiki.debian.org/Multiarch/TheCaseForMultiarch
+        # https://wiki.ubuntu.com/MultiarchSpec
+        # https://wiki.gentoo.org/wiki/Project:AMD64/Multilib_layout
+        # https://wiki.archlinux.org/title/official_repositories#multilib
+        # https://documentation.suse.com/sles/15-SP3/html/SLES-all/cha-64bit.html
+        # https://pilotlogic.com/sitejoom/index.php/wiki?id=398
+        # https://unix.stackexchange.com/questions/458069/multilib-and-multiarch
+        #
+        # CMake can usually find 32 bit libraries just fine on its own.
+        # find_library will use prefixes from CMAKE_PREFIX_PATH / CMAKE_SYSTEM_PREFIX_PATH
+        # and add arch-specific lib folders like 'lib/i386-linux-gnu' on debian based systems
+        # or lib32/lib64 on other distros.
+        # The problem is that if no 32 bit library is found, a 64 bit one might get picked up.
+        # That's why we need to specify additional ignore paths.
+        #
+        # The paths used in the code below are Ubuntu specific.
+        # You can opt out of using them if you are using a different distro, but then you need to
+        # specify appropriate paths yourself in your own CMake toolchain file.
+        #
+        # Note that to absolutely ensure no x86_64 library is picked up on a multiarch /
+        # multilib-enabled system, you might need to specify extra directories in
+        # CMAKE_INGORE_PATH for each sub-directory containing a library.
+        #
+        # For example to exclude /usr/lib/x86_64-linux-gnu/mit-krb5/libgssapi_krb5.so
+        # you need to add /usr/lib/x86_64-linux-gnu/mit-krb5 explicitly to CMAKE_IGNORE_PATH.
+        # Adding just /usr/lib/x86_64-linux-gnu to either CMAKE_IGNORE_PATH or
+        # CMAKE_IGNORE_PREFIX_PATH is not enough.
+        #
+        # Another consideration are results returned by CMake's pkg_check_modules which uses
+        # pkg-config.
+        # CMAKE_IGNORE_PATH is not read by pkg_check_modules, but CMAKE_PREFIX_PATH
+        # values are passed as additional prefixes to look for .pc files, IN ADDITION to the default
+        # prefixes searched by pkg-config of each specific distro.
+        # For example on Ubuntu, the default searched paths on an x86_64 host are:
+        #   /usr/local/lib/x86_64-linux-gnu/pkgconfig
+        #   /usr/local/lib/pkgconfig
+        #   /usr/local/share/pkgconfig
+        #   /usr/lib/x86_64-linux-gnu/pkgconfig
+        #   /usr/lib/pkgconfig
+        #   /usr/share/pkgconfig
+        # To ensure the x86_64 packages are not picked up, the PKG_CONFIG_LIBDIR environment
+        # variable can be overridden with an explicit list of prefixes.
+        # Again, the paths below are Ubuntu specific.
+        if(NOT QT_NO_OVERRIDE_CMAKE_IGNORE_PATH)
+            set(linux_x86_ignore_path "/usr/lib/x86_64-linux-gnu;/lib/x86_64-linux-gnu")
+            set(CMAKE_IGNORE_PATH "${linux_x86_ignore_path}" PARENT_SCOPE)
+            set_property(GLOBAL PROPERTY
+                _qt_internal_linux_x86_ignore_path "${linux_x86_ignore_path}")
+        endif()
+        if(NOT QT_NO_OVERRIDE_PKG_CONFIG_LIBDIR)
+            set(pc_config_libdir "")
+            list(APPEND pc_config_libdir "/usr/local/lib/i386-linux-gnu/pkgconfig")
+            list(APPEND pc_config_libdir "/usr/local/lib/pkgconfig")
+            list(APPEND pc_config_libdir "/usr/local/share/pkgconfig")
+            list(APPEND pc_config_libdir "/usr/lib/i386-linux-gnu/pkgconfig")
+            list(APPEND pc_config_libdir "/usr/lib/pkgconfig")
+            list(APPEND pc_config_libdir "/usr/share/pkgconfig")
+            list(JOIN pc_config_libdir ":" pc_config_libdir)
+
+            set_property(GLOBAL PROPERTY
+                _qt_internal_linux_x86_pc_config_libdir "${pc_config_libdir}")
+
+            # Overrides the default prefix list.
+            set(ENV{PKG_CONFIG_LIBDIR} "${pc_config_libdir}")
+
+            # Overrides the additional prefixes list.
+            set(ENV{PKG_CONFIG_DIR} "")
+        endif()
+    endif()
+endfunction()
+
 function(qt_auto_detect_integrity)
     if(
        # Qt's custom CMake toolchain file sets this value.
@@ -449,4 +534,5 @@ qt_auto_detect_vcpkg()
 qt_auto_detect_pch()
 qt_auto_detect_wasm()
 qt_auto_detect_win32_arm()
+qt_auto_detect_linux_x86()
 qt_auto_detect_integrity()
diff --git a/cmake/QtToolchainHelpers.cmake b/cmake/QtToolchainHelpers.cmake
index a674973f4d..f5cf783912 100644
--- a/cmake/QtToolchainHelpers.cmake
+++ b/cmake/QtToolchainHelpers.cmake
@@ -43,6 +43,57 @@ set(__qt_chainload_toolchain_file \"\${__qt_initially_configured_toolchain_file}
         list(APPEND init_platform "set(CMAKE_SYSTEM_PROCESSOR arm64 CACHE STRING \"\")")
     endif()
 
+    if("${QT_QMAKE_TARGET_MKSPEC}" STREQUAL "linux-g++-32" AND NOT QT_NO_AUTO_DETECT_LINUX_X86)
+        set(__qt_toolchain_common_flags_init "-m32")
+
+        if(NOT QT_NO_OVERRIDE_LANG_FLAGS_INIT)
+            list(APPEND init_platform
+                "if(NOT QT_NO_OVERRIDE_LANG_FLAGS_INIT)")
+
+            list(APPEND init_platform
+                "    set(__qt_toolchain_common_flags_init \"-m32\")")
+            list(APPEND init_platform
+                "    set(CMAKE_C_FLAGS_INIT \"\${__qt_toolchain_common_flags_init}\")")
+            list(APPEND init_platform
+                "    set(CMAKE_CXX_FLAGS_INIT \"\${__qt_toolchain_common_flags_init}\")")
+            list(APPEND init_platform
+                "    set(CMAKE_ASM_FLAGS_INIT \"\${__qt_toolchain_common_flags_init}\")")
+
+            list(APPEND init_platform "endif()")
+        endif()
+
+        # Ubuntu-specific paths are used below.
+        # See comments of qt_auto_detect_linux_x86() for details.
+        if(NOT QT_NO_OVERRIDE_CMAKE_IGNORE_PATH)
+            list(APPEND init_platform
+                "if(NOT QT_NO_OVERRIDE_CMAKE_IGNORE_PATH)")
+
+            get_property(linux_x86_ignore_path GLOBAL PROPERTY _qt_internal_linux_x86_ignore_path)
+
+            string(REPLACE ";" "LITERAL_SEMICOLON"
+                linux_x86_ignore_path "${linux_x86_ignore_path}")
+
+            list(APPEND init_platform
+                "    set(CMAKE_IGNORE_PATH \"${linux_x86_ignore_path}\")")
+
+            list(APPEND init_platform "endif()")
+        endif()
+
+        if(NOT QT_NO_OVERRIDE_PKG_CONFIG_LIBDIR)
+            list(APPEND init_platform
+                "if(NOT QT_NO_OVERRIDE_PKG_CONFIG_LIBDIR)")
+
+            get_property(pc_config_libdir GLOBAL PROPERTY _qt_internal_linux_x86_pc_config_libdir)
+
+            list(APPEND init_platform
+                "    set(ENV{PKG_CONFIG_LIBDIR} \"${pc_config_libdir}\")")
+            list(APPEND init_platform
+                "    set(ENV{PKG_CONFIG_DIR} \"\")")
+
+            list(APPEND init_platform "endif()")
+        endif()
+    endif()
+
     # By default we don't want to allow mixing compilers for building different repositories, so we
     # embed the initially chosen compilers into the toolchain.
     # This is because on Windows compilers aren't easily mixed.