diff options
author | Alexandru Croitor <alexandru.croitor@qt.io> | 2021-09-29 19:01:51 +0200 |
---|---|---|
committer | Alexandru Croitor <alexandru.croitor@qt.io> | 2022-02-04 15:51:04 +0100 |
commit | 57866a57586d401c784f809f9f7994b0e4623706 (patch) | |
tree | 48b9d5a7d1a03d1edd15359858adcefd18430467 | |
parent | 14e4527cc427ce8c5e7c1758a95a1bbce0498471 (diff) |
setup.py: Add support for cross-building
setup.py can now be used to cross-compile PySide to a target Linux
distribution from a Linux host.
For example you could cross-compile PySide targeting an arm64
Raspberry Pi4 sysroot on an Ubuntu x86_64 host machine.
Cross-compiling PySide has a few requirements:
- a sysroot to cross-compile against, with a pre-installed Qt,
Python interpreter, library and development packages (which
provides C++ headers)
- a host Qt installation of the same version that is in the target
sysroot
- a host Python installation, preferably of the same version as the
target one (to run setup.py)
- a working cross-compiling toolchain (cross-compiler, linker, etc)
- a custom written CMake toolchain file
- CMake version 3.17+
- Qt version 6.3+
The CMake toolchain file is required to set up all the relevant
cross-compilation information: where the sysroot is, where the
toolchain is, the compiler name, compiler flags, etc.
Once are requirements are met, to cross-compile one has to specify a
few additional options when calling setup.py: the path to the cmake
toolchain file, the path to the host Qt installation
and the target python platform name.
An example setup.py invocation to build a wheel for an armv7 machine
might look like the following:
python setup.py bdist_wheel --parallel=8 --ignore-git --reuse-build
--cmake-toolchain-file=$PWD/rpi/toolchain_armv7.cmake
--qt-host-path=/opt/Qt/6.3.0/gcc_64
--plat-name=linux_armv7l
--limited-api=yes
--standalone
Sample platform names that can be used are: linux_armv7, linux_aarch64.
If the auto-detection code fails to find the target Python or Qt
installation, one can specify their location by providing the
--python-target-path=<path>
and
--qt-target-path=<path>
options to setup.py.
If the automatic build of the host shiboken code generator fails,
one can specify the path to a custom built host shiboken via the
--shiboken-host-path option.
Documentation about the build process and a sample CMake
toolchain file will be added in a separate change.
Implementation details.
Internally, setup.py will build a host shiboken executable using
the provided host Qt path, and then use it for the cross-build.
This is achieved via an extra setup.py sub-invocation with some
heuristics on which options should be passed to the sub-invocation.
The host shiboken is not included in the target wheels.
Introspection of where the host / target Qt and Python are located
is done via CMake compile tests, because we can't query information
from a qmake that is built for a different architecture / platform.
When limited API is enabled, we modify the wheel name to contain the
manylinux2014 tag, despite the wheel not fully qualifying for that
tag.
When copying the Qt libraries / plugins from the target sysroot in a
standalone build, we need to adjust all their rpaths to match the
destination directory layout of the wheel.
Fixes: PYSIDE-802
Task-number: PYSIDE-1033
Change-Id: I6e8c51ef5127d85949de650396d615ca95194db0
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
-rw-r--r-- | build_scripts/build_info_collector.py | 121 | ||||
-rw-r--r-- | build_scripts/config.py | 25 | ||||
-rw-r--r-- | build_scripts/main.py | 189 | ||||
-rw-r--r-- | build_scripts/options.py | 217 | ||||
-rw-r--r-- | build_scripts/platforms/linux.py | 16 | ||||
-rw-r--r-- | build_scripts/qtinfo.py | 139 | ||||
-rw-r--r-- | build_scripts/setup_runner.py | 139 | ||||
-rw-r--r-- | build_scripts/utils.py | 78 | ||||
-rw-r--r-- | build_scripts/wheel_override.py | 140 | ||||
-rw-r--r-- | sources/shiboken6/config.tests/target_qt_mkspec/CMakeLists.txt | 22 |
10 files changed, 890 insertions, 196 deletions
diff --git a/build_scripts/build_info_collector.py b/build_scripts/build_info_collector.py index f4146af51..592b4c873 100644 --- a/build_scripts/build_info_collector.py +++ b/build_scripts/build_info_collector.py @@ -48,6 +48,7 @@ from setuptools._distutils import sysconfig as sconfig from .options import OPTION from .qtinfo import QtInfo +from .utils import configure_cmake_project, parse_cmake_project_message_info from .wheel_utils import get_qt_version @@ -166,6 +167,7 @@ class BuildInfoCollectorMixin(object): build_lib: str cmake: str cmake_toolchain_file: str + internal_cmake_install_dir_query_file_path: str is_cross_compile: bool plat_name: str python_target_path: str @@ -185,43 +187,97 @@ class BuildInfoCollectorMixin(object): sources_dir = os.path.join(script_dir, "sources") - platform_arch = platform.architecture()[0] - log.info(f"Python architecture is {platform_arch}") - self.py_arch = platform_arch[:-3] + if self.is_cross_compile: + config_tests_dir = os.path.join(script_dir, build_base, "config.tests") + python_target_info_dir = os.path.join(sources_dir, "shiboken6", "config.tests", + "target_python_info") + cmake_cache_args = [] + + if self.python_target_path: + cmake_cache_args.append(("Python_ROOT_DIR", self.python_target_path)) + + if self.cmake_toolchain_file: + cmake_cache_args.append(("CMAKE_TOOLCHAIN_FILE", self.cmake_toolchain_file)) + python_target_info_output = configure_cmake_project( + python_target_info_dir, + self.cmake, + temp_prefix_build_path=config_tests_dir, + cmake_cache_args=cmake_cache_args) + python_target_info = parse_cmake_project_message_info(python_target_info_output) + self.python_target_info = python_target_info build_type = "Debug" if OPTION["DEBUG"] else "Release" if OPTION["RELWITHDEBINFO"]: build_type = 'RelWithDebInfo' # Prepare parameters - py_executable = sys.executable - py_version = f"{sys.version_info[0]}.{sys.version_info[1]}" - py_include_dir = get_config_var("INCLUDEPY") - py_libdir = get_config_var("LIBDIR") - # distutils.sysconfig.get_config_var('prefix') returned the - # virtual environment base directory, but - # sysconfig.get_config_var returns the system's prefix. - # We use 'base' instead (although, platbase points to the - # same location) - py_prefix = get_config_var("base") - if not py_prefix or not os.path.exists(py_prefix): - py_prefix = sys.prefix - self.py_prefix = py_prefix - if sys.platform == "win32": - py_scripts_dir = os.path.join(py_prefix, "Scripts") + if not self.is_cross_compile: + platform_arch = platform.architecture()[0] + self.py_arch = platform_arch[:-3] + + py_executable = sys.executable + py_version = f"{sys.version_info[0]}.{sys.version_info[1]}" + py_include_dir = get_config_var("INCLUDEPY") + py_libdir = get_config_var("LIBDIR") + # distutils.sysconfig.get_config_var('prefix') returned the + # virtual environment base directory, but + # sysconfig.get_config_var returns the system's prefix. + # We use 'base' instead (although, platbase points to the + # same location) + py_prefix = get_config_var("base") + if not py_prefix or not os.path.exists(py_prefix): + py_prefix = sys.prefix + self.py_prefix = py_prefix + if sys.platform == "win32": + py_scripts_dir = os.path.join(py_prefix, "Scripts") + else: + py_scripts_dir = os.path.join(py_prefix, "bin") + self.py_scripts_dir = py_scripts_dir else: - py_scripts_dir = os.path.join(py_prefix, "bin") - self.py_scripts_dir = py_scripts_dir + # We don't look for an interpreter when cross-compiling. + py_executable = None + + python_info = self.python_target_info['python_info'] + py_version = python_info['version'].split('.') + py_version = f"{py_version[0]}.{py_version[1]}" + py_include_dir = python_info['include_dirs'] + py_libdir = python_info['library_dirs'] + py_library = python_info['libraries'] + self.py_library = py_library + + # Prefix might not be set because the project that extracts + # the info is using internal API to get it. It shouldn't be + # critical though, because we don't really use neither + # py_prefix nor py_scripts_dir in important places + # when cross-compiling. + if 'prefix' in python_info: + py_prefix = python_info['prefix'] + self.py_prefix = py_prefix + + py_scripts_dir = os.path.join(py_prefix, 'bin') + if os.path.exists(py_scripts_dir): + self.py_scripts_dir = py_scripts_dir + else: + self.py_scripts_dir = None + else: + py_prefix = None + self.py_prefix = py_prefix + self.py_scripts_dir = None self.qtinfo = QtInfo() qt_version = get_qt_version() # Used for test blacklists and registry test. - self.build_classifiers = (f"py{py_version}-qt{qt_version}-{platform.architecture()[0]}-" - f"{build_type.lower()}") - if hasattr(sys, "pypy_version_info"): - pypy_version = ".".join(map(str, sys.pypy_version_info[:3])) - self.build_classifiers += f"-pypy.{pypy_version}" + if self.is_cross_compile: + # Querying the host platform architecture makes no sense when cross-compiling. + build_classifiers = f"py{py_version}-qt{qt_version}-{self.plat_name}-" + else: + build_classifiers = f"py{py_version}-qt{qt_version}-{platform.architecture()[0]}-" + if hasattr(sys, "pypy_version_info"): + pypy_version = ".".join(map(str, sys.pypy_version_info[:3])) + build_classifiers += f"pypy.{pypy_version}-" + build_classifiers += f"{build_type.lower()}" + self.build_classifiers = build_classifiers venv_prefix, has_virtual_env = prefix() @@ -229,6 +285,8 @@ class BuildInfoCollectorMixin(object): # and we consider it is distinct enough that we don't have to # append the build classifiers, thus keeping dir names shorter. build_name = f"{venv_prefix}" + if self.is_cross_compile and has_virtual_env: + build_name += f"-{self.plat_name}" # If short paths are requested and no virtual env is found, at # least append the python version for more uniqueness. @@ -259,11 +317,18 @@ class BuildInfoCollectorMixin(object): self.install_dir = install_dir self.py_executable = py_executable self.py_include_dir = py_include_dir - self.py_library = get_py_library(build_type, py_version, py_prefix, - py_libdir, py_include_dir) + + if not self.is_cross_compile: + self.py_library = get_py_library(build_type, py_version, py_prefix, + py_libdir, py_include_dir) self.py_version = py_version self.build_type = build_type - self.site_packages_dir = sconfig.get_python_lib(1, 0, prefix=install_dir) + + if self.is_cross_compile: + site_packages_without_prefix = self.python_target_info['python_info']['site_packages_dir'] + self.site_packages_dir = os.path.join(install_dir, site_packages_without_prefix) + else: + self.site_packages_dir = sconfig.get_python_lib(1, 0, prefix=install_dir) def post_collect_and_assign(self): # self.build_lib is only available after the base class diff --git a/build_scripts/config.py b/build_scripts/config.py index ad3dd1542..2199269b4 100644 --- a/build_scripts/config.py +++ b/build_scripts/config.py @@ -82,6 +82,13 @@ class Config(object): self.shiboken_generator_st_name = f"{SHIBOKEN}-generator" self.pyside_st_name = PYSIDE_MODULE + # Path to CMake toolchain file when intending to cross compile + # the project. + self.cmake_toolchain_file = None + + # Store where host shiboken is built during a cross-build. + self.shiboken_host_query_path = None + # Used by check_allowed_python_version to validate the # interpreter version. self.python_version_classifiers = [ @@ -96,9 +103,14 @@ class Config(object): self.setup_script_dir = None - def init_config(self, build_type=None, internal_build_type=None, - cmd_class_dict=None, package_version=None, - ext_modules=None, setup_script_dir=None, + def init_config(self, + build_type=None, + internal_build_type=None, + cmd_class_dict=None, + package_version=None, + ext_modules=None, + setup_script_dir=None, + cmake_toolchain_file=None, quiet=False): """ Sets up the global singleton config which is used in many parts @@ -122,6 +134,8 @@ class Config(object): self.setup_script_dir = setup_script_dir + self.cmake_toolchain_file = cmake_toolchain_file + setup_kwargs = {} setup_kwargs['long_description'] = self.get_long_description() setup_kwargs['long_description_content_type'] = 'text/markdown' @@ -367,6 +381,11 @@ class Config(object): def is_top_level_build_pyside(self): return self.build_type == self.pyside_option_name + def is_cross_compile(self): + if not self.cmake_toolchain_file: + return False + return True + def set_internal_build_type(self, internal_build_type): self.internal_build_type = internal_build_type diff --git a/build_scripts/main.py b/build_scripts/main.py index a69b4fa69..2a1f46797 100644 --- a/build_scripts/main.py +++ b/build_scripts/main.py @@ -196,13 +196,18 @@ class PysideInstall(_install, DistUtilsCommandMixin): user_options = _install.user_options + DistUtilsCommandMixin.mixin_user_options def __init__(self, *args, **kwargs): + self.command_name = "install" _install.__init__(self, *args, **kwargs) DistUtilsCommandMixin.__init__(self) def initialize_options(self): _install.initialize_options(self) - if sys.platform == 'darwin': + def finalize_options(self): + DistUtilsCommandMixin.mixin_finalize_options(self) + _install.finalize_options(self) + + if sys.platform == 'darwin' or self.is_cross_compile: # Because we change the plat_name to include a correct # deployment target on macOS distutils thinks we are # cross-compiling, and throws an exception when trying to @@ -214,12 +219,11 @@ class PysideInstall(_install, DistUtilsCommandMixin): # target. The fix is to disable the warn_dir flag, which # was created for bdist_* derived classes to override, for # similar cases. + # We also do it when cross-compiling. While calling install + # command directly is dubious, bdist_wheel calls install + # internally before creating a wheel. self.warn_dir = False - def finalize_options(self): - DistUtilsCommandMixin.mixin_finalize_options(self) - _install.finalize_options(self) - def run(self): _install.run(self) log.info(f"--- Install completed ({elapsed()}s)") @@ -257,6 +261,7 @@ class PysideBuildExt(_build_ext): class PysideBuildPy(_build_py): def __init__(self, *args, **kwargs): + self.command_name = "build_py" _build_py.__init__(self, *args, **kwargs) @@ -289,6 +294,7 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): user_options = _build.user_options + DistUtilsCommandMixin.mixin_user_options def __init__(self, *args, **kwargs): + self.command_name = "build" _build.__init__(self, *args, **kwargs) DistUtilsCommandMixin.__init__(self) BuildInfoCollectorMixin.__init__(self) @@ -298,8 +304,14 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): DistUtilsCommandMixin.mixin_finalize_options(self) BuildInfoCollectorMixin.collect_and_assign(self) - if sys.platform == 'darwin': + use_os_name_hack = False + if self.is_cross_compile: + use_os_name_hack = True + elif sys.platform == 'darwin': self.plat_name = macos_plat_name() + use_os_name_hack = True + + if use_os_name_hack: # This is a hack to circumvent the dubious check in # distutils.commands.build -> finalize_options, which only # allows setting the plat_name for windows NT. @@ -314,7 +326,7 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): # Must come after _build.finalize_options BuildInfoCollectorMixin.post_collect_and_assign(self) - if sys.platform == 'darwin': + if use_os_name_hack: os.name = os_name_backup def initialize_options(self): @@ -333,6 +345,7 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): self.build_type = "Release" self.qtinfo = None self.build_tests = False + self.python_target_info = {} def run(self): prepare_build() @@ -349,7 +362,13 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): # Don't add Qt to PATH env var, we don't want it to interfere # with CMake's find_package calls which will use # CMAKE_PREFIX_PATH. - additional_paths = [self.py_scripts_dir] + # Don't add the Python scripts dir to PATH env when + # cross-compiling, it could be in the device sysroot (/usr) + # which can cause CMake device QtFooToolsConfig packages to be + # picked up instead of host QtFooToolsConfig packages. + additional_paths = [] + if self.py_scripts_dir and not self.is_cross_compile: + additional_paths.append(self.py_scripts_dir) # Add Clang to path for Windows. # Revisit once Clang is bundled with Qt. @@ -389,6 +408,13 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): log.info(f"Creating install folder {self.install_dir}...") os.makedirs(self.install_dir) + # Write the CMake install path into a file. Is used by + # SetupRunner to provide a nicer UX when cross-compiling (no + # need to specify a host shiboken path explicitly) + if self.internal_cmake_install_dir_query_file_path: + with open(self.internal_cmake_install_dir_query_file_path, 'w') as f: + f.write(self.install_dir) + if (not OPTION["ONLYPACKAGE"] and not config.is_internal_shiboken_generator_build_and_part_of_top_level_all()): # Build extensions @@ -464,7 +490,10 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): log.info(f"Python library: {self.py_library}") log.info(f"Python prefix: {self.py_prefix}") log.info(f"Python scripts: {self.py_scripts_dir}") + log.info(f"Python arch: {self.py_arch}") + log.info("-" * 3) + log.info(f"Qt prefix: {self.qtinfo.prefix_dir}") log.info(f"Qt qmake: {self.qtinfo.qmake_command}") log.info(f"Qt qtpaths: {self.qtinfo.qtpaths_command}") log.info(f"Qt version: {self.qtinfo.version}") @@ -545,9 +574,20 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): f"-DCMAKE_INSTALL_PREFIX={self.install_dir}", module_src_dir ] - cmake_cmd.append(f"-DPYTHON_EXECUTABLE={self.py_executable}") - cmake_cmd.append(f"-DPYTHON_INCLUDE_DIR={self.py_include_dir}") - cmake_cmd.append(f"-DPYTHON_LIBRARY={self.py_library}") + + # When cross-compiling we set Python_ROOT_DIR to tell + # FindPython.cmake where to pick up the device python libs. + if self.is_cross_compile: + if self.python_target_path: + cmake_cmd.append(f"-DPython_ROOT_DIR={self.python_target_path}") + + # Host python is needed when cross compiling to run + # embedding_generator.py. Pass it as a separate option. + cmake_cmd.append(f"-DQFP_PYTHON_HOST_PATH={sys.executable}") + else: + cmake_cmd.append(f"-DPYTHON_EXECUTABLE={self.py_executable}") + cmake_cmd.append(f"-DPYTHON_INCLUDE_DIR={self.py_include_dir}") + cmake_cmd.append(f"-DPYTHON_LIBRARY={self.py_library}") # If a custom shiboken cmake config directory path was provided, pass it to CMake. if OPTION["SHIBOKEN_CONFIG_DIR"] and config.is_internal_pyside_build(): @@ -597,10 +637,10 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): cmake_cmd.append("-DAVOID_PROTECTED_HACK=1") numpy = get_numpy_location() - if numpy: + if numpy and not self.is_cross_compile: cmake_cmd.append(f"-DNUMPY_INCLUDE_DIR={numpy}") - if self.build_type.lower() == 'debug': + if self.build_type.lower() == 'debug' and not self.is_cross_compile: cmake_cmd.append(f"-DPYTHON_DEBUG_LIBRARY={self.py_library}") if OPTION["LIMITED_API"] == "yes": @@ -713,7 +753,27 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): cmake_cmd.append("-DPYSIDE_NUMPY_SUPPORT=1") target_qt_prefix_path = self.qtinfo.prefix_dir - cmake_cmd.append(f"-DCMAKE_PREFIX_PATH={target_qt_prefix_path}") + cmake_cmd.append(f"-DQFP_QT_TARGET_PATH={target_qt_prefix_path}") + if self.qt_host_path: + cmake_cmd.append(f"-DQFP_QT_HOST_PATH={self.qt_host_path}") + + if self.is_cross_compile and (not OPTION["SHIBOKEN_HOST_PATH"] + or not os.path.exists(OPTION["SHIBOKEN_HOST_PATH"])): + raise DistutilsSetupError( + f"Please specify the location of host shiboken tools via --shiboken-host-path=") + + if self.shiboken_host_path: + cmake_cmd.append(f"-DQFP_SHIBOKEN_HOST_PATH={self.shiboken_host_path}") + + if self.shiboken_target_path: + cmake_cmd.append(f"-DQFP_SHIBOKEN_TARGET_PATH={self.shiboken_target_path}") + elif self.cmake_toolchain_file and not extension.lower() == SHIBOKEN: + # Need to tell where to find target shiboken when + # cross-compiling pyside. + cmake_cmd.append(f"-DQFP_SHIBOKEN_TARGET_PATH={self.install_dir}") + + if self.cmake_toolchain_file: + cmake_cmd.append(f"-DCMAKE_TOOLCHAIN_FILE={self.cmake_toolchain_file}") if not OPTION["SKIP_CMAKE"]: log.info(f"Configuring module {extension} ({module_src_dir})...") @@ -806,6 +866,11 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): "qt_prefix_dir": self.qtinfo.prefix_dir, "qt_translations_dir": self.qtinfo.translations_dir, "qt_qml_dir": self.qtinfo.qml_dir, + + # TODO: This is currently None when cross-compiling + # There doesn't seem to be any place where we can query + # it. Fortunately it's currently only used when + # packaging Windows vcredist. "target_arch": self.py_arch, } @@ -936,13 +1001,42 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): raise RuntimeError("Error copying libclang library " f"from {clang_lib_path} to {destination_dir}. ") + def get_shared_library_filters(self): + unix_filters = ["*.so", "*.so.*"] + darwin_filters = ["*.so", "*.dylib"] + filters = [] + if self.is_cross_compile: + if 'darwin' in self.plat_name or 'macos' in self.plat_name: + filters = darwin_filters + elif 'linux' in self.plat_name: + filters = unix_filters + else: + log.warn(f"No shared library filters found for platform {self.plat_name}. " + f"The package might miss Qt libraries and plugins.") + else: + if sys.platform == 'darwin': + filters = darwin_filters + else: + filters = unix_filters + return filters + def package_libraries(self, package_path): """Returns the libraries of the Python module""" - UNIX_FILTERS = ["*.so", "*.so.*"] - DARWIN_FILTERS = ["*.so", "*.dylib"] - FILTERS = DARWIN_FILTERS if sys.platform == 'darwin' else UNIX_FILTERS + filters = self.get_shared_library_filters() return [lib for lib in os.listdir( - package_path) if filter_match(lib, FILTERS)] + package_path) if filter_match(lib, filters)] + + def get_shared_libraries_in_path_recursively(self, initial_path): + """Returns shared library plugins in given path (collected + recursively)""" + filters = self.get_shared_library_filters() + libraries = [] + for dir_path, dir_names, file_names in os.walk(initial_path): + for name in file_names: + if filter_match(name, filters): + library_path = os.path.join(dir_path, name) + libraries.append(library_path) + return libraries def update_rpath(self, package_path, executables, libexec=False): ROOT = '@loader_path' if sys.platform == 'darwin' else '$ORIGIN' @@ -993,13 +1087,70 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin): log.info("Patched rpath to '$ORIGIN/' (Linux) or " f"updated rpath (OS/X) in {srcpath}.") + def update_rpath_for_linux_plugins( + self, + plugin_paths, + qt_lib_dir=None, + is_qml_plugin=False): + # If the linux sysroot (where the plugins are copied from) + # is from a mainline distribution, it might have a different + # directory layout than then one we expect to have in the + # wheel. + # We have to ensure that any plugins copied have rpath + # values that can find Qt libs in the newly assembled wheel + # dir layout. + if not (self.is_cross_compile and sys.platform.startswith('linux') and self.standalone): + return + + log.info(f"Patching rpath for Qt and QML plugins.") + for plugin in plugin_paths: + if os.path.isdir(plugin) or os.path.islink(plugin): + continue + if not os.path.exists(plugin): + continue + + if is_qml_plugin: + plugin_dir = os.path.dirname(plugin) + rel_path_from_qml_plugin_qt_lib_dir = os.path.relpath(qt_lib_dir, plugin_dir) + rpath_value = os.path.join("$ORIGIN", rel_path_from_qml_plugin_qt_lib_dir) + else: + rpath_value = "$ORIGIN/../../lib" + + linux_fix_rpaths_for_library(self._patchelf_path, plugin, rpath_value, + override=True) + log.info(f"Patched rpath to '{rpath_value}' in {plugin}.") + + def update_rpath_for_linux_qt_libraries(self, qt_lib_dir): + # Ensure that Qt libs and ICU libs have $ORIGIN in their rpath. + # Especially important for ICU lib, so that they don't + # accidentally load dependencies from the system. + if not (self.is_cross_compile and sys.platform.startswith('linux') and self.standalone): + return + + rpath_value = "$ORIGIN" + log.info(f"Patching rpath for Qt and ICU libraries in {qt_lib_dir}.") + libs = self.package_libraries(qt_lib_dir) + lib_paths = [os.path.join(qt_lib_dir, lib) for lib in libs] + for library in lib_paths: + if os.path.isdir(library) or os.path.islink(library): + continue + if not os.path.exists(library): + continue + + linux_fix_rpaths_for_library(self._patchelf_path, library, rpath_value, override=True) + log.info(f"Patched rpath to '{rpath_value}' in {library}.") + class PysideRstDocs(Command, DistUtilsCommandMixin): description = "Build .rst documentation only" user_options = DistUtilsCommandMixin.mixin_user_options - def initialize_options(self): + def __init__(self, *args, **kwargs): + self.command_name = "build_rst_docs" + Command.__init__(self, *args, **kwargs) DistUtilsCommandMixin.__init__(self) + + def initialize_options(self): log.info("-- This build process will not include the API documentation." "API documentation requires a full build of pyside/shiboken.") self.skip = False diff --git a/build_scripts/options.py b/build_scripts/options.py index 045eb05d0..6ce53b982 100644 --- a/build_scripts/options.py +++ b/build_scripts/options.py @@ -39,11 +39,13 @@ try: from setuptools._distutils import log + from setuptools import Command except ModuleNotFoundError: # This is motivated by our CI using an old version of setuptools # so then the coin_build_instructions.py script is executed, and # import from this file, it was failing. from distutils import log + from distutils.cmd import Command from shutil import which import sys import os @@ -51,6 +53,7 @@ import warnings from pathlib import Path from .qtinfo import QtInfo +from .utils import memoize _AVAILABLE_MKSPECS = ["ninja", "msvc", "mingw"] if sys.platform == "win32" else ["ninja", "make"] @@ -63,6 +66,9 @@ Additional options: ---macos-use-libc++ Use libc++ on macOS --snapshot-build Snapshot build --package-timestamp Package Timestamp + --cmake-toolchain-file Path to CMake toolchain to enable cross-compiling + --shiboken-host-path Path to host shiboken package when cross-compiling + --qt-host-path Path to host Qt installation when cross-compiling """ @@ -164,7 +170,7 @@ def _jobs_option_value(): # Declare options which need to be known when instantiating the DistUtils -# commands. +# commands or even earlier during SetupRunner.run(). OPTION = { "BUILD_TYPE": option_value("build-type"), "INTERNAL_BUILD_TYPE": option_value("internal-build-type"), @@ -179,7 +185,11 @@ OPTION = { "PACKAGE_TIMESTAMP": option_value("package-timestamp"), # This is used automatically by distutils.command.install object, to # specify the final installation location. - "FINAL_INSTALL_PREFIX": option_value("prefix", remove=False) + "FINAL_INSTALL_PREFIX": option_value("prefix", remove=False), + "CMAKE_TOOLCHAIN_FILE": option_value("cmake-toolchain-file"), + "SHIBOKEN_HOST_PATH": option_value("shiboken-host-path"), + "SHIBOKEN_HOST_PATH_QUERY_FILE": option_value("internal-shiboken-host-path-query-file"), + "QT_HOST_PATH": option_value("qt-host-path") # This is used to identify the template for doc builds } _deprecated_option_jobs = option_value('jobs') @@ -191,7 +201,7 @@ if _deprecated_option_jobs: class DistUtilsCommandMixin(object): """Mixin for the DistUtils build/install commands handling the options.""" - _finalized = False + _static_class_finalized_once = False mixin_user_options = [ ('avoid-protected-hack', None, 'Force --avoid-protected-hack'), @@ -217,9 +227,16 @@ class DistUtilsCommandMixin(object): ('qtpaths=', None, 'Path to qtpaths'), ('qmake=', None, 'Path to qmake (deprecated, use qtpaths)'), ('qt=', None, 'Qt version'), + ('qt-target-path=', None, + 'Path to device Qt installation (use Qt libs when cross-compiling)'), ('cmake=', None, 'Path to CMake'), ('openssl=', None, 'Path to OpenSSL libraries'), + + # FIXME: Deprecated in favor of shiboken-target-path ('shiboken-config-dir=', None, 'shiboken configuration directory'), + + ('shiboken-target-path=', None, 'Path to target shiboken package'), + ('python-target-path=', None, 'Path to target Python installation / prefix'), ('make-spec=', None, 'Qt make-spec'), ('macos-arch=', None, 'macOS architecture'), ('macos-sysroot=', None, 'macOS sysroot'), @@ -230,7 +247,13 @@ class DistUtilsCommandMixin(object): ('qt-conf-prefix=', None, 'Qt configuration prefix'), ('qt-src-dir=', None, 'Qt source directory'), ('no-qt-tools', None, 'Do not copy the Qt tools'), - ('pyside-numpy-support', None, 'libpyside: Add (experimental) numpy support') + ('pyside-numpy-support', None, 'libpyside: Add (experimental) numpy support'), + ('internal-cmake-install-dir-query-file-path=', None, + 'Path to file where the CMake install path of the project will be saved'), + + # We redeclare plat-name as an option so it's recognized by the + # install command and doesn't throw an error. + ('plat-name=', None, 'The platform name for which we are cross-compiling'), ] def __init__(self): @@ -259,9 +282,17 @@ class DistUtilsCommandMixin(object): self.qmake = None self.has_qmake_option = False self.qt = '5' + self.qt_host_path = None + self.qt_target_path = None self.cmake = None self.openssl = None self.shiboken_config_dir = None + self.shiboken_host_path = None + self.shiboken_host_path_query_file = None + self.shiboken_target_path = None + self.python_target_path = None + self.is_cross_compile = False + self.cmake_toolchain_file = None self.make_spec = None self.macos_arch = None self.macos_sysroot = None @@ -273,16 +304,62 @@ class DistUtilsCommandMixin(object): self.qt_src_dir = None self.no_qt_tools = False self.pyside_numpy_support = False + self.plat_name = None + self.internal_cmake_install_dir_query_file_path = None + self._per_command_mixin_options_finalized = False + + # When initializing a command other than the main one (so the + # first one), we need to copy the user options from the main + # command to the new command options dict. Then + # Distribution.get_command_obj will pick up the copied options + # ensuring that all commands that inherit from + # the mixin, get our custom properties set by the time + # finalize_options is called. + if DistUtilsCommandMixin._static_class_finalized_once: + current_command: Command = self + dist = current_command.distribution + main_command_name = dist.commands[0] + main_command_opts = dist.get_option_dict(main_command_name) + current_command_name = current_command.get_command_name() + current_command_opts = dist.get_option_dict(current_command_name) + mixin_options_set = self.get_mixin_options_set() + for key, value in main_command_opts.items(): + if key not in current_command_opts and key in mixin_options_set: + current_command_opts[key] = value + + @staticmethod + @memoize + def get_mixin_options_set(): + keys = set() + for (name, _, _) in DistUtilsCommandMixin.mixin_user_options: + keys.add(name.rstrip("=").replace("-", "_")) + return keys + def mixin_finalize_options(self): - # Bail out on 2nd call to mixin_finalize_options() since that is the - # build command following the install command when invoking - # setup.py install - if not DistUtilsCommandMixin._finalized: - DistUtilsCommandMixin._finalized = True + # The very first we finalize options, record that. + if not DistUtilsCommandMixin._static_class_finalized_once: + DistUtilsCommandMixin._static_class_finalized_once = True + + # Ensure we finalize once per command object, rather than per + # setup.py invocation. We want to have the option values + # available in all commands that derive from the mixin. + if not self._per_command_mixin_options_finalized: + self._per_command_mixin_options_finalized = True self._do_finalize() def _do_finalize(self): + # is_cross_compile must be set before checking for qtpaths/qmake + # because we DON'T want those to be found when cross compiling. + # Currently when cross compiling, qt-target-path MUST be used. + using_cmake_toolchain_file = False + cmake_toolchain_file = None + if OPTION["CMAKE_TOOLCHAIN_FILE"]: + self.is_cross_compile = True + using_cmake_toolchain_file = True + cmake_toolchain_file = OPTION["CMAKE_TOOLCHAIN_FILE"] + self.cmake_toolchain_file = cmake_toolchain_file + if not self._determine_defaults_and_check(): sys.exit(-1) OPTION['AVOID_PROTECTED_HACK'] = self.avoid_protected_hack @@ -320,12 +397,62 @@ class DistUtilsCommandMixin(object): OPTION['QMAKE'] = qmake_abs_path OPTION['HAS_QMAKE_OPTION'] = self.has_qmake_option OPTION['QT_VERSION'] = self.qt + self.qt_host_path = OPTION['QT_HOST_PATH'] + OPTION['QT_TARGET_PATH'] = self.qt_target_path + + qt_target_path = None + if self.qt_target_path: + qt_target_path = self.qt_target_path + + # We use the CMake project to find host Qt if neither qmake or + # qtpaths is available. This happens when building the host + # tools in the overall cross-building process. + use_cmake = False + if using_cmake_toolchain_file or \ + (not self.qmake and not self.qtpaths and self.qt_target_path): + use_cmake = True + QtInfo().setup(qtpaths_abs_path, self.cmake, qmake_abs_path, - self.has_qmake_option) + self.has_qmake_option, + use_cmake=use_cmake, + qt_target_path=qt_target_path, + cmake_toolchain_file=cmake_toolchain_file) + + try: + QtInfo().prefix_dir + except Exception as e: + if not self.qt_target_path: + log.error( + "\nCould not find Qt. You can pass the --qt-target-path=<qt-dir> option as a " + "hint where to find Qt. Error was:\n\n\n") + else: + log.error( + f"\nCould not find Qt via provided option --qt-target-path={qt_target_path} " + "Error was:\n\n\n") + raise e OPTION['CMAKE'] = os.path.abspath(self.cmake) OPTION['OPENSSL'] = self.openssl OPTION['SHIBOKEN_CONFIG_DIR'] = self.shiboken_config_dir + if self.shiboken_config_dir: + _warn_deprecated_option('shiboken-config-dir', 'shiboken-target-path') + + self.shiboken_host_path = OPTION['SHIBOKEN_HOST_PATH'] + self.shiboken_host_path_query_file = OPTION['SHIBOKEN_HOST_PATH_QUERY_FILE'] + + if not self.shiboken_host_path and self.shiboken_host_path_query_file: + try: + queried_shiboken_host_path = Path(self.shiboken_host_path_query_file).read_text() + self.shiboken_host_path = queried_shiboken_host_path + OPTION['SHIBOKEN_HOST_PATH'] = queried_shiboken_host_path + except Exception as e: + log.error( + f"\n Could not find shiboken host tools via the query file: " + f"{self.shiboken_host_path_query_file:} Error was:\n\n\n") + raise e + + OPTION['SHIBOKEN_TARGET_PATH'] = self.shiboken_target_path + OPTION['PYTHON_TARGET_PATH'] = self.python_target_path OPTION['MAKESPEC'] = self.make_spec OPTION['MACOS_ARCH'] = self.macos_arch OPTION['MACOS_SYSROOT'] = self.macos_sysroot @@ -338,6 +465,15 @@ class DistUtilsCommandMixin(object): OPTION['NO_QT_TOOLS'] = self.no_qt_tools OPTION['PYSIDE_NUMPY_SUPPORT'] = self.pyside_numpy_support + if not self._extra_checks(): + sys.exit(-1) + + def _extra_checks(self): + if self.is_cross_compile and not self.plat_name: + log.error(f"No value provided to --plat-name while cross-compiling.") + return False + return True + def _find_qtpaths_in_path(self): if not self.qtpaths: self.qtpaths = which("qtpaths") @@ -354,30 +490,43 @@ class DistUtilsCommandMixin(object): log.error(f"'{self.cmake}' does not exist.") return False - # Enforce usage of qmake in QtInfo if it was given explicitly. - if self.qmake: - self.has_qmake_option = True - _warn_deprecated_option('qmake', 'qtpaths') - - # If no option was given explicitly, prefer to find qtpaths - # in PATH. - if not self.qmake and not self.qtpaths: - self._find_qtpaths_in_path() - - # If no tool was specified and qtpaths was not found in PATH, - # ask to provide a path to qtpaths. - if not self.qtpaths and not self.qmake: - log.error("No value provided to --qtpaths option. Please provide one to find Qt.") - return False - - # Validate that the given tool path exists. - if self.qtpaths and not os.path.exists(self.qtpaths): - log.error(f"The specified qtpaths path '{self.qtpaths}' does not exist.") - return False - - if self.qmake and not os.path.exists(self.qmake): - log.error(f"The specified qmake path '{self.qmake}' does not exist.") - return False + # When cross-compiling, we only accept the qt-target-path + # option and don't rely on auto-searching in PATH or the other + # qtpaths / qmake options. + # We also don't do auto-searching if qt-target-path is passed + # explicitly. This is to help with the building of host tools + # while cross-compiling. + if not self.is_cross_compile and not self.qt_target_path: + # Enforce usage of qmake in QtInfo if it was given explicitly. + if self.qmake: + self.has_qmake_option = True + _warn_deprecated_option('qmake', 'qtpaths') + + # If no option was given explicitly, prefer to find qtpaths + # in PATH. + if not self.qmake and not self.qtpaths: + self._find_qtpaths_in_path() + + # If no tool was specified and qtpaths was not found in PATH, + # ask to provide a path to qtpaths. + if not self.qtpaths and not self.qmake and not self.qt_target_path: + log.error("No value provided to --qtpaths option. Please provide one to find Qt.") + return False + + # Validate that the given tool path exists. + if self.qtpaths and not os.path.exists(self.qtpaths): + log.error(f"The specified qtpaths path '{self.qtpaths}' does not exist.") + return False + + if self.qmake and not os.path.exists(self.qmake): + log.error(f"The specified qmake path '{self.qmake}' does not exist.") + return False + else: + # Check for existence, but don't require if it's not set. A + # check later will be done to see if it's needed. + if self.qt_target_path and not os.path.exists(self.qt_target_path): + log.error(f"Provided --qt-target-path='{self.qt_target_path}' path does not exist.") + return False if not self.make_spec: self.make_spec = _AVAILABLE_MKSPECS[0] diff --git a/build_scripts/platforms/linux.py b/build_scripts/platforms/linux.py index 324b962db..31e2613ef 100644 --- a/build_scripts/platforms/linux.py +++ b/build_scripts/platforms/linux.py @@ -89,6 +89,9 @@ def prepare_standalone_package_linux(self, vars): if not maybe_icu_libs: copy_icu_libs(self._patchelf_path, resolved_destination_lib_dir) + # Set RPATH for Qt libs. + self.update_rpath_for_linux_qt_libraries(destination_lib_dir.format(**vars)) + # Patching designer to use the Qt libraries provided in the wheel if config.is_internal_pyside_build(): assistant_path = "{st_build_dir}/{st_package_name}/assistant".format(**vars) @@ -116,15 +119,26 @@ def prepare_standalone_package_linux(self, vars): recursive=False, vars=vars) + copied_plugins = self.get_shared_libraries_in_path_recursively( + plugins_target.format(**vars)) + self.update_rpath_for_linux_plugins(copied_plugins) + if copy_qml: # <qt>/qml/* -> <setup>/{st_package_name}/Qt/qml + qml_plugins_target = "{st_build_dir}/{st_package_name}/Qt/qml" copydir("{qt_qml_dir}", - "{st_build_dir}/{st_package_name}/Qt/qml", + qml_plugins_target, filter=None, force=False, recursive=True, ignore=["*.so.debug"], vars=vars) + copied_plugins = self.get_shared_libraries_in_path_recursively( + qml_plugins_target.format(**vars)) + self.update_rpath_for_linux_plugins( + copied_plugins, + qt_lib_dir=destination_lib_dir.format(**vars), + is_qml_plugin=True) if copy_translations: # <qt>/translations/* -> diff --git a/build_scripts/qtinfo.py b/build_scripts/qtinfo.py index f038adddc..95e087096 100644 --- a/build_scripts/qtinfo.py +++ b/build_scripts/qtinfo.py @@ -43,27 +43,10 @@ import re import subprocess import tempfile from pathlib import Path - +from .utils import configure_cmake_project, parse_cmake_project_message_info from .utils import platform_cmake_options -_CMAKE_LISTS = """cmake_minimum_required(VERSION 3.16) -project(dummy LANGUAGES CXX) - -find_package(Qt6 COMPONENTS Core) - -get_target_property(darwin_target Qt6::Core QT_DARWIN_MIN_DEPLOYMENT_TARGET) -message(STATUS "mkspec_qt_darwin_min_deployment_target=${darwin_target}") - -if(QT_FEATURE_debug_and_release) - message(STATUS "mkspec_build_type=debug_and_release") -elseif(QT_FEATURE_debug) - message(STATUS "mkspec_build_type=debug") -else() - message(STATUS "mkspec_build_type=release") -endif() -""" - class QtInfo(object): _instance = None # singleton helpers @@ -85,14 +68,21 @@ class QtInfo(object): self._cmake_command = None self._qmake_command = None self._force_qmake = False + self._use_cmake = False + self._qt_target_path = None + self._cmake_toolchain_file = None # Dict to cache qmake values. self._query_dict = {} - def setup(self, qtpaths, cmake, qmake, force_qmake): + def setup(self, qtpaths, cmake, qmake, force_qmake, use_cmake, qt_target_path, + cmake_toolchain_file): self._qtpaths_command = qtpaths self._cmake_command = cmake self._qmake_command = qmake self._force_qmake = force_qmake + self._use_cmake = use_cmake + self._qt_target_path = qt_target_path + self._cmake_toolchain_file = cmake_toolchain_file @property def qmake_command(self): @@ -213,68 +203,75 @@ class QtInfo(object): return props def _get_query_properties(self): - if self._force_qmake: - output = self._get_qmake_output(["-query"]) + if self._use_cmake: + setup_script_dir = Path.cwd() + sources_dir = setup_script_dir / "sources" + qt_target_info_dir = sources_dir / "shiboken6" / "config.tests" / "target_qt_info" + qt_target_info_dir = os.fspath(qt_target_info_dir) + config_tests_dir = setup_script_dir / "build" / "config.tests" + config_tests_dir = os.fspath(config_tests_dir) + + cmake_cache_args = [] + if self._cmake_toolchain_file: + cmake_cache_args.append(("CMAKE_TOOLCHAIN_FILE", self._cmake_toolchain_file)) + + if self._qt_target_path: + cmake_cache_args.append(("QFP_QT_TARGET_PATH", self._qt_target_path)) + qt_target_info_output = configure_cmake_project( + qt_target_info_dir, + self._cmake_command, + temp_prefix_build_path=config_tests_dir, + cmake_cache_args=cmake_cache_args) + qt_target_info = parse_cmake_project_message_info(qt_target_info_output) + self._query_dict = qt_target_info['qt_info'] else: - output = self._get_qtpaths_output(["--qt-query"]) - self._query_dict = self._parse_query_properties(output) + if self._force_qmake: + output = self._get_qmake_output(["-query"]) + else: + output = self._get_qtpaths_output(["--qt-query"]) + self._query_dict = self._parse_query_properties(output) def _get_other_properties(self): # Get the src property separately, because it is not returned by # qmake unless explicitly specified. key = "QT_INSTALL_PREFIX/src" - if self._force_qmake: - result = self._get_qmake_output(["-query", key]) - else: - result = self._get_qtpaths_output(["--qt-query", key]) - self._query_dict[key] = result + if not self._use_cmake: + if self._force_qmake: + result = self._get_qmake_output(["-query", key]) + else: + result = self._get_qtpaths_output(["--qt-query", key]) + self._query_dict[key] = result # Get mkspecs variables and cache them. # FIXME Python 3.9 self._query_dict |= other_dict for key, value in self._get_cmake_mkspecs_variables().items(): self._query_dict[key] = value - @staticmethod - def _parse_cmake_mkspecs_variables(output): - # Helper for _get_cmake_mkspecs_variables(). Parse the output for - # anything prefixed '-- mkspec_' as created by the message() calls - # in _CMAKE_LISTS. - result = {} - pattern = re.compile(r"^-- mkspec_(.*)=(.*)$") - for line in output.splitlines(): - found = pattern.search(line.strip()) - if found: - key = found.group(1).strip() - value = found.group(2).strip() - # Get macOS minimum deployment target. - if key == 'qt_darwin_min_deployment_target': - result['QMAKE_MACOSX_DEPLOYMENT_TARGET'] = value - # Figure out how Qt was built - elif key == 'build_type': - result['BUILD_TYPE'] = value - return result - def _get_cmake_mkspecs_variables(self): - # Create an empty cmake project file in a temporary directory and - # parse the output to determine some mkspec values. - output = '' - error = '' - return_code = 0 - with tempfile.TemporaryDirectory() as tempdir: - cmake_list_file = Path(tempdir) / 'CMakeLists.txt' - cmake_list_file.write_text(_CMAKE_LISTS) - cmd = [self._cmake_command, '-G', 'Ninja', '.'] - qt_prefix = self.prefix_dir - cmd.extend([f'-DCMAKE_PREFIX_PATH={qt_prefix}']) - cmd += platform_cmake_options() - - # FIXME Python 3.7: Use subprocess.run() - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=False, - cwd=tempdir, universal_newlines=True) - output, error = proc.communicate() - proc.wait() - return_code = proc.returncode - - if return_code != 0: - raise RuntimeError(f"Could not determine cmake variables: {error}") - return QtInfo.__QtInfo._parse_cmake_mkspecs_variables(output) + setup_script_dir = Path.cwd() + sources_dir = setup_script_dir / "sources" + qt_target_mkspec_dir = sources_dir / "shiboken6" / "config.tests" / "target_qt_mkspec" + qt_target_mkspec_dir = qt_target_mkspec_dir.as_posix() + config_tests_dir = setup_script_dir / "build" / "config.tests" + config_tests_dir = config_tests_dir.as_posix() + + cmake_cache_args = [] + if self._cmake_toolchain_file: + cmake_cache_args.append(("CMAKE_TOOLCHAIN_FILE", self._cmake_toolchain_file)) + if self._qt_target_path: + cmake_cache_args.append(("QFP_QT_TARGET_PATH", self._qt_target_path)) + else: + qt_prefix = Path(self.prefix_dir).as_posix() + cmake_cache_args.append(("CMAKE_PREFIX_PATH", qt_prefix)) + + cmake_cache_args.extend(platform_cmake_options(as_tuple_list=True)) + qt_target_mkspec_output = configure_cmake_project( + qt_target_mkspec_dir, + self._cmake_command, + temp_prefix_build_path=config_tests_dir, + cmake_cache_args=cmake_cache_args) + + qt_target_mkspec_info = parse_cmake_project_message_info(qt_target_mkspec_output) + qt_target_mkspec_info = qt_target_mkspec_info['qt_info'] + + return qt_target_mkspec_info diff --git a/build_scripts/setup_runner.py b/build_scripts/setup_runner.py index f77b9d1fe..362432d2a 100644 --- a/build_scripts/setup_runner.py +++ b/build_scripts/setup_runner.py @@ -39,6 +39,7 @@ import sys import os +import tempfile import textwrap from setuptools import setup # Import setuptools before distutils @@ -68,6 +69,16 @@ class SetupRunner(object): return any(arg for arg in list(args) if "--" + argument in arg) @staticmethod + def get_cmd_line_argument_in_args(argument, args): + """ Gets the value of a cmd line argument passed in args. """ + for arg in list(args): + if "--" + argument in arg: + prefix = f"--{argument}" + prefix_len = len(prefix) + 1 + return arg[prefix_len:] + return None + + @staticmethod def remove_cmd_line_argument_in_args(argument, args): """ Remove command line argument from args. """ return [arg for arg in list(args) if "--" + argument not in arg] @@ -83,20 +94,107 @@ class SetupRunner(object): def construct_internal_build_type_cmd_line_argument(internal_build_type): return SetupRunner.construct_cmd_line_argument("internal-build-type", internal_build_type) - def add_setup_internal_invocation(self, build_type, reuse_build=False): - """ Enqueues a script sub-invocation to be executed later. """ + def enqueue_setup_internal_invocation(self, setup_cmd): + self.invocations_list.append(setup_cmd) + + def add_setup_internal_invocation(self, build_type, reuse_build=False, extra_args=None): + setup_cmd = self.new_setup_internal_invocation(build_type, reuse_build, extra_args) + self.enqueue_setup_internal_invocation(setup_cmd) + + def new_setup_internal_invocation(self, build_type, + reuse_build=False, + extra_args=None, + replace_command_with=None): + """ Creates a script sub-invocation to be executed later. """ internal_build_type_arg = self.construct_internal_build_type_cmd_line_argument(build_type) - setup_cmd = [sys.executable] + self.sub_argv + [internal_build_type_arg] - command = self.sub_argv[0] + command_index = 0 + command = self.sub_argv[command_index] if command == 'setup.py' and len(self.sub_argv) > 1: - command = self.sub_argv[1] + command_index = 1 + command = self.sub_argv[command_index] + + # Make a copy + modified_argv = list(self.sub_argv) + + if replace_command_with: + modified_argv[command_index] = replace_command_with + + setup_cmd = [sys.executable] + modified_argv + [internal_build_type_arg] + + if extra_args: + for (name, value) in extra_args: + setup_cmd.append(self.construct_cmd_line_argument(name, value)) # Add --reuse-build option if requested and not already present. if (reuse_build and command in ('bdist_wheel', 'build', 'build_rst_docs', 'install') - and not self.cmd_line_argument_is_in_args("reuse-build", self.sub_argv)): + and not self.cmd_line_argument_is_in_args("reuse-build", modified_argv)): setup_cmd.append(self.construct_cmd_line_argument("reuse-build")) - self.invocations_list.append(setup_cmd) + return setup_cmd + + def add_host_tools_setup_internal_invocation(self, initialized_config): + extra_args = [] + extra_host_args = [] + + # When cross-compiling, build the host shiboken generator tool + # only if a path to an existing one was not provided. + if not self.cmd_line_argument_is_in_args("shiboken-host-path", self.sub_argv): + handle, initialized_config.shiboken_host_query_path = tempfile.mkstemp() + os.close(handle) + + # Tell the setup process to create a file with the location + # of the installed host shiboken as its contents. + extra_host_args.append( + ("internal-cmake-install-dir-query-file-path", + initialized_config.shiboken_host_query_path)) + + # Tell the other setup invocations to read that file and use + # the read path as the location of the host shiboken. + extra_args.append( + ("internal-shiboken-host-path-query-file", + initialized_config.shiboken_host_query_path) + ) + + # This is specifying shiboken_module_option_name + # instead of shiboken_generator_option_name, but it will + # actually build the generator. + host_cmd = self.new_setup_internal_invocation( + initialized_config.shiboken_module_option_name, + extra_args=extra_host_args, + replace_command_with="build") + + # To build the host tools, we reuse the initial target + # command line arguments, but we remove some options that + # don't make sense for the host build. + + # Drop the toolchain arg. + host_cmd = self.remove_cmd_line_argument_in_args("cmake-toolchain-file", + host_cmd) + + # Drop the target plat-name arg if there is one. + if self.cmd_line_argument_is_in_args("plat-name", host_cmd): + host_cmd = self.remove_cmd_line_argument_in_args("plat-name", host_cmd) + + # Drop the python-target-path arg if there is one. + if self.cmd_line_argument_is_in_args("python-target-path", host_cmd): + host_cmd = self.remove_cmd_line_argument_in_args("python-target-path", host_cmd) + + # Drop the target build-tests arg if there is one. + if self.cmd_line_argument_is_in_args("build-tests", host_cmd): + host_cmd = self.remove_cmd_line_argument_in_args("build-tests", host_cmd) + + # Make sure to pass the qt host path as the target path + # when doing the host build. And make sure to remove any + # existing qt target path. + if self.cmd_line_argument_is_in_args("qt-host-path", host_cmd): + qt_host_path = self.get_cmd_line_argument_in_args("qt-host-path", host_cmd) + host_cmd = self.remove_cmd_line_argument_in_args("qt-host-path", host_cmd) + host_cmd = self.remove_cmd_line_argument_in_args("qt-target-path", host_cmd) + host_cmd.append(self.construct_cmd_line_argument("qt-target-path", + qt_host_path)) + + self.enqueue_setup_internal_invocation(host_cmd) + return extra_args def run_setup(self): """ @@ -118,6 +216,7 @@ class SetupRunner(object): package_version=get_package_version(), ext_modules=get_setuptools_extension_modules(), setup_script_dir=self.setup_script_dir, + cmake_toolchain_file=OPTION["CMAKE_TOOLCHAIN_FILE"], quiet=OPTION["QUIET"]) # Enable logging for both the top-level invocation of setup.py @@ -149,18 +248,33 @@ class SetupRunner(object): # Build everything: shiboken6, shiboken6-generator and PySide6. help_requested = '--help' in self.sub_argv or '-h' in self.sub_argv + if help_requested: self.add_setup_internal_invocation(config.pyside_option_name) elif config.is_top_level_build_all(): - self.add_setup_internal_invocation(config.shiboken_module_option_name) + extra_args = [] + + # extra_args might contain the location of the built host + # shiboken, which needs to be passed to the other + # target invocations. + if config.is_cross_compile(): + extra_args = self.add_host_tools_setup_internal_invocation(config) + + self.add_setup_internal_invocation( + config.shiboken_module_option_name, + extra_args=extra_args) # Reuse the shiboken build for the generator package instead # of rebuilding it again. - self.add_setup_internal_invocation(config.shiboken_generator_option_name, - reuse_build=True) + # Don't build it in a cross-build though. + if not config.is_cross_compile(): + self.add_setup_internal_invocation( + config.shiboken_generator_option_name, + reuse_build=True) - self.add_setup_internal_invocation(config.pyside_option_name) + self.add_setup_internal_invocation(config.pyside_option_name, + extra_args=extra_args) elif config.is_top_level_build_shiboken_module(): self.add_setup_internal_invocation(config.shiboken_module_option_name) @@ -184,6 +298,9 @@ class SetupRunner(object): if help_requested: print(ADDITIONAL_OPTIONS) + # Cleanup temp query file. + if config.shiboken_host_query_path: + os.remove(config.shiboken_host_query_path) @staticmethod def run_setuptools_setup(): diff --git a/build_scripts/utils.py b/build_scripts/utils.py index 39cff6743..2daad642e 100644 --- a/build_scripts/utils.py +++ b/build_scripts/utils.py @@ -48,6 +48,8 @@ import subprocess import fnmatch import itertools import glob +import tempfile +from collections import defaultdict import urllib.request as urllib @@ -244,12 +246,16 @@ def init_msvc_env(platform_arch, build_type): log.info("Done initializing MSVC env") -def platform_cmake_options(): +def platform_cmake_options(as_tuple_list=False): result = [] if sys.platform == 'win32': # Prevent cmake from auto-detecting clang if it is in path. - result.append("-DCMAKE_C_COMPILER=cl.exe") - result.append("-DCMAKE_CXX_COMPILER=cl.exe") + if as_tuple_list: + result.append(("CMAKE_C_COMPILER", "cl.exe")) + result.append(("CMAKE_CXX_COMPILER", "cl.exe")) + else: + result.append("-DCMAKE_C_COMPILER=cl.exe") + result.append("-DCMAKE_CXX_COMPILER=cl.exe") return result @@ -1250,3 +1256,69 @@ def parse_cmake_conf_assignments_by_key(source_dir): d[key] = value return d + +def configure_cmake_project(project_path, + cmake_path, + build_path=None, + temp_prefix_build_path=None, + cmake_args=None, + cmake_cache_args=None, + ): + clean_temp_dir = False + if not build_path: + # Ensure parent dir exists. + if temp_prefix_build_path: + os.makedirs(temp_prefix_build_path, exist_ok=True) + + project_name = Path(project_path).name + build_path = tempfile.mkdtemp(prefix=f"{project_name}_", dir=temp_prefix_build_path) + + if 'QFP_SETUP_KEEP_TEMP_FILES' not in os.environ: + clean_temp_dir = True + + cmd = [cmake_path, '-G', 'Ninja', '-S', project_path, '-B', build_path] + + if cmake_args: + cmd.extend(cmake_args) + + for arg, value in cmake_cache_args: + cmd.extend([f'-D{arg}={value}']) + + cmd_string = ' '.join(cmd) + # FIXME Python 3.7: Use subprocess.run() + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + cwd=build_path, + universal_newlines=True) + output, error = proc.communicate() + proc.wait() + return_code = proc.returncode + + if return_code != 0: + raise RuntimeError(f"\nFailed to configure CMake project \n " + f"'{project_path}' \n with error: \n {error}\n " + f"Return code: {return_code}\n" + f"Configure args were:\n {cmd_string}") + + if clean_temp_dir: + rmtree(build_path) + + return output + + +def parse_cmake_project_message_info(output): + # Parse the output for anything prefixed + # '-- qfp:<category>:<key>: <value>' as created by the message() + # calls in a given CMake project and store it in a python dict. + result = defaultdict(lambda: defaultdict(str)) + pattern = re.compile(r"^-- qfp:(.+?):(.+?):(.*)$") + for line in output.splitlines(): + found = pattern.search(line) + if found: + category = found.group(1).strip() + key = found.group(2).strip() + value = found.group(3).strip() + result[category][key] = str(value) + return result diff --git a/build_scripts/wheel_override.py b/build_scripts/wheel_override.py index 4706863d5..ce5ba11ab 100644 --- a/build_scripts/wheel_override.py +++ b/build_scripts/wheel_override.py @@ -72,6 +72,7 @@ class PysideBuildWheel(_bdist_wheel, DistUtilsCommandMixin): if wheel_module_exists else None) def __init__(self, *args, **kwargs): + self.command_name = "bdist_wheel" self._package_version = None _bdist_wheel.__init__(self, *args, **kwargs) DistUtilsCommandMixin.__init__(self) @@ -108,12 +109,113 @@ class PysideBuildWheel(_bdist_wheel, DistUtilsCommandMixin): components += (self.build_number,) return '-'.join(components) - # Copy of get_tag from bdist_wheel.py, to allow setting a - # multi-python impl tag, by removing an assert. Otherwise we - # would have to rename wheels manually for limited api - # packages. Also we set "none" abi tag on Windows, because - # pip does not yet support "abi3" tag, leading to - # installation failure when tried. + # Modify the returned wheel tag tuple to use correct python version + # info when cross-compiling. We use the python info extracted from + # the shiboken python config test. + # setuptools / wheel don't support cross compiling out of the box + # at the moment. Relevant discussion at + # https://discuss.python.org/t/towards-standardizing-cross-compiling/10357 + def get_cross_compiling_tag_tuple(self, tag_tuple): + (old_impl, old_abi_tag, plat_name) = tag_tuple + + # Compute tag from the python version that the build command + # queried. + build_command = self.get_finalized_command('build') + python_target_info = build_command.python_target_info['python_info'] + + impl = 'no-py-ver-impl-available' + abi = 'no-abi-tag-info-available' + py_version = python_target_info['version'].split('.') + py_version_major = py_version[0] + py_version_minor = py_version[1] + + so_abi = python_target_info['so_abi'] + if so_abi and so_abi.startswith('cpython-'): + interpreter_name = so_abi.split('-')[0] + impl_name = tags.INTERPRETER_SHORT_NAMES.get(interpreter_name) or interpreter_name + impl_ver = f"{py_version_major}{py_version_minor}" + impl = impl_name + impl_ver + abi = 'cp' + so_abi.split('-')[1] + tag_tuple = (impl, abi, plat_name) + return tag_tuple + + # Adjust wheel tag for limited api and cross compilation. + @staticmethod + def adjust_cross_compiled_many_linux_tag(old_tag): + (old_impl, old_abi_tag, old_plat_name) = old_tag + + new_plat_name = old_plat_name + + # TODO: Detect glibc version instead. We're abusing the + # manylinux2014 tag here, just like we did with manylinux1 + # for x86_64 builds. + many_linux_prefix = 'manylinux2014' + linux_prefix = "linux_" + if old_plat_name.startswith(linux_prefix): + # Extract the arch suffix like -armv7l or -aarch64 + plat_name_arch_suffix = \ + old_plat_name[old_plat_name.index(linux_prefix) + len(linux_prefix):] + + new_plat_name = f"{many_linux_prefix}_{plat_name_arch_suffix}" + + tag = (old_impl, old_abi_tag, new_plat_name) + return tag + + # Adjust wheel tag for limited api and cross compilation. + def adjust_tag_and_supported_tags(self, old_tag, supported_tags): + tag = old_tag + (old_impl, old_abi_tag, old_plat_name) = old_tag + + # Get new tag for cross builds. + if self.is_cross_compile: + tag = self.get_cross_compiling_tag_tuple(old_tag) + + # Get new tag for manylinux builds. + # To allow uploading to pypi, we need the wheel name + # to contain 'manylinux1'. + # The wheel which will be uploaded to pypi will be + # built on RHEL_8_2, so it doesn't completely qualify for + # manylinux1 support, but it's the minimum requirement + # for building Qt. We only enable this for x64 limited + # api builds (which are the only ones uploaded to pypi). + # TODO: Add actual distro detection, instead of + # relying on limited_api option if possible. + if (old_plat_name in ('linux-x86_64', 'linux_x86_64') + and sys.maxsize > 2147483647 + and self.py_limited_api): + tag = (old_impl, old_abi_tag, 'manylinux1_x86_64') + + # Set manylinux tag for cross-compiled builds when targeting + # limited api. + if self.is_cross_compile and self.py_limited_api: + tag = self.adjust_cross_compiled_many_linux_tag(tag) + + # Reset the abi name and python versions supported by this wheel + # when targeting limited API. This is the same code that's + # in get_tag(), but done later after our own customizations. + if self.py_limited_api and old_impl.startswith('cp3'): + (_, _, adjusted_plat_name) = tag + impl = self.py_limited_api + abi_tag = 'abi3' + tag = (impl, abi_tag, adjusted_plat_name) + + # Adjust abi name on Windows for limited api. It needs to be + # 'none' instead of 'abi3' because pip does not yet support + # the "abi3" tag on Windows, leading to a installation failure. + if self.py_limited_api and old_impl.startswith('cp3') and sys.platform == 'win32': + tag = (old_impl, 'none', old_plat_name) + + # If building for limited API or we created a new tag, add it + # to the list of supported tags. + if tag != old_tag or self.py_limited_api: + supported_tags.append(tag) + return tag + + # A slightly modified copy of get_tag from bdist_wheel.py, to allow + # adjusting the returned tag without triggering an assert. Otherwise + # we would have to rename wheels manually. + # Copy is up-to-date since commit + # 0acd203cd896afec7f715aa2ff5980a403459a3b in the wheel repo. def get_tag(self): # bdist sets self.plat_name if unset, we should only use it for purepy # wheels if the user supplied it. @@ -137,21 +239,7 @@ class PysideBuildWheel(_bdist_wheel, DistUtilsCommandMixin): if plat_name in ('linux-x86_64', 'linux_x86_64') and sys.maxsize == 2147483647: plat_name = 'linux_i686' - # To allow uploading to pypi, we need the wheel name - # to contain 'manylinux1'. - # The wheel which will be uploaded to pypi will be - # built on RHEL7, so it doesn't completely qualify for - # manylinux1 support, but it's the minimum requirement - # for building Qt. We only enable this for x64 limited - # api builds (which are the only ones uploaded to - # pypi). - # TODO: Add actual distro detection, instead of - # relying on limited_api option. - if (plat_name in ('linux-x86_64', 'linux_x86_64') - and sys.maxsize > 2147483647 - and (self.py_limited_api)): - plat_name = 'manylinux1_x86_64' - plat_name = plat_name.replace('-', '_').replace('.', '_') + plat_name = plat_name.lower().replace('-', '_').replace('.', '_') if self.root_is_pure: if self.universal: @@ -166,15 +254,15 @@ class PysideBuildWheel(_bdist_wheel, DistUtilsCommandMixin): # We don't work on CPython 3.1, 3.0. if self.py_limited_api and (impl_name + impl_ver).startswith('cp3'): impl = self.py_limited_api - abi_tag = "abi3" if sys.platform != "win32" else "none" + abi_tag = 'abi3' else: abi_tag = str(get_abi_tag()).lower() tag = (impl, abi_tag, plat_name) - supported_tags = [(t.interpreter, t.abi, t.platform) + # issue gh-374: allow overriding plat_name + supported_tags = [(t.interpreter, t.abi, plat_name) for t in tags.sys_tags()] - # XXX switch to this alternate implementation for non-pure: - if (self.py_limited_api) or (plat_name in ('manylinux1_x86_64')): - return tag + # PySide's custom override. + tag = self.adjust_tag_and_supported_tags(tag, supported_tags) assert tag in supported_tags, (f"would build wheel with unsupported tag {tag}") return tag diff --git a/sources/shiboken6/config.tests/target_qt_mkspec/CMakeLists.txt b/sources/shiboken6/config.tests/target_qt_mkspec/CMakeLists.txt new file mode 100644 index 000000000..033369188 --- /dev/null +++ b/sources/shiboken6/config.tests/target_qt_mkspec/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.16) +project(dummy LANGUAGES CXX) + +include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/ShibokenHelpers.cmake") + +shiboken_internal_detect_if_cross_building() +shiboken_internal_set_up_extra_dependency_paths() +find_package(Qt6 REQUIRED COMPONENTS Core) + +get_target_property(darwin_target Qt6::Core QT_DARWIN_MIN_DEPLOYMENT_TARGET) + +# Get macOS minimum deployment target +message(STATUS "qfp:qt_info:QMAKE_MACOSX_DEPLOYMENT_TARGET: ${darwin_target}") + +# Get Qt build type +if(QT_FEATURE_debug_and_release) + message(STATUS "qfp:qt_info:BUILD_TYPE: debug_and_release") +elseif(QT_FEATURE_debug) + message(STATUS "qfp:qt_info:BUILD_TYPE: debug") +else() + message(STATUS "qfp:qt_info:BUILD_TYPE: release") +endif() |