aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexandru Croitor <alexandru.croitor@qt.io>2021-09-29 19:01:51 +0200
committerAlexandru Croitor <alexandru.croitor@qt.io>2022-02-04 15:51:04 +0100
commit57866a57586d401c784f809f9f7994b0e4623706 (patch)
tree48b9d5a7d1a03d1edd15359858adcefd18430467
parent14e4527cc427ce8c5e7c1758a95a1bbce0498471 (diff)
setup.py: Add support for cross-building
setup.py can now be used to cross-compile PySide to a target Linux distribution from a Linux host. For example you could cross-compile PySide targeting an arm64 Raspberry Pi4 sysroot on an Ubuntu x86_64 host machine. Cross-compiling PySide has a few requirements: - a sysroot to cross-compile against, with a pre-installed Qt, Python interpreter, library and development packages (which provides C++ headers) - a host Qt installation of the same version that is in the target sysroot - a host Python installation, preferably of the same version as the target one (to run setup.py) - a working cross-compiling toolchain (cross-compiler, linker, etc) - a custom written CMake toolchain file - CMake version 3.17+ - Qt version 6.3+ The CMake toolchain file is required to set up all the relevant cross-compilation information: where the sysroot is, where the toolchain is, the compiler name, compiler flags, etc. Once are requirements are met, to cross-compile one has to specify a few additional options when calling setup.py: the path to the cmake toolchain file, the path to the host Qt installation and the target python platform name. An example setup.py invocation to build a wheel for an armv7 machine might look like the following: python setup.py bdist_wheel --parallel=8 --ignore-git --reuse-build --cmake-toolchain-file=$PWD/rpi/toolchain_armv7.cmake --qt-host-path=/opt/Qt/6.3.0/gcc_64 --plat-name=linux_armv7l --limited-api=yes --standalone Sample platform names that can be used are: linux_armv7, linux_aarch64. If the auto-detection code fails to find the target Python or Qt installation, one can specify their location by providing the --python-target-path=<path> and --qt-target-path=<path> options to setup.py. If the automatic build of the host shiboken code generator fails, one can specify the path to a custom built host shiboken via the --shiboken-host-path option. Documentation about the build process and a sample CMake toolchain file will be added in a separate change. Implementation details. Internally, setup.py will build a host shiboken executable using the provided host Qt path, and then use it for the cross-build. This is achieved via an extra setup.py sub-invocation with some heuristics on which options should be passed to the sub-invocation. The host shiboken is not included in the target wheels. Introspection of where the host / target Qt and Python are located is done via CMake compile tests, because we can't query information from a qmake that is built for a different architecture / platform. When limited API is enabled, we modify the wheel name to contain the manylinux2014 tag, despite the wheel not fully qualifying for that tag. When copying the Qt libraries / plugins from the target sysroot in a standalone build, we need to adjust all their rpaths to match the destination directory layout of the wheel. Fixes: PYSIDE-802 Task-number: PYSIDE-1033 Change-Id: I6e8c51ef5127d85949de650396d615ca95194db0 Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io> Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
-rw-r--r--build_scripts/build_info_collector.py121
-rw-r--r--build_scripts/config.py25
-rw-r--r--build_scripts/main.py189
-rw-r--r--build_scripts/options.py217
-rw-r--r--build_scripts/platforms/linux.py16
-rw-r--r--build_scripts/qtinfo.py139
-rw-r--r--build_scripts/setup_runner.py139
-rw-r--r--build_scripts/utils.py78
-rw-r--r--build_scripts/wheel_override.py140
-rw-r--r--sources/shiboken6/config.tests/target_qt_mkspec/CMakeLists.txt22
10 files changed, 890 insertions, 196 deletions
diff --git a/build_scripts/build_info_collector.py b/build_scripts/build_info_collector.py
index f4146af51..592b4c873 100644
--- a/build_scripts/build_info_collector.py
+++ b/build_scripts/build_info_collector.py
@@ -48,6 +48,7 @@ from setuptools._distutils import sysconfig as sconfig
from .options import OPTION
from .qtinfo import QtInfo
+from .utils import configure_cmake_project, parse_cmake_project_message_info
from .wheel_utils import get_qt_version
@@ -166,6 +167,7 @@ class BuildInfoCollectorMixin(object):
build_lib: str
cmake: str
cmake_toolchain_file: str
+ internal_cmake_install_dir_query_file_path: str
is_cross_compile: bool
plat_name: str
python_target_path: str
@@ -185,43 +187,97 @@ class BuildInfoCollectorMixin(object):
sources_dir = os.path.join(script_dir, "sources")
- platform_arch = platform.architecture()[0]
- log.info(f"Python architecture is {platform_arch}")
- self.py_arch = platform_arch[:-3]
+ if self.is_cross_compile:
+ config_tests_dir = os.path.join(script_dir, build_base, "config.tests")
+ python_target_info_dir = os.path.join(sources_dir, "shiboken6", "config.tests",
+ "target_python_info")
+ cmake_cache_args = []
+
+ if self.python_target_path:
+ cmake_cache_args.append(("Python_ROOT_DIR", self.python_target_path))
+
+ if self.cmake_toolchain_file:
+ cmake_cache_args.append(("CMAKE_TOOLCHAIN_FILE", self.cmake_toolchain_file))
+ python_target_info_output = configure_cmake_project(
+ python_target_info_dir,
+ self.cmake,
+ temp_prefix_build_path=config_tests_dir,
+ cmake_cache_args=cmake_cache_args)
+ python_target_info = parse_cmake_project_message_info(python_target_info_output)
+ self.python_target_info = python_target_info
build_type = "Debug" if OPTION["DEBUG"] else "Release"
if OPTION["RELWITHDEBINFO"]:
build_type = 'RelWithDebInfo'
# Prepare parameters
- py_executable = sys.executable
- py_version = f"{sys.version_info[0]}.{sys.version_info[1]}"
- py_include_dir = get_config_var("INCLUDEPY")
- py_libdir = get_config_var("LIBDIR")
- # distutils.sysconfig.get_config_var('prefix') returned the
- # virtual environment base directory, but
- # sysconfig.get_config_var returns the system's prefix.
- # We use 'base' instead (although, platbase points to the
- # same location)
- py_prefix = get_config_var("base")
- if not py_prefix or not os.path.exists(py_prefix):
- py_prefix = sys.prefix
- self.py_prefix = py_prefix
- if sys.platform == "win32":
- py_scripts_dir = os.path.join(py_prefix, "Scripts")
+ if not self.is_cross_compile:
+ platform_arch = platform.architecture()[0]
+ self.py_arch = platform_arch[:-3]
+
+ py_executable = sys.executable
+ py_version = f"{sys.version_info[0]}.{sys.version_info[1]}"
+ py_include_dir = get_config_var("INCLUDEPY")
+ py_libdir = get_config_var("LIBDIR")
+ # distutils.sysconfig.get_config_var('prefix') returned the
+ # virtual environment base directory, but
+ # sysconfig.get_config_var returns the system's prefix.
+ # We use 'base' instead (although, platbase points to the
+ # same location)
+ py_prefix = get_config_var("base")
+ if not py_prefix or not os.path.exists(py_prefix):
+ py_prefix = sys.prefix
+ self.py_prefix = py_prefix
+ if sys.platform == "win32":
+ py_scripts_dir = os.path.join(py_prefix, "Scripts")
+ else:
+ py_scripts_dir = os.path.join(py_prefix, "bin")
+ self.py_scripts_dir = py_scripts_dir
else:
- py_scripts_dir = os.path.join(py_prefix, "bin")
- self.py_scripts_dir = py_scripts_dir
+ # We don't look for an interpreter when cross-compiling.
+ py_executable = None
+
+ python_info = self.python_target_info['python_info']
+ py_version = python_info['version'].split('.')
+ py_version = f"{py_version[0]}.{py_version[1]}"
+ py_include_dir = python_info['include_dirs']
+ py_libdir = python_info['library_dirs']
+ py_library = python_info['libraries']
+ self.py_library = py_library
+
+ # Prefix might not be set because the project that extracts
+ # the info is using internal API to get it. It shouldn't be
+ # critical though, because we don't really use neither
+ # py_prefix nor py_scripts_dir in important places
+ # when cross-compiling.
+ if 'prefix' in python_info:
+ py_prefix = python_info['prefix']
+ self.py_prefix = py_prefix
+
+ py_scripts_dir = os.path.join(py_prefix, 'bin')
+ if os.path.exists(py_scripts_dir):
+ self.py_scripts_dir = py_scripts_dir
+ else:
+ self.py_scripts_dir = None
+ else:
+ py_prefix = None
+ self.py_prefix = py_prefix
+ self.py_scripts_dir = None
self.qtinfo = QtInfo()
qt_version = get_qt_version()
# Used for test blacklists and registry test.
- self.build_classifiers = (f"py{py_version}-qt{qt_version}-{platform.architecture()[0]}-"
- f"{build_type.lower()}")
- if hasattr(sys, "pypy_version_info"):
- pypy_version = ".".join(map(str, sys.pypy_version_info[:3]))
- self.build_classifiers += f"-pypy.{pypy_version}"
+ if self.is_cross_compile:
+ # Querying the host platform architecture makes no sense when cross-compiling.
+ build_classifiers = f"py{py_version}-qt{qt_version}-{self.plat_name}-"
+ else:
+ build_classifiers = f"py{py_version}-qt{qt_version}-{platform.architecture()[0]}-"
+ if hasattr(sys, "pypy_version_info"):
+ pypy_version = ".".join(map(str, sys.pypy_version_info[:3]))
+ build_classifiers += f"pypy.{pypy_version}-"
+ build_classifiers += f"{build_type.lower()}"
+ self.build_classifiers = build_classifiers
venv_prefix, has_virtual_env = prefix()
@@ -229,6 +285,8 @@ class BuildInfoCollectorMixin(object):
# and we consider it is distinct enough that we don't have to
# append the build classifiers, thus keeping dir names shorter.
build_name = f"{venv_prefix}"
+ if self.is_cross_compile and has_virtual_env:
+ build_name += f"-{self.plat_name}"
# If short paths are requested and no virtual env is found, at
# least append the python version for more uniqueness.
@@ -259,11 +317,18 @@ class BuildInfoCollectorMixin(object):
self.install_dir = install_dir
self.py_executable = py_executable
self.py_include_dir = py_include_dir
- self.py_library = get_py_library(build_type, py_version, py_prefix,
- py_libdir, py_include_dir)
+
+ if not self.is_cross_compile:
+ self.py_library = get_py_library(build_type, py_version, py_prefix,
+ py_libdir, py_include_dir)
self.py_version = py_version
self.build_type = build_type
- self.site_packages_dir = sconfig.get_python_lib(1, 0, prefix=install_dir)
+
+ if self.is_cross_compile:
+ site_packages_without_prefix = self.python_target_info['python_info']['site_packages_dir']
+ self.site_packages_dir = os.path.join(install_dir, site_packages_without_prefix)
+ else:
+ self.site_packages_dir = sconfig.get_python_lib(1, 0, prefix=install_dir)
def post_collect_and_assign(self):
# self.build_lib is only available after the base class
diff --git a/build_scripts/config.py b/build_scripts/config.py
index ad3dd1542..2199269b4 100644
--- a/build_scripts/config.py
+++ b/build_scripts/config.py
@@ -82,6 +82,13 @@ class Config(object):
self.shiboken_generator_st_name = f"{SHIBOKEN}-generator"
self.pyside_st_name = PYSIDE_MODULE
+ # Path to CMake toolchain file when intending to cross compile
+ # the project.
+ self.cmake_toolchain_file = None
+
+ # Store where host shiboken is built during a cross-build.
+ self.shiboken_host_query_path = None
+
# Used by check_allowed_python_version to validate the
# interpreter version.
self.python_version_classifiers = [
@@ -96,9 +103,14 @@ class Config(object):
self.setup_script_dir = None
- def init_config(self, build_type=None, internal_build_type=None,
- cmd_class_dict=None, package_version=None,
- ext_modules=None, setup_script_dir=None,
+ def init_config(self,
+ build_type=None,
+ internal_build_type=None,
+ cmd_class_dict=None,
+ package_version=None,
+ ext_modules=None,
+ setup_script_dir=None,
+ cmake_toolchain_file=None,
quiet=False):
"""
Sets up the global singleton config which is used in many parts
@@ -122,6 +134,8 @@ class Config(object):
self.setup_script_dir = setup_script_dir
+ self.cmake_toolchain_file = cmake_toolchain_file
+
setup_kwargs = {}
setup_kwargs['long_description'] = self.get_long_description()
setup_kwargs['long_description_content_type'] = 'text/markdown'
@@ -367,6 +381,11 @@ class Config(object):
def is_top_level_build_pyside(self):
return self.build_type == self.pyside_option_name
+ def is_cross_compile(self):
+ if not self.cmake_toolchain_file:
+ return False
+ return True
+
def set_internal_build_type(self, internal_build_type):
self.internal_build_type = internal_build_type
diff --git a/build_scripts/main.py b/build_scripts/main.py
index a69b4fa69..2a1f46797 100644
--- a/build_scripts/main.py
+++ b/build_scripts/main.py
@@ -196,13 +196,18 @@ class PysideInstall(_install, DistUtilsCommandMixin):
user_options = _install.user_options + DistUtilsCommandMixin.mixin_user_options
def __init__(self, *args, **kwargs):
+ self.command_name = "install"
_install.__init__(self, *args, **kwargs)
DistUtilsCommandMixin.__init__(self)
def initialize_options(self):
_install.initialize_options(self)
- if sys.platform == 'darwin':
+ def finalize_options(self):
+ DistUtilsCommandMixin.mixin_finalize_options(self)
+ _install.finalize_options(self)
+
+ if sys.platform == 'darwin' or self.is_cross_compile:
# Because we change the plat_name to include a correct
# deployment target on macOS distutils thinks we are
# cross-compiling, and throws an exception when trying to
@@ -214,12 +219,11 @@ class PysideInstall(_install, DistUtilsCommandMixin):
# target. The fix is to disable the warn_dir flag, which
# was created for bdist_* derived classes to override, for
# similar cases.
+ # We also do it when cross-compiling. While calling install
+ # command directly is dubious, bdist_wheel calls install
+ # internally before creating a wheel.
self.warn_dir = False
- def finalize_options(self):
- DistUtilsCommandMixin.mixin_finalize_options(self)
- _install.finalize_options(self)
-
def run(self):
_install.run(self)
log.info(f"--- Install completed ({elapsed()}s)")
@@ -257,6 +261,7 @@ class PysideBuildExt(_build_ext):
class PysideBuildPy(_build_py):
def __init__(self, *args, **kwargs):
+ self.command_name = "build_py"
_build_py.__init__(self, *args, **kwargs)
@@ -289,6 +294,7 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
user_options = _build.user_options + DistUtilsCommandMixin.mixin_user_options
def __init__(self, *args, **kwargs):
+ self.command_name = "build"
_build.__init__(self, *args, **kwargs)
DistUtilsCommandMixin.__init__(self)
BuildInfoCollectorMixin.__init__(self)
@@ -298,8 +304,14 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
DistUtilsCommandMixin.mixin_finalize_options(self)
BuildInfoCollectorMixin.collect_and_assign(self)
- if sys.platform == 'darwin':
+ use_os_name_hack = False
+ if self.is_cross_compile:
+ use_os_name_hack = True
+ elif sys.platform == 'darwin':
self.plat_name = macos_plat_name()
+ use_os_name_hack = True
+
+ if use_os_name_hack:
# This is a hack to circumvent the dubious check in
# distutils.commands.build -> finalize_options, which only
# allows setting the plat_name for windows NT.
@@ -314,7 +326,7 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
# Must come after _build.finalize_options
BuildInfoCollectorMixin.post_collect_and_assign(self)
- if sys.platform == 'darwin':
+ if use_os_name_hack:
os.name = os_name_backup
def initialize_options(self):
@@ -333,6 +345,7 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
self.build_type = "Release"
self.qtinfo = None
self.build_tests = False
+ self.python_target_info = {}
def run(self):
prepare_build()
@@ -349,7 +362,13 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
# Don't add Qt to PATH env var, we don't want it to interfere
# with CMake's find_package calls which will use
# CMAKE_PREFIX_PATH.
- additional_paths = [self.py_scripts_dir]
+ # Don't add the Python scripts dir to PATH env when
+ # cross-compiling, it could be in the device sysroot (/usr)
+ # which can cause CMake device QtFooToolsConfig packages to be
+ # picked up instead of host QtFooToolsConfig packages.
+ additional_paths = []
+ if self.py_scripts_dir and not self.is_cross_compile:
+ additional_paths.append(self.py_scripts_dir)
# Add Clang to path for Windows.
# Revisit once Clang is bundled with Qt.
@@ -389,6 +408,13 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
log.info(f"Creating install folder {self.install_dir}...")
os.makedirs(self.install_dir)
+ # Write the CMake install path into a file. Is used by
+ # SetupRunner to provide a nicer UX when cross-compiling (no
+ # need to specify a host shiboken path explicitly)
+ if self.internal_cmake_install_dir_query_file_path:
+ with open(self.internal_cmake_install_dir_query_file_path, 'w') as f:
+ f.write(self.install_dir)
+
if (not OPTION["ONLYPACKAGE"]
and not config.is_internal_shiboken_generator_build_and_part_of_top_level_all()):
# Build extensions
@@ -464,7 +490,10 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
log.info(f"Python library: {self.py_library}")
log.info(f"Python prefix: {self.py_prefix}")
log.info(f"Python scripts: {self.py_scripts_dir}")
+ log.info(f"Python arch: {self.py_arch}")
+
log.info("-" * 3)
+ log.info(f"Qt prefix: {self.qtinfo.prefix_dir}")
log.info(f"Qt qmake: {self.qtinfo.qmake_command}")
log.info(f"Qt qtpaths: {self.qtinfo.qtpaths_command}")
log.info(f"Qt version: {self.qtinfo.version}")
@@ -545,9 +574,20 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
f"-DCMAKE_INSTALL_PREFIX={self.install_dir}",
module_src_dir
]
- cmake_cmd.append(f"-DPYTHON_EXECUTABLE={self.py_executable}")
- cmake_cmd.append(f"-DPYTHON_INCLUDE_DIR={self.py_include_dir}")
- cmake_cmd.append(f"-DPYTHON_LIBRARY={self.py_library}")
+
+ # When cross-compiling we set Python_ROOT_DIR to tell
+ # FindPython.cmake where to pick up the device python libs.
+ if self.is_cross_compile:
+ if self.python_target_path:
+ cmake_cmd.append(f"-DPython_ROOT_DIR={self.python_target_path}")
+
+ # Host python is needed when cross compiling to run
+ # embedding_generator.py. Pass it as a separate option.
+ cmake_cmd.append(f"-DQFP_PYTHON_HOST_PATH={sys.executable}")
+ else:
+ cmake_cmd.append(f"-DPYTHON_EXECUTABLE={self.py_executable}")
+ cmake_cmd.append(f"-DPYTHON_INCLUDE_DIR={self.py_include_dir}")
+ cmake_cmd.append(f"-DPYTHON_LIBRARY={self.py_library}")
# If a custom shiboken cmake config directory path was provided, pass it to CMake.
if OPTION["SHIBOKEN_CONFIG_DIR"] and config.is_internal_pyside_build():
@@ -597,10 +637,10 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
cmake_cmd.append("-DAVOID_PROTECTED_HACK=1")
numpy = get_numpy_location()
- if numpy:
+ if numpy and not self.is_cross_compile:
cmake_cmd.append(f"-DNUMPY_INCLUDE_DIR={numpy}")
- if self.build_type.lower() == 'debug':
+ if self.build_type.lower() == 'debug' and not self.is_cross_compile:
cmake_cmd.append(f"-DPYTHON_DEBUG_LIBRARY={self.py_library}")
if OPTION["LIMITED_API"] == "yes":
@@ -713,7 +753,27 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
cmake_cmd.append("-DPYSIDE_NUMPY_SUPPORT=1")
target_qt_prefix_path = self.qtinfo.prefix_dir
- cmake_cmd.append(f"-DCMAKE_PREFIX_PATH={target_qt_prefix_path}")
+ cmake_cmd.append(f"-DQFP_QT_TARGET_PATH={target_qt_prefix_path}")
+ if self.qt_host_path:
+ cmake_cmd.append(f"-DQFP_QT_HOST_PATH={self.qt_host_path}")
+
+ if self.is_cross_compile and (not OPTION["SHIBOKEN_HOST_PATH"]
+ or not os.path.exists(OPTION["SHIBOKEN_HOST_PATH"])):
+ raise DistutilsSetupError(
+ f"Please specify the location of host shiboken tools via --shiboken-host-path=")
+
+ if self.shiboken_host_path:
+ cmake_cmd.append(f"-DQFP_SHIBOKEN_HOST_PATH={self.shiboken_host_path}")
+
+ if self.shiboken_target_path:
+ cmake_cmd.append(f"-DQFP_SHIBOKEN_TARGET_PATH={self.shiboken_target_path}")
+ elif self.cmake_toolchain_file and not extension.lower() == SHIBOKEN:
+ # Need to tell where to find target shiboken when
+ # cross-compiling pyside.
+ cmake_cmd.append(f"-DQFP_SHIBOKEN_TARGET_PATH={self.install_dir}")
+
+ if self.cmake_toolchain_file:
+ cmake_cmd.append(f"-DCMAKE_TOOLCHAIN_FILE={self.cmake_toolchain_file}")
if not OPTION["SKIP_CMAKE"]:
log.info(f"Configuring module {extension} ({module_src_dir})...")
@@ -806,6 +866,11 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
"qt_prefix_dir": self.qtinfo.prefix_dir,
"qt_translations_dir": self.qtinfo.translations_dir,
"qt_qml_dir": self.qtinfo.qml_dir,
+
+ # TODO: This is currently None when cross-compiling
+ # There doesn't seem to be any place where we can query
+ # it. Fortunately it's currently only used when
+ # packaging Windows vcredist.
"target_arch": self.py_arch,
}
@@ -936,13 +1001,42 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
raise RuntimeError("Error copying libclang library "
f"from {clang_lib_path} to {destination_dir}. ")
+ def get_shared_library_filters(self):
+ unix_filters = ["*.so", "*.so.*"]
+ darwin_filters = ["*.so", "*.dylib"]
+ filters = []
+ if self.is_cross_compile:
+ if 'darwin' in self.plat_name or 'macos' in self.plat_name:
+ filters = darwin_filters
+ elif 'linux' in self.plat_name:
+ filters = unix_filters
+ else:
+ log.warn(f"No shared library filters found for platform {self.plat_name}. "
+ f"The package might miss Qt libraries and plugins.")
+ else:
+ if sys.platform == 'darwin':
+ filters = darwin_filters
+ else:
+ filters = unix_filters
+ return filters
+
def package_libraries(self, package_path):
"""Returns the libraries of the Python module"""
- UNIX_FILTERS = ["*.so", "*.so.*"]
- DARWIN_FILTERS = ["*.so", "*.dylib"]
- FILTERS = DARWIN_FILTERS if sys.platform == 'darwin' else UNIX_FILTERS
+ filters = self.get_shared_library_filters()
return [lib for lib in os.listdir(
- package_path) if filter_match(lib, FILTERS)]
+ package_path) if filter_match(lib, filters)]
+
+ def get_shared_libraries_in_path_recursively(self, initial_path):
+ """Returns shared library plugins in given path (collected
+ recursively)"""
+ filters = self.get_shared_library_filters()
+ libraries = []
+ for dir_path, dir_names, file_names in os.walk(initial_path):
+ for name in file_names:
+ if filter_match(name, filters):
+ library_path = os.path.join(dir_path, name)
+ libraries.append(library_path)
+ return libraries
def update_rpath(self, package_path, executables, libexec=False):
ROOT = '@loader_path' if sys.platform == 'darwin' else '$ORIGIN'
@@ -993,13 +1087,70 @@ class PysideBuild(_build, DistUtilsCommandMixin, BuildInfoCollectorMixin):
log.info("Patched rpath to '$ORIGIN/' (Linux) or "
f"updated rpath (OS/X) in {srcpath}.")
+ def update_rpath_for_linux_plugins(
+ self,
+ plugin_paths,
+ qt_lib_dir=None,
+ is_qml_plugin=False):
+ # If the linux sysroot (where the plugins are copied from)
+ # is from a mainline distribution, it might have a different
+ # directory layout than then one we expect to have in the
+ # wheel.
+ # We have to ensure that any plugins copied have rpath
+ # values that can find Qt libs in the newly assembled wheel
+ # dir layout.
+ if not (self.is_cross_compile and sys.platform.startswith('linux') and self.standalone):
+ return
+
+ log.info(f"Patching rpath for Qt and QML plugins.")
+ for plugin in plugin_paths:
+ if os.path.isdir(plugin) or os.path.islink(plugin):
+ continue
+ if not os.path.exists(plugin):
+ continue
+
+ if is_qml_plugin:
+ plugin_dir = os.path.dirname(plugin)
+ rel_path_from_qml_plugin_qt_lib_dir = os.path.relpath(qt_lib_dir, plugin_dir)
+ rpath_value = os.path.join("$ORIGIN", rel_path_from_qml_plugin_qt_lib_dir)
+ else:
+ rpath_value = "$ORIGIN/../../lib"
+
+ linux_fix_rpaths_for_library(self._patchelf_path, plugin, rpath_value,
+ override=True)
+ log.info(f"Patched rpath to '{rpath_value}' in {plugin}.")
+
+ def update_rpath_for_linux_qt_libraries(self, qt_lib_dir):
+ # Ensure that Qt libs and ICU libs have $ORIGIN in their rpath.
+ # Especially important for ICU lib, so that they don't
+ # accidentally load dependencies from the system.
+ if not (self.is_cross_compile and sys.platform.startswith('linux') and self.standalone):
+ return
+
+ rpath_value = "$ORIGIN"
+ log.info(f"Patching rpath for Qt and ICU libraries in {qt_lib_dir}.")
+ libs = self.package_libraries(qt_lib_dir)
+ lib_paths = [os.path.join(qt_lib_dir, lib) for lib in libs]
+ for library in lib_paths:
+ if os.path.isdir(library) or os.path.islink(library):
+ continue
+ if not os.path.exists(library):
+ continue
+
+ linux_fix_rpaths_for_library(self._patchelf_path, library, rpath_value, override=True)
+ log.info(f"Patched rpath to '{rpath_value}' in {library}.")
+
class PysideRstDocs(Command, DistUtilsCommandMixin):
description = "Build .rst documentation only"
user_options = DistUtilsCommandMixin.mixin_user_options
- def initialize_options(self):
+ def __init__(self, *args, **kwargs):
+ self.command_name = "build_rst_docs"
+ Command.__init__(self, *args, **kwargs)
DistUtilsCommandMixin.__init__(self)
+
+ def initialize_options(self):
log.info("-- This build process will not include the API documentation."
"API documentation requires a full build of pyside/shiboken.")
self.skip = False
diff --git a/build_scripts/options.py b/build_scripts/options.py
index 045eb05d0..6ce53b982 100644
--- a/build_scripts/options.py
+++ b/build_scripts/options.py
@@ -39,11 +39,13 @@
try:
from setuptools._distutils import log
+ from setuptools import Command
except ModuleNotFoundError:
# This is motivated by our CI using an old version of setuptools
# so then the coin_build_instructions.py script is executed, and
# import from this file, it was failing.
from distutils import log
+ from distutils.cmd import Command
from shutil import which
import sys
import os
@@ -51,6 +53,7 @@ import warnings
from pathlib import Path
from .qtinfo import QtInfo
+from .utils import memoize
_AVAILABLE_MKSPECS = ["ninja", "msvc", "mingw"] if sys.platform == "win32" else ["ninja", "make"]
@@ -63,6 +66,9 @@ Additional options:
---macos-use-libc++ Use libc++ on macOS
--snapshot-build Snapshot build
--package-timestamp Package Timestamp
+ --cmake-toolchain-file Path to CMake toolchain to enable cross-compiling
+ --shiboken-host-path Path to host shiboken package when cross-compiling
+ --qt-host-path Path to host Qt installation when cross-compiling
"""
@@ -164,7 +170,7 @@ def _jobs_option_value():
# Declare options which need to be known when instantiating the DistUtils
-# commands.
+# commands or even earlier during SetupRunner.run().
OPTION = {
"BUILD_TYPE": option_value("build-type"),
"INTERNAL_BUILD_TYPE": option_value("internal-build-type"),
@@ -179,7 +185,11 @@ OPTION = {
"PACKAGE_TIMESTAMP": option_value("package-timestamp"),
# This is used automatically by distutils.command.install object, to
# specify the final installation location.
- "FINAL_INSTALL_PREFIX": option_value("prefix", remove=False)
+ "FINAL_INSTALL_PREFIX": option_value("prefix", remove=False),
+ "CMAKE_TOOLCHAIN_FILE": option_value("cmake-toolchain-file"),
+ "SHIBOKEN_HOST_PATH": option_value("shiboken-host-path"),
+ "SHIBOKEN_HOST_PATH_QUERY_FILE": option_value("internal-shiboken-host-path-query-file"),
+ "QT_HOST_PATH": option_value("qt-host-path")
# This is used to identify the template for doc builds
}
_deprecated_option_jobs = option_value('jobs')
@@ -191,7 +201,7 @@ if _deprecated_option_jobs:
class DistUtilsCommandMixin(object):
"""Mixin for the DistUtils build/install commands handling the options."""
- _finalized = False
+ _static_class_finalized_once = False
mixin_user_options = [
('avoid-protected-hack', None, 'Force --avoid-protected-hack'),
@@ -217,9 +227,16 @@ class DistUtilsCommandMixin(object):
('qtpaths=', None, 'Path to qtpaths'),
('qmake=', None, 'Path to qmake (deprecated, use qtpaths)'),
('qt=', None, 'Qt version'),
+ ('qt-target-path=', None,
+ 'Path to device Qt installation (use Qt libs when cross-compiling)'),
('cmake=', None, 'Path to CMake'),
('openssl=', None, 'Path to OpenSSL libraries'),
+
+ # FIXME: Deprecated in favor of shiboken-target-path
('shiboken-config-dir=', None, 'shiboken configuration directory'),
+
+ ('shiboken-target-path=', None, 'Path to target shiboken package'),
+ ('python-target-path=', None, 'Path to target Python installation / prefix'),
('make-spec=', None, 'Qt make-spec'),
('macos-arch=', None, 'macOS architecture'),
('macos-sysroot=', None, 'macOS sysroot'),
@@ -230,7 +247,13 @@ class DistUtilsCommandMixin(object):
('qt-conf-prefix=', None, 'Qt configuration prefix'),
('qt-src-dir=', None, 'Qt source directory'),
('no-qt-tools', None, 'Do not copy the Qt tools'),
- ('pyside-numpy-support', None, 'libpyside: Add (experimental) numpy support')
+ ('pyside-numpy-support', None, 'libpyside: Add (experimental) numpy support'),
+ ('internal-cmake-install-dir-query-file-path=', None,
+ 'Path to file where the CMake install path of the project will be saved'),
+
+ # We redeclare plat-name as an option so it's recognized by the
+ # install command and doesn't throw an error.
+ ('plat-name=', None, 'The platform name for which we are cross-compiling'),
]
def __init__(self):
@@ -259,9 +282,17 @@ class DistUtilsCommandMixin(object):
self.qmake = None
self.has_qmake_option = False
self.qt = '5'
+ self.qt_host_path = None
+ self.qt_target_path = None
self.cmake = None
self.openssl = None
self.shiboken_config_dir = None
+ self.shiboken_host_path = None
+ self.shiboken_host_path_query_file = None
+ self.shiboken_target_path = None
+ self.python_target_path = None
+ self.is_cross_compile = False
+ self.cmake_toolchain_file = None
self.make_spec = None
self.macos_arch = None
self.macos_sysroot = None
@@ -273,16 +304,62 @@ class DistUtilsCommandMixin(object):
self.qt_src_dir = None
self.no_qt_tools = False
self.pyside_numpy_support = False
+ self.plat_name = None
+ self.internal_cmake_install_dir_query_file_path = None
+ self._per_command_mixin_options_finalized = False
+
+ # When initializing a command other than the main one (so the
+ # first one), we need to copy the user options from the main
+ # command to the new command options dict. Then
+ # Distribution.get_command_obj will pick up the copied options
+ # ensuring that all commands that inherit from
+ # the mixin, get our custom properties set by the time
+ # finalize_options is called.
+ if DistUtilsCommandMixin._static_class_finalized_once:
+ current_command: Command = self
+ dist = current_command.distribution
+ main_command_name = dist.commands[0]
+ main_command_opts = dist.get_option_dict(main_command_name)
+ current_command_name = current_command.get_command_name()
+ current_command_opts = dist.get_option_dict(current_command_name)
+ mixin_options_set = self.get_mixin_options_set()
+ for key, value in main_command_opts.items():
+ if key not in current_command_opts and key in mixin_options_set:
+ current_command_opts[key] = value
+
+ @staticmethod
+ @memoize
+ def get_mixin_options_set():
+ keys = set()
+ for (name, _, _) in DistUtilsCommandMixin.mixin_user_options:
+ keys.add(name.rstrip("=").replace("-", "_"))
+ return keys
+
def mixin_finalize_options(self):
- # Bail out on 2nd call to mixin_finalize_options() since that is the
- # build command following the install command when invoking
- # setup.py install
- if not DistUtilsCommandMixin._finalized:
- DistUtilsCommandMixin._finalized = True
+ # The very first we finalize options, record that.
+ if not DistUtilsCommandMixin._static_class_finalized_once:
+ DistUtilsCommandMixin._static_class_finalized_once = True
+
+ # Ensure we finalize once per command object, rather than per
+ # setup.py invocation. We want to have the option values
+ # available in all commands that derive from the mixin.
+ if not self._per_command_mixin_options_finalized:
+ self._per_command_mixin_options_finalized = True
self._do_finalize()
def _do_finalize(self):
+ # is_cross_compile must be set before checking for qtpaths/qmake
+ # because we DON'T want those to be found when cross compiling.
+ # Currently when cross compiling, qt-target-path MUST be used.
+ using_cmake_toolchain_file = False
+ cmake_toolchain_file = None
+ if OPTION["CMAKE_TOOLCHAIN_FILE"]:
+ self.is_cross_compile = True
+ using_cmake_toolchain_file = True
+ cmake_toolchain_file = OPTION["CMAKE_TOOLCHAIN_FILE"]
+ self.cmake_toolchain_file = cmake_toolchain_file
+
if not self._determine_defaults_and_check():
sys.exit(-1)
OPTION['AVOID_PROTECTED_HACK'] = self.avoid_protected_hack
@@ -320,12 +397,62 @@ class DistUtilsCommandMixin(object):
OPTION['QMAKE'] = qmake_abs_path
OPTION['HAS_QMAKE_OPTION'] = self.has_qmake_option
OPTION['QT_VERSION'] = self.qt
+ self.qt_host_path = OPTION['QT_HOST_PATH']
+ OPTION['QT_TARGET_PATH'] = self.qt_target_path
+
+ qt_target_path = None
+ if self.qt_target_path:
+ qt_target_path = self.qt_target_path
+
+ # We use the CMake project to find host Qt if neither qmake or
+ # qtpaths is available. This happens when building the host
+ # tools in the overall cross-building process.
+ use_cmake = False
+ if using_cmake_toolchain_file or \
+ (not self.qmake and not self.qtpaths and self.qt_target_path):
+ use_cmake = True
+
QtInfo().setup(qtpaths_abs_path, self.cmake, qmake_abs_path,
- self.has_qmake_option)
+ self.has_qmake_option,
+ use_cmake=use_cmake,
+ qt_target_path=qt_target_path,
+ cmake_toolchain_file=cmake_toolchain_file)
+
+ try:
+ QtInfo().prefix_dir
+ except Exception as e:
+ if not self.qt_target_path:
+ log.error(
+ "\nCould not find Qt. You can pass the --qt-target-path=<qt-dir> option as a "
+ "hint where to find Qt. Error was:\n\n\n")
+ else:
+ log.error(
+ f"\nCould not find Qt via provided option --qt-target-path={qt_target_path} "
+ "Error was:\n\n\n")
+ raise e
OPTION['CMAKE'] = os.path.abspath(self.cmake)
OPTION['OPENSSL'] = self.openssl
OPTION['SHIBOKEN_CONFIG_DIR'] = self.shiboken_config_dir
+ if self.shiboken_config_dir:
+ _warn_deprecated_option('shiboken-config-dir', 'shiboken-target-path')
+
+ self.shiboken_host_path = OPTION['SHIBOKEN_HOST_PATH']
+ self.shiboken_host_path_query_file = OPTION['SHIBOKEN_HOST_PATH_QUERY_FILE']
+
+ if not self.shiboken_host_path and self.shiboken_host_path_query_file:
+ try:
+ queried_shiboken_host_path = Path(self.shiboken_host_path_query_file).read_text()
+ self.shiboken_host_path = queried_shiboken_host_path
+ OPTION['SHIBOKEN_HOST_PATH'] = queried_shiboken_host_path
+ except Exception as e:
+ log.error(
+ f"\n Could not find shiboken host tools via the query file: "
+ f"{self.shiboken_host_path_query_file:} Error was:\n\n\n")
+ raise e
+
+ OPTION['SHIBOKEN_TARGET_PATH'] = self.shiboken_target_path
+ OPTION['PYTHON_TARGET_PATH'] = self.python_target_path
OPTION['MAKESPEC'] = self.make_spec
OPTION['MACOS_ARCH'] = self.macos_arch
OPTION['MACOS_SYSROOT'] = self.macos_sysroot
@@ -338,6 +465,15 @@ class DistUtilsCommandMixin(object):
OPTION['NO_QT_TOOLS'] = self.no_qt_tools
OPTION['PYSIDE_NUMPY_SUPPORT'] = self.pyside_numpy_support
+ if not self._extra_checks():
+ sys.exit(-1)
+
+ def _extra_checks(self):
+ if self.is_cross_compile and not self.plat_name:
+ log.error(f"No value provided to --plat-name while cross-compiling.")
+ return False
+ return True
+
def _find_qtpaths_in_path(self):
if not self.qtpaths:
self.qtpaths = which("qtpaths")
@@ -354,30 +490,43 @@ class DistUtilsCommandMixin(object):
log.error(f"'{self.cmake}' does not exist.")
return False
- # Enforce usage of qmake in QtInfo if it was given explicitly.
- if self.qmake:
- self.has_qmake_option = True
- _warn_deprecated_option('qmake', 'qtpaths')
-
- # If no option was given explicitly, prefer to find qtpaths
- # in PATH.
- if not self.qmake and not self.qtpaths:
- self._find_qtpaths_in_path()
-
- # If no tool was specified and qtpaths was not found in PATH,
- # ask to provide a path to qtpaths.
- if not self.qtpaths and not self.qmake:
- log.error("No value provided to --qtpaths option. Please provide one to find Qt.")
- return False
-
- # Validate that the given tool path exists.
- if self.qtpaths and not os.path.exists(self.qtpaths):
- log.error(f"The specified qtpaths path '{self.qtpaths}' does not exist.")
- return False
-
- if self.qmake and not os.path.exists(self.qmake):
- log.error(f"The specified qmake path '{self.qmake}' does not exist.")
- return False
+ # When cross-compiling, we only accept the qt-target-path
+ # option and don't rely on auto-searching in PATH or the other
+ # qtpaths / qmake options.
+ # We also don't do auto-searching if qt-target-path is passed
+ # explicitly. This is to help with the building of host tools
+ # while cross-compiling.
+ if not self.is_cross_compile and not self.qt_target_path:
+ # Enforce usage of qmake in QtInfo if it was given explicitly.
+ if self.qmake:
+ self.has_qmake_option = True
+ _warn_deprecated_option('qmake', 'qtpaths')
+
+ # If no option was given explicitly, prefer to find qtpaths
+ # in PATH.
+ if not self.qmake and not self.qtpaths:
+ self._find_qtpaths_in_path()
+
+ # If no tool was specified and qtpaths was not found in PATH,
+ # ask to provide a path to qtpaths.
+ if not self.qtpaths and not self.qmake and not self.qt_target_path:
+ log.error("No value provided to --qtpaths option. Please provide one to find Qt.")
+ return False
+
+ # Validate that the given tool path exists.
+ if self.qtpaths and not os.path.exists(self.qtpaths):
+ log.error(f"The specified qtpaths path '{self.qtpaths}' does not exist.")
+ return False
+
+ if self.qmake and not os.path.exists(self.qmake):
+ log.error(f"The specified qmake path '{self.qmake}' does not exist.")
+ return False
+ else:
+ # Check for existence, but don't require if it's not set. A
+ # check later will be done to see if it's needed.
+ if self.qt_target_path and not os.path.exists(self.qt_target_path):
+ log.error(f"Provided --qt-target-path='{self.qt_target_path}' path does not exist.")
+ return False
if not self.make_spec:
self.make_spec = _AVAILABLE_MKSPECS[0]
diff --git a/build_scripts/platforms/linux.py b/build_scripts/platforms/linux.py
index 324b962db..31e2613ef 100644
--- a/build_scripts/platforms/linux.py
+++ b/build_scripts/platforms/linux.py
@@ -89,6 +89,9 @@ def prepare_standalone_package_linux(self, vars):
if not maybe_icu_libs:
copy_icu_libs(self._patchelf_path, resolved_destination_lib_dir)
+ # Set RPATH for Qt libs.
+ self.update_rpath_for_linux_qt_libraries(destination_lib_dir.format(**vars))
+
# Patching designer to use the Qt libraries provided in the wheel
if config.is_internal_pyside_build():
assistant_path = "{st_build_dir}/{st_package_name}/assistant".format(**vars)
@@ -116,15 +119,26 @@ def prepare_standalone_package_linux(self, vars):
recursive=False,
vars=vars)
+ copied_plugins = self.get_shared_libraries_in_path_recursively(
+ plugins_target.format(**vars))
+ self.update_rpath_for_linux_plugins(copied_plugins)
+
if copy_qml:
# <qt>/qml/* -> <setup>/{st_package_name}/Qt/qml
+ qml_plugins_target = "{st_build_dir}/{st_package_name}/Qt/qml"
copydir("{qt_qml_dir}",
- "{st_build_dir}/{st_package_name}/Qt/qml",
+ qml_plugins_target,
filter=None,
force=False,
recursive=True,
ignore=["*.so.debug"],
vars=vars)
+ copied_plugins = self.get_shared_libraries_in_path_recursively(
+ qml_plugins_target.format(**vars))
+ self.update_rpath_for_linux_plugins(
+ copied_plugins,
+ qt_lib_dir=destination_lib_dir.format(**vars),
+ is_qml_plugin=True)
if copy_translations:
# <qt>/translations/* ->
diff --git a/build_scripts/qtinfo.py b/build_scripts/qtinfo.py
index f038adddc..95e087096 100644
--- a/build_scripts/qtinfo.py
+++ b/build_scripts/qtinfo.py
@@ -43,27 +43,10 @@ import re
import subprocess
import tempfile
from pathlib import Path
-
+from .utils import configure_cmake_project, parse_cmake_project_message_info
from .utils import platform_cmake_options
-_CMAKE_LISTS = """cmake_minimum_required(VERSION 3.16)
-project(dummy LANGUAGES CXX)
-
-find_package(Qt6 COMPONENTS Core)
-
-get_target_property(darwin_target Qt6::Core QT_DARWIN_MIN_DEPLOYMENT_TARGET)
-message(STATUS "mkspec_qt_darwin_min_deployment_target=${darwin_target}")
-
-if(QT_FEATURE_debug_and_release)
- message(STATUS "mkspec_build_type=debug_and_release")
-elseif(QT_FEATURE_debug)
- message(STATUS "mkspec_build_type=debug")
-else()
- message(STATUS "mkspec_build_type=release")
-endif()
-"""
-
class QtInfo(object):
_instance = None # singleton helpers
@@ -85,14 +68,21 @@ class QtInfo(object):
self._cmake_command = None
self._qmake_command = None
self._force_qmake = False
+ self._use_cmake = False
+ self._qt_target_path = None
+ self._cmake_toolchain_file = None
# Dict to cache qmake values.
self._query_dict = {}
- def setup(self, qtpaths, cmake, qmake, force_qmake):
+ def setup(self, qtpaths, cmake, qmake, force_qmake, use_cmake, qt_target_path,
+ cmake_toolchain_file):
self._qtpaths_command = qtpaths
self._cmake_command = cmake
self._qmake_command = qmake
self._force_qmake = force_qmake
+ self._use_cmake = use_cmake
+ self._qt_target_path = qt_target_path
+ self._cmake_toolchain_file = cmake_toolchain_file
@property
def qmake_command(self):
@@ -213,68 +203,75 @@ class QtInfo(object):
return props
def _get_query_properties(self):
- if self._force_qmake:
- output = self._get_qmake_output(["-query"])
+ if self._use_cmake:
+ setup_script_dir = Path.cwd()
+ sources_dir = setup_script_dir / "sources"
+ qt_target_info_dir = sources_dir / "shiboken6" / "config.tests" / "target_qt_info"
+ qt_target_info_dir = os.fspath(qt_target_info_dir)
+ config_tests_dir = setup_script_dir / "build" / "config.tests"
+ config_tests_dir = os.fspath(config_tests_dir)
+
+ cmake_cache_args = []
+ if self._cmake_toolchain_file:
+ cmake_cache_args.append(("CMAKE_TOOLCHAIN_FILE", self._cmake_toolchain_file))
+
+ if self._qt_target_path:
+ cmake_cache_args.append(("QFP_QT_TARGET_PATH", self._qt_target_path))
+ qt_target_info_output = configure_cmake_project(
+ qt_target_info_dir,
+ self._cmake_command,
+ temp_prefix_build_path=config_tests_dir,
+ cmake_cache_args=cmake_cache_args)
+ qt_target_info = parse_cmake_project_message_info(qt_target_info_output)
+ self._query_dict = qt_target_info['qt_info']
else:
- output = self._get_qtpaths_output(["--qt-query"])
- self._query_dict = self._parse_query_properties(output)
+ if self._force_qmake:
+ output = self._get_qmake_output(["-query"])
+ else:
+ output = self._get_qtpaths_output(["--qt-query"])
+ self._query_dict = self._parse_query_properties(output)
def _get_other_properties(self):
# Get the src property separately, because it is not returned by
# qmake unless explicitly specified.
key = "QT_INSTALL_PREFIX/src"
- if self._force_qmake:
- result = self._get_qmake_output(["-query", key])
- else:
- result = self._get_qtpaths_output(["--qt-query", key])
- self._query_dict[key] = result
+ if not self._use_cmake:
+ if self._force_qmake:
+ result = self._get_qmake_output(["-query", key])
+ else:
+ result = self._get_qtpaths_output(["--qt-query", key])
+ self._query_dict[key] = result
# Get mkspecs variables and cache them.
# FIXME Python 3.9 self._query_dict |= other_dict
for key, value in self._get_cmake_mkspecs_variables().items():
self._query_dict[key] = value
- @staticmethod
- def _parse_cmake_mkspecs_variables(output):
- # Helper for _get_cmake_mkspecs_variables(). Parse the output for
- # anything prefixed '-- mkspec_' as created by the message() calls
- # in _CMAKE_LISTS.
- result = {}
- pattern = re.compile(r"^-- mkspec_(.*)=(.*)$")
- for line in output.splitlines():
- found = pattern.search(line.strip())
- if found:
- key = found.group(1).strip()
- value = found.group(2).strip()
- # Get macOS minimum deployment target.
- if key == 'qt_darwin_min_deployment_target':
- result['QMAKE_MACOSX_DEPLOYMENT_TARGET'] = value
- # Figure out how Qt was built
- elif key == 'build_type':
- result['BUILD_TYPE'] = value
- return result
-
def _get_cmake_mkspecs_variables(self):
- # Create an empty cmake project file in a temporary directory and
- # parse the output to determine some mkspec values.
- output = ''
- error = ''
- return_code = 0
- with tempfile.TemporaryDirectory() as tempdir:
- cmake_list_file = Path(tempdir) / 'CMakeLists.txt'
- cmake_list_file.write_text(_CMAKE_LISTS)
- cmd = [self._cmake_command, '-G', 'Ninja', '.']
- qt_prefix = self.prefix_dir
- cmd.extend([f'-DCMAKE_PREFIX_PATH={qt_prefix}'])
- cmd += platform_cmake_options()
-
- # FIXME Python 3.7: Use subprocess.run()
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=False,
- cwd=tempdir, universal_newlines=True)
- output, error = proc.communicate()
- proc.wait()
- return_code = proc.returncode
-
- if return_code != 0:
- raise RuntimeError(f"Could not determine cmake variables: {error}")
- return QtInfo.__QtInfo._parse_cmake_mkspecs_variables(output)
+ setup_script_dir = Path.cwd()
+ sources_dir = setup_script_dir / "sources"
+ qt_target_mkspec_dir = sources_dir / "shiboken6" / "config.tests" / "target_qt_mkspec"
+ qt_target_mkspec_dir = qt_target_mkspec_dir.as_posix()
+ config_tests_dir = setup_script_dir / "build" / "config.tests"
+ config_tests_dir = config_tests_dir.as_posix()
+
+ cmake_cache_args = []
+ if self._cmake_toolchain_file:
+ cmake_cache_args.append(("CMAKE_TOOLCHAIN_FILE", self._cmake_toolchain_file))
+ if self._qt_target_path:
+ cmake_cache_args.append(("QFP_QT_TARGET_PATH", self._qt_target_path))
+ else:
+ qt_prefix = Path(self.prefix_dir).as_posix()
+ cmake_cache_args.append(("CMAKE_PREFIX_PATH", qt_prefix))
+
+ cmake_cache_args.extend(platform_cmake_options(as_tuple_list=True))
+ qt_target_mkspec_output = configure_cmake_project(
+ qt_target_mkspec_dir,
+ self._cmake_command,
+ temp_prefix_build_path=config_tests_dir,
+ cmake_cache_args=cmake_cache_args)
+
+ qt_target_mkspec_info = parse_cmake_project_message_info(qt_target_mkspec_output)
+ qt_target_mkspec_info = qt_target_mkspec_info['qt_info']
+
+ return qt_target_mkspec_info
diff --git a/build_scripts/setup_runner.py b/build_scripts/setup_runner.py
index f77b9d1fe..362432d2a 100644
--- a/build_scripts/setup_runner.py
+++ b/build_scripts/setup_runner.py
@@ -39,6 +39,7 @@
import sys
import os
+import tempfile
import textwrap
from setuptools import setup # Import setuptools before distutils
@@ -68,6 +69,16 @@ class SetupRunner(object):
return any(arg for arg in list(args) if "--" + argument in arg)
@staticmethod
+ def get_cmd_line_argument_in_args(argument, args):
+ """ Gets the value of a cmd line argument passed in args. """
+ for arg in list(args):
+ if "--" + argument in arg:
+ prefix = f"--{argument}"
+ prefix_len = len(prefix) + 1
+ return arg[prefix_len:]
+ return None
+
+ @staticmethod
def remove_cmd_line_argument_in_args(argument, args):
""" Remove command line argument from args. """
return [arg for arg in list(args) if "--" + argument not in arg]
@@ -83,20 +94,107 @@ class SetupRunner(object):
def construct_internal_build_type_cmd_line_argument(internal_build_type):
return SetupRunner.construct_cmd_line_argument("internal-build-type", internal_build_type)
- def add_setup_internal_invocation(self, build_type, reuse_build=False):
- """ Enqueues a script sub-invocation to be executed later. """
+ def enqueue_setup_internal_invocation(self, setup_cmd):
+ self.invocations_list.append(setup_cmd)
+
+ def add_setup_internal_invocation(self, build_type, reuse_build=False, extra_args=None):
+ setup_cmd = self.new_setup_internal_invocation(build_type, reuse_build, extra_args)
+ self.enqueue_setup_internal_invocation(setup_cmd)
+
+ def new_setup_internal_invocation(self, build_type,
+ reuse_build=False,
+ extra_args=None,
+ replace_command_with=None):
+ """ Creates a script sub-invocation to be executed later. """
internal_build_type_arg = self.construct_internal_build_type_cmd_line_argument(build_type)
- setup_cmd = [sys.executable] + self.sub_argv + [internal_build_type_arg]
- command = self.sub_argv[0]
+ command_index = 0
+ command = self.sub_argv[command_index]
if command == 'setup.py' and len(self.sub_argv) > 1:
- command = self.sub_argv[1]
+ command_index = 1
+ command = self.sub_argv[command_index]
+
+ # Make a copy
+ modified_argv = list(self.sub_argv)
+
+ if replace_command_with:
+ modified_argv[command_index] = replace_command_with
+
+ setup_cmd = [sys.executable] + modified_argv + [internal_build_type_arg]
+
+ if extra_args:
+ for (name, value) in extra_args:
+ setup_cmd.append(self.construct_cmd_line_argument(name, value))
# Add --reuse-build option if requested and not already present.
if (reuse_build and command in ('bdist_wheel', 'build', 'build_rst_docs', 'install')
- and not self.cmd_line_argument_is_in_args("reuse-build", self.sub_argv)):
+ and not self.cmd_line_argument_is_in_args("reuse-build", modified_argv)):
setup_cmd.append(self.construct_cmd_line_argument("reuse-build"))
- self.invocations_list.append(setup_cmd)
+ return setup_cmd
+
+ def add_host_tools_setup_internal_invocation(self, initialized_config):
+ extra_args = []
+ extra_host_args = []
+
+ # When cross-compiling, build the host shiboken generator tool
+ # only if a path to an existing one was not provided.
+ if not self.cmd_line_argument_is_in_args("shiboken-host-path", self.sub_argv):
+ handle, initialized_config.shiboken_host_query_path = tempfile.mkstemp()
+ os.close(handle)
+
+ # Tell the setup process to create a file with the location
+ # of the installed host shiboken as its contents.
+ extra_host_args.append(
+ ("internal-cmake-install-dir-query-file-path",
+ initialized_config.shiboken_host_query_path))
+
+ # Tell the other setup invocations to read that file and use
+ # the read path as the location of the host shiboken.
+ extra_args.append(
+ ("internal-shiboken-host-path-query-file",
+ initialized_config.shiboken_host_query_path)
+ )
+
+ # This is specifying shiboken_module_option_name
+ # instead of shiboken_generator_option_name, but it will
+ # actually build the generator.
+ host_cmd = self.new_setup_internal_invocation(
+ initialized_config.shiboken_module_option_name,
+ extra_args=extra_host_args,
+ replace_command_with="build")
+
+ # To build the host tools, we reuse the initial target
+ # command line arguments, but we remove some options that
+ # don't make sense for the host build.
+
+ # Drop the toolchain arg.
+ host_cmd = self.remove_cmd_line_argument_in_args("cmake-toolchain-file",
+ host_cmd)
+
+ # Drop the target plat-name arg if there is one.
+ if self.cmd_line_argument_is_in_args("plat-name", host_cmd):
+ host_cmd = self.remove_cmd_line_argument_in_args("plat-name", host_cmd)
+
+ # Drop the python-target-path arg if there is one.
+ if self.cmd_line_argument_is_in_args("python-target-path", host_cmd):
+ host_cmd = self.remove_cmd_line_argument_in_args("python-target-path", host_cmd)
+
+ # Drop the target build-tests arg if there is one.
+ if self.cmd_line_argument_is_in_args("build-tests", host_cmd):
+ host_cmd = self.remove_cmd_line_argument_in_args("build-tests", host_cmd)
+
+ # Make sure to pass the qt host path as the target path
+ # when doing the host build. And make sure to remove any
+ # existing qt target path.
+ if self.cmd_line_argument_is_in_args("qt-host-path", host_cmd):
+ qt_host_path = self.get_cmd_line_argument_in_args("qt-host-path", host_cmd)
+ host_cmd = self.remove_cmd_line_argument_in_args("qt-host-path", host_cmd)
+ host_cmd = self.remove_cmd_line_argument_in_args("qt-target-path", host_cmd)
+ host_cmd.append(self.construct_cmd_line_argument("qt-target-path",
+ qt_host_path))
+
+ self.enqueue_setup_internal_invocation(host_cmd)
+ return extra_args
def run_setup(self):
"""
@@ -118,6 +216,7 @@ class SetupRunner(object):
package_version=get_package_version(),
ext_modules=get_setuptools_extension_modules(),
setup_script_dir=self.setup_script_dir,
+ cmake_toolchain_file=OPTION["CMAKE_TOOLCHAIN_FILE"],
quiet=OPTION["QUIET"])
# Enable logging for both the top-level invocation of setup.py
@@ -149,18 +248,33 @@ class SetupRunner(object):
# Build everything: shiboken6, shiboken6-generator and PySide6.
help_requested = '--help' in self.sub_argv or '-h' in self.sub_argv
+
if help_requested:
self.add_setup_internal_invocation(config.pyside_option_name)
elif config.is_top_level_build_all():
- self.add_setup_internal_invocation(config.shiboken_module_option_name)
+ extra_args = []
+
+ # extra_args might contain the location of the built host
+ # shiboken, which needs to be passed to the other
+ # target invocations.
+ if config.is_cross_compile():
+ extra_args = self.add_host_tools_setup_internal_invocation(config)
+
+ self.add_setup_internal_invocation(
+ config.shiboken_module_option_name,
+ extra_args=extra_args)
# Reuse the shiboken build for the generator package instead
# of rebuilding it again.
- self.add_setup_internal_invocation(config.shiboken_generator_option_name,
- reuse_build=True)
+ # Don't build it in a cross-build though.
+ if not config.is_cross_compile():
+ self.add_setup_internal_invocation(
+ config.shiboken_generator_option_name,
+ reuse_build=True)
- self.add_setup_internal_invocation(config.pyside_option_name)
+ self.add_setup_internal_invocation(config.pyside_option_name,
+ extra_args=extra_args)
elif config.is_top_level_build_shiboken_module():
self.add_setup_internal_invocation(config.shiboken_module_option_name)
@@ -184,6 +298,9 @@ class SetupRunner(object):
if help_requested:
print(ADDITIONAL_OPTIONS)
+ # Cleanup temp query file.
+ if config.shiboken_host_query_path:
+ os.remove(config.shiboken_host_query_path)
@staticmethod
def run_setuptools_setup():
diff --git a/build_scripts/utils.py b/build_scripts/utils.py
index 39cff6743..2daad642e 100644
--- a/build_scripts/utils.py
+++ b/build_scripts/utils.py
@@ -48,6 +48,8 @@ import subprocess
import fnmatch
import itertools
import glob
+import tempfile
+from collections import defaultdict
import urllib.request as urllib
@@ -244,12 +246,16 @@ def init_msvc_env(platform_arch, build_type):
log.info("Done initializing MSVC env")
-def platform_cmake_options():
+def platform_cmake_options(as_tuple_list=False):
result = []
if sys.platform == 'win32':
# Prevent cmake from auto-detecting clang if it is in path.
- result.append("-DCMAKE_C_COMPILER=cl.exe")
- result.append("-DCMAKE_CXX_COMPILER=cl.exe")
+ if as_tuple_list:
+ result.append(("CMAKE_C_COMPILER", "cl.exe"))
+ result.append(("CMAKE_CXX_COMPILER", "cl.exe"))
+ else:
+ result.append("-DCMAKE_C_COMPILER=cl.exe")
+ result.append("-DCMAKE_CXX_COMPILER=cl.exe")
return result
@@ -1250,3 +1256,69 @@ def parse_cmake_conf_assignments_by_key(source_dir):
d[key] = value
return d
+
+def configure_cmake_project(project_path,
+ cmake_path,
+ build_path=None,
+ temp_prefix_build_path=None,
+ cmake_args=None,
+ cmake_cache_args=None,
+ ):
+ clean_temp_dir = False
+ if not build_path:
+ # Ensure parent dir exists.
+ if temp_prefix_build_path:
+ os.makedirs(temp_prefix_build_path, exist_ok=True)
+
+ project_name = Path(project_path).name
+ build_path = tempfile.mkdtemp(prefix=f"{project_name}_", dir=temp_prefix_build_path)
+
+ if 'QFP_SETUP_KEEP_TEMP_FILES' not in os.environ:
+ clean_temp_dir = True
+
+ cmd = [cmake_path, '-G', 'Ninja', '-S', project_path, '-B', build_path]
+
+ if cmake_args:
+ cmd.extend(cmake_args)
+
+ for arg, value in cmake_cache_args:
+ cmd.extend([f'-D{arg}={value}'])
+
+ cmd_string = ' '.join(cmd)
+ # FIXME Python 3.7: Use subprocess.run()
+ proc = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ shell=False,
+ cwd=build_path,
+ universal_newlines=True)
+ output, error = proc.communicate()
+ proc.wait()
+ return_code = proc.returncode
+
+ if return_code != 0:
+ raise RuntimeError(f"\nFailed to configure CMake project \n "
+ f"'{project_path}' \n with error: \n {error}\n "
+ f"Return code: {return_code}\n"
+ f"Configure args were:\n {cmd_string}")
+
+ if clean_temp_dir:
+ rmtree(build_path)
+
+ return output
+
+
+def parse_cmake_project_message_info(output):
+ # Parse the output for anything prefixed
+ # '-- qfp:<category>:<key>: <value>' as created by the message()
+ # calls in a given CMake project and store it in a python dict.
+ result = defaultdict(lambda: defaultdict(str))
+ pattern = re.compile(r"^-- qfp:(.+?):(.+?):(.*)$")
+ for line in output.splitlines():
+ found = pattern.search(line)
+ if found:
+ category = found.group(1).strip()
+ key = found.group(2).strip()
+ value = found.group(3).strip()
+ result[category][key] = str(value)
+ return result
diff --git a/build_scripts/wheel_override.py b/build_scripts/wheel_override.py
index 4706863d5..ce5ba11ab 100644
--- a/build_scripts/wheel_override.py
+++ b/build_scripts/wheel_override.py
@@ -72,6 +72,7 @@ class PysideBuildWheel(_bdist_wheel, DistUtilsCommandMixin):
if wheel_module_exists else None)
def __init__(self, *args, **kwargs):
+ self.command_name = "bdist_wheel"
self._package_version = None
_bdist_wheel.__init__(self, *args, **kwargs)
DistUtilsCommandMixin.__init__(self)
@@ -108,12 +109,113 @@ class PysideBuildWheel(_bdist_wheel, DistUtilsCommandMixin):
components += (self.build_number,)
return '-'.join(components)
- # Copy of get_tag from bdist_wheel.py, to allow setting a
- # multi-python impl tag, by removing an assert. Otherwise we
- # would have to rename wheels manually for limited api
- # packages. Also we set "none" abi tag on Windows, because
- # pip does not yet support "abi3" tag, leading to
- # installation failure when tried.
+ # Modify the returned wheel tag tuple to use correct python version
+ # info when cross-compiling. We use the python info extracted from
+ # the shiboken python config test.
+ # setuptools / wheel don't support cross compiling out of the box
+ # at the moment. Relevant discussion at
+ # https://discuss.python.org/t/towards-standardizing-cross-compiling/10357
+ def get_cross_compiling_tag_tuple(self, tag_tuple):
+ (old_impl, old_abi_tag, plat_name) = tag_tuple
+
+ # Compute tag from the python version that the build command
+ # queried.
+ build_command = self.get_finalized_command('build')
+ python_target_info = build_command.python_target_info['python_info']
+
+ impl = 'no-py-ver-impl-available'
+ abi = 'no-abi-tag-info-available'
+ py_version = python_target_info['version'].split('.')
+ py_version_major = py_version[0]
+ py_version_minor = py_version[1]
+
+ so_abi = python_target_info['so_abi']
+ if so_abi and so_abi.startswith('cpython-'):
+ interpreter_name = so_abi.split('-')[0]
+ impl_name = tags.INTERPRETER_SHORT_NAMES.get(interpreter_name) or interpreter_name
+ impl_ver = f"{py_version_major}{py_version_minor}"
+ impl = impl_name + impl_ver
+ abi = 'cp' + so_abi.split('-')[1]
+ tag_tuple = (impl, abi, plat_name)
+ return tag_tuple
+
+ # Adjust wheel tag for limited api and cross compilation.
+ @staticmethod
+ def adjust_cross_compiled_many_linux_tag(old_tag):
+ (old_impl, old_abi_tag, old_plat_name) = old_tag
+
+ new_plat_name = old_plat_name
+
+ # TODO: Detect glibc version instead. We're abusing the
+ # manylinux2014 tag here, just like we did with manylinux1
+ # for x86_64 builds.
+ many_linux_prefix = 'manylinux2014'
+ linux_prefix = "linux_"
+ if old_plat_name.startswith(linux_prefix):
+ # Extract the arch suffix like -armv7l or -aarch64
+ plat_name_arch_suffix = \
+ old_plat_name[old_plat_name.index(linux_prefix) + len(linux_prefix):]
+
+ new_plat_name = f"{many_linux_prefix}_{plat_name_arch_suffix}"
+
+ tag = (old_impl, old_abi_tag, new_plat_name)
+ return tag
+
+ # Adjust wheel tag for limited api and cross compilation.
+ def adjust_tag_and_supported_tags(self, old_tag, supported_tags):
+ tag = old_tag
+ (old_impl, old_abi_tag, old_plat_name) = old_tag
+
+ # Get new tag for cross builds.
+ if self.is_cross_compile:
+ tag = self.get_cross_compiling_tag_tuple(old_tag)
+
+ # Get new tag for manylinux builds.
+ # To allow uploading to pypi, we need the wheel name
+ # to contain 'manylinux1'.
+ # The wheel which will be uploaded to pypi will be
+ # built on RHEL_8_2, so it doesn't completely qualify for
+ # manylinux1 support, but it's the minimum requirement
+ # for building Qt. We only enable this for x64 limited
+ # api builds (which are the only ones uploaded to pypi).
+ # TODO: Add actual distro detection, instead of
+ # relying on limited_api option if possible.
+ if (old_plat_name in ('linux-x86_64', 'linux_x86_64')
+ and sys.maxsize > 2147483647
+ and self.py_limited_api):
+ tag = (old_impl, old_abi_tag, 'manylinux1_x86_64')
+
+ # Set manylinux tag for cross-compiled builds when targeting
+ # limited api.
+ if self.is_cross_compile and self.py_limited_api:
+ tag = self.adjust_cross_compiled_many_linux_tag(tag)
+
+ # Reset the abi name and python versions supported by this wheel
+ # when targeting limited API. This is the same code that's
+ # in get_tag(), but done later after our own customizations.
+ if self.py_limited_api and old_impl.startswith('cp3'):
+ (_, _, adjusted_plat_name) = tag
+ impl = self.py_limited_api
+ abi_tag = 'abi3'
+ tag = (impl, abi_tag, adjusted_plat_name)
+
+ # Adjust abi name on Windows for limited api. It needs to be
+ # 'none' instead of 'abi3' because pip does not yet support
+ # the "abi3" tag on Windows, leading to a installation failure.
+ if self.py_limited_api and old_impl.startswith('cp3') and sys.platform == 'win32':
+ tag = (old_impl, 'none', old_plat_name)
+
+ # If building for limited API or we created a new tag, add it
+ # to the list of supported tags.
+ if tag != old_tag or self.py_limited_api:
+ supported_tags.append(tag)
+ return tag
+
+ # A slightly modified copy of get_tag from bdist_wheel.py, to allow
+ # adjusting the returned tag without triggering an assert. Otherwise
+ # we would have to rename wheels manually.
+ # Copy is up-to-date since commit
+ # 0acd203cd896afec7f715aa2ff5980a403459a3b in the wheel repo.
def get_tag(self):
# bdist sets self.plat_name if unset, we should only use it for purepy
# wheels if the user supplied it.
@@ -137,21 +239,7 @@ class PysideBuildWheel(_bdist_wheel, DistUtilsCommandMixin):
if plat_name in ('linux-x86_64', 'linux_x86_64') and sys.maxsize == 2147483647:
plat_name = 'linux_i686'
- # To allow uploading to pypi, we need the wheel name
- # to contain 'manylinux1'.
- # The wheel which will be uploaded to pypi will be
- # built on RHEL7, so it doesn't completely qualify for
- # manylinux1 support, but it's the minimum requirement
- # for building Qt. We only enable this for x64 limited
- # api builds (which are the only ones uploaded to
- # pypi).
- # TODO: Add actual distro detection, instead of
- # relying on limited_api option.
- if (plat_name in ('linux-x86_64', 'linux_x86_64')
- and sys.maxsize > 2147483647
- and (self.py_limited_api)):
- plat_name = 'manylinux1_x86_64'
- plat_name = plat_name.replace('-', '_').replace('.', '_')
+ plat_name = plat_name.lower().replace('-', '_').replace('.', '_')
if self.root_is_pure:
if self.universal:
@@ -166,15 +254,15 @@ class PysideBuildWheel(_bdist_wheel, DistUtilsCommandMixin):
# We don't work on CPython 3.1, 3.0.
if self.py_limited_api and (impl_name + impl_ver).startswith('cp3'):
impl = self.py_limited_api
- abi_tag = "abi3" if sys.platform != "win32" else "none"
+ abi_tag = 'abi3'
else:
abi_tag = str(get_abi_tag()).lower()
tag = (impl, abi_tag, plat_name)
- supported_tags = [(t.interpreter, t.abi, t.platform)
+ # issue gh-374: allow overriding plat_name
+ supported_tags = [(t.interpreter, t.abi, plat_name)
for t in tags.sys_tags()]
- # XXX switch to this alternate implementation for non-pure:
- if (self.py_limited_api) or (plat_name in ('manylinux1_x86_64')):
- return tag
+ # PySide's custom override.
+ tag = self.adjust_tag_and_supported_tags(tag, supported_tags)
assert tag in supported_tags, (f"would build wheel with unsupported tag {tag}")
return tag
diff --git a/sources/shiboken6/config.tests/target_qt_mkspec/CMakeLists.txt b/sources/shiboken6/config.tests/target_qt_mkspec/CMakeLists.txt
new file mode 100644
index 000000000..033369188
--- /dev/null
+++ b/sources/shiboken6/config.tests/target_qt_mkspec/CMakeLists.txt
@@ -0,0 +1,22 @@
+cmake_minimum_required(VERSION 3.16)
+project(dummy LANGUAGES CXX)
+
+include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/ShibokenHelpers.cmake")
+
+shiboken_internal_detect_if_cross_building()
+shiboken_internal_set_up_extra_dependency_paths()
+find_package(Qt6 REQUIRED COMPONENTS Core)
+
+get_target_property(darwin_target Qt6::Core QT_DARWIN_MIN_DEPLOYMENT_TARGET)
+
+# Get macOS minimum deployment target
+message(STATUS "qfp:qt_info:QMAKE_MACOSX_DEPLOYMENT_TARGET: ${darwin_target}")
+
+# Get Qt build type
+if(QT_FEATURE_debug_and_release)
+ message(STATUS "qfp:qt_info:BUILD_TYPE: debug_and_release")
+elseif(QT_FEATURE_debug)
+ message(STATUS "qfp:qt_info:BUILD_TYPE: debug")
+else()
+ message(STATUS "qfp:qt_info:BUILD_TYPE: release")
+endif()