diff options
Diffstat (limited to 'build_scripts/utils.py')
| -rw-r--r-- | build_scripts/utils.py | 546 |
1 files changed, 206 insertions, 340 deletions
diff --git a/build_scripts/utils.py b/build_scripts/utils.py index fd9b8f71d..74d9e6fc5 100644 --- a/build_scripts/utils.py +++ b/build_scripts/utils.py @@ -4,7 +4,6 @@ import errno import fnmatch import glob -import itertools import os import re import shutil @@ -15,17 +14,12 @@ import tempfile import urllib.request as urllib from collections import defaultdict from pathlib import Path +from textwrap import dedent, indent + +from .log import log +from . import (PYSIDE_PYTHON_TOOLS, PYSIDE_LINUX_BIN_TOOLS, PYSIDE_UNIX_LIBEXEC_TOOLS, + PYSIDE_WINDOWS_BIN_TOOLS, PYSIDE_UNIX_BIN_TOOLS, PYSIDE_UNIX_BUNDLED_TOOLS) -try: - # Using the distutils implementation within setuptools - from setuptools._distutils import log - from setuptools._distutils.errors import DistutilsSetupError -except ModuleNotFoundError: - # This is motivated by our CI using an old version of setuptools - # so then the coin_build_instructions.py script is executed, and - # import from this file, it was failing. - from distutils import log - from distutils.errors import DistutilsSetupError try: WindowsError @@ -33,6 +27,23 @@ except NameError: WindowsError = None +def which(name): + """ + Like shutil.which, but accepts a string or a PathLike and returns a Path + """ + path = None + try: + if isinstance(name, Path): + name = str(name) + path = shutil.which(name) + if path is None: + raise TypeError("None was returned") + path = Path(path) + except TypeError as e: + log.error(f"{name} was not found in PATH: {e}") + return path + + def is_64bit(): return sys.maxsize > 2147483647 @@ -49,7 +60,7 @@ def filter_match(name, patterns): def update_env_path(newpaths): paths = os.environ['PATH'].lower().split(os.pathsep) for path in newpaths: - if not path.lower() in paths: + if str(path).lower() not in paths: log.info(f"Inserting path '{path}' to environment") paths.insert(0, path) os.environ['PATH'] = f"{path}{os.pathsep}{os.environ['PATH']}" @@ -64,157 +75,6 @@ def get_numpy_location(): return None -def winsdk_setenv(platform_arch, build_type): - from setuptools._distutils import msvc9compiler as msvc9 - - sdk_version_map = { - "v6.0a": 9.0, - "v6.1": 9.0, - "v7.0": 9.0, - "v7.0a": 10.0, - "v7.1": 10.0 - } - - log.info(f"Searching Windows SDK with MSVC compiler version {msvc9.VERSION}") - setenv_paths = [] - for base in msvc9.HKEYS: - sdk_versions = msvc9.Reg.read_keys(base, msvc9.WINSDK_BASE) - if sdk_versions: - for sdk_version in sdk_versions: - installationfolder = msvc9.Reg.get_value(f"{msvc9.WINSDK_BASE}\\{sdk_version}", - "installationfolder") - # productversion = msvc9.Reg.get_value( - # "{}\\{}".format(msvc9.WINSDK_BASE, sdk_version), - # "productversion") - setenv_path = os.path.join(installationfolder, os.path.join('bin', 'SetEnv.cmd')) - if not os.path.exists(setenv_path): - continue - if sdk_version not in sdk_version_map: - continue - if sdk_version_map[sdk_version] != msvc9.VERSION: - continue - setenv_paths.append(setenv_path) - if len(setenv_paths) == 0: - raise DistutilsSetupError("Failed to find the Windows SDK with MSVC compiler " - f"version {msvc9.VERSION}") - for setenv_path in setenv_paths: - log.info(f"Found {setenv_path}") - - # Get SDK env (use latest SDK version installed on system) - setenv_path = setenv_paths[-1] - log.info(f"Using {setenv_path} ") - build_arch = "/x86" if platform_arch.startswith("32") else "/x64" - build_type = "/Debug" if build_type.lower() == "debug" else "/Release" - setenv_cmd = [setenv_path, build_arch, build_type] - setenv_env = get_environment_from_batch_command(setenv_cmd) - _setenv_paths = [setenv_env[k] for k in setenv_env if k.upper() == 'PATH'] - setenv_env_paths = os.pathsep.join(_setenv_paths).split(os.pathsep) - setenv_env_without_paths = {k: setenv_env[k] for k in setenv_env if k.upper() != 'PATH'} - - # Extend os.environ with SDK env - log.info("Initializing Windows SDK env...") - update_env_path(setenv_env_paths) - for k in sorted(setenv_env_without_paths): - v = setenv_env_without_paths[k] - log.info(f"Inserting '{k} = {v}' to environment") - os.environ[k] = v - log.info("Done initializing Windows SDK env") - - -def find_vcdir(version): - """ - This is the customized version of - setuptools._distutils.msvc9compiler.find_vcvarsall method - """ - from setuptools._distutils import msvc9compiler as msvc9 - vsbase = msvc9.VS_BASE % version - try: - productdir = msvc9.Reg.get_value(rf"{vsbase}\Setup\VC", "productdir") - except KeyError: - productdir = None - - # trying Express edition - if productdir is None: - try: - hasattr(msvc9, VSEXPRESS_BASE) # noqa: VSEXPRESS_BASE get defined with msvc9 - except AttributeError: - pass - else: - vsbase = VSEXPRESS_BASE % version # noqa: VSEXPRESS_BASE get defined with msvc9 - try: - productdir = msvc9.Reg.get_value(rf"{vsbase}\Setup\VC", "productdir") - except KeyError: - productdir = None - log.debug("Unable to find productdir in registry") - - if not productdir or not os.path.isdir(productdir): - toolskey = f"VS{version:0.0f}0COMNTOOLS" - toolsdir = os.environ.get(toolskey, None) - - if toolsdir and os.path.isdir(toolsdir): - productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") - productdir = os.path.abspath(productdir) - if not os.path.isdir(productdir): - log.debug(f"{productdir} is not a valid directory") - return None - else: - log.debug(f"Env var {toolskey} is not set or invalid") - if not productdir: - log.debug("No productdir found") - return None - return productdir - - -def init_msvc_env(platform_arch, build_type): - from setuptools._distutils import msvc9compiler as msvc9 - - log.info(f"Searching MSVC compiler version {msvc9.VERSION}") - vcdir_path = find_vcdir(msvc9.VERSION) - if not vcdir_path: - raise DistutilsSetupError(f"Failed to find the MSVC compiler version {msvc9.VERSION} on " - "your system.") - else: - log.info(f"Found {vcdir_path}") - - log.info(f"Searching MSVC compiler {msvc9.VERSION} environment init script") - if platform_arch.startswith("32"): - vcvars_path = os.path.join(vcdir_path, "bin", "vcvars32.bat") - else: - vcvars_path = os.path.join(vcdir_path, "bin", "vcvars64.bat") - if not os.path.exists(vcvars_path): - vcvars_path = os.path.join(vcdir_path, "bin", "amd64", "vcvars64.bat") - if not os.path.exists(vcvars_path): - vcvars_path = os.path.join(vcdir_path, "bin", "amd64", "vcvarsamd64.bat") - - if not os.path.exists(vcvars_path): - # MSVC init script not found, try to find and init Windows SDK env - log.error("Failed to find the MSVC compiler environment init script " - "(vcvars.bat) on your system.") - winsdk_setenv(platform_arch, build_type) - return - else: - log.info(f"Found {vcvars_path}") - - # Get MSVC env - log.info(f"Using MSVC {msvc9.VERSION} in {vcvars_path}") - msvc_arch = "x86" if platform_arch.startswith("32") else "amd64" - log.info(f"Getting MSVC env for {msvc_arch} architecture") - vcvars_cmd = [vcvars_path, msvc_arch] - msvc_env = get_environment_from_batch_command(vcvars_cmd) - _msvc_paths = [msvc_env[k] for k in msvc_env if k.upper() == 'PATH'] - msvc_env_paths = os.pathsep.join(_msvc_paths).split(os.pathsep) - msvc_env_without_paths = {k: msvc_env[k] for k in msvc_env if k.upper() != 'PATH'} - - # Extend os.environ with MSVC env - log.info("Initializing MSVC env...") - update_env_path(msvc_env_paths) - for k in sorted(msvc_env_without_paths): - v = msvc_env_without_paths[k] - log.info(f"Inserting '{k} = {v}' to environment") - os.environ[k] = v - log.info("Done initializing MSVC env") - - def platform_cmake_options(as_tuple_list=False): result = [] if sys.platform == 'win32': @@ -230,16 +90,19 @@ def platform_cmake_options(as_tuple_list=False): def copyfile(src, dst, force=True, _vars=None, force_copy_symlink=False, make_writable_by_owner=False): - if _vars is not None: - src = src.format(**_vars) - dst = dst.format(**_vars) - - if not os.path.exists(src) and not force: + if isinstance(src, str): + src = Path(src.format(**_vars)) if _vars else Path(src) + if isinstance(dst, str): + dst = Path(dst.format(**_vars)) if _vars else Path(dst) + assert (isinstance(src, Path)) + assert (isinstance(dst, Path)) + + if not src.exists() and not force: log.info(f"**Skipping copy file\n {src} to\n {dst}\n Source does not exist") return - if not os.path.islink(src) or force_copy_symlink: - if os.path.isfile(dst): + if not src.is_symlink() or force_copy_symlink: + if dst.is_file(): src_stat = os.stat(src) dst_stat = os.stat(dst) if (src_stat.st_size == dst_stat.st_size @@ -247,23 +110,25 @@ def copyfile(src, dst, force=True, _vars=None, force_copy_symlink=False, log.info(f"{dst} is up to date.") return dst - log.info(f"Copying file\n {src} to\n {dst}.") + log.debug(f"Copying file\n {src} to\n {dst}.") shutil.copy2(src, dst) if make_writable_by_owner: make_file_writable_by_owner(dst) return dst - link_target_path = os.path.realpath(src) - if os.path.dirname(link_target_path) == os.path.dirname(src): - link_target = os.path.basename(link_target_path) - link_name = os.path.basename(src) - current_directory = os.getcwd() + # We use 'strict=False' to mimic os.path.realpath in case + # the directory doesn't exist. + link_target_path = src.resolve(strict=False) + if link_target_path.parent == src.parent: + link_target = Path(link_target_path.name) + link_name = Path(src.name) + current_directory = Path.cwd() try: - target_dir = dst if os.path.isdir(dst) else os.path.dirname(dst) + target_dir = dst if dst.is_dir() else dst.parent os.chdir(target_dir) - if os.path.exists(link_name): - if (os.path.islink(link_name) + if link_name.exists(): + if (link_name.is_symlink() and os.readlink(link_name) == link_target): log.info(f"Symlink already exists\n {link_name} ->\n {link_target}") return dst @@ -285,13 +150,13 @@ def makefile(dst, content=None, _vars=None): if _vars is not None: if content is not None: content = content.format(**_vars) - dst = dst.format(**_vars) + dst = Path(dst.format(**_vars)) log.info(f"Making file {dst}.") - dstdir = os.path.dirname(dst) - if not os.path.exists(dstdir): - os.makedirs(dstdir) + dstdir = dst.parent + if not dstdir.exists(): + dstdir.mkdir(parents=True) with open(dst, "wt") as f: if content is not None: @@ -301,30 +166,35 @@ def makefile(dst, content=None, _vars=None): def copydir(src, dst, _filter=None, ignore=None, force=True, recursive=True, _vars=None, dir_filter_function=None, file_filter_function=None, force_copy_symlinks=False): + if isinstance(src, str): + src = Path(src.format(**_vars)) if _vars else Path(src) + if isinstance(dst, str): + dst = Path(dst.format(**_vars)) if _vars else Path(dst) + assert (isinstance(src, Path)) + assert (isinstance(dst, Path)) + if _vars is not None: - src = src.format(**_vars) - dst = dst.format(**_vars) if _filter is not None: _filter = [i.format(**_vars) for i in _filter] if ignore is not None: ignore = [i.format(**_vars) for i in ignore] - if not os.path.exists(src) and not force: + if not src.exists() and not force: log.info(f"**Skipping copy tree\n {src} to\n {dst}\n Source does not exist. " f"filter={_filter}. ignore={ignore}.") return [] - log.info(f"Copying tree\n {src} to\n {dst}. filter={_filter}. ignore={ignore}.") + log.debug(f"Copying tree\n {src} to\n {dst}. filter={_filter}. ignore={ignore}.") names = os.listdir(src) results = [] copy_errors = [] for name in names: - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) + srcname = src / name + dstname = dst / name try: - if os.path.isdir(srcname): + if srcname.is_dir(): if (dir_filter_function and not dir_filter_function(name, src, srcname)): continue if recursive: @@ -336,8 +206,8 @@ def copydir(src, dst, _filter=None, ignore=None, force=True, recursive=True, _va or (_filter is not None and not filter_match(name, _filter)) or (ignore is not None and filter_match(name, ignore))): continue - if not os.path.exists(dst): - os.makedirs(dst) + if not dst.is_dir(): + dst.mkdir(parents=True) results.append(copyfile(srcname, dstname, True, _vars, force_copy_symlinks)) # catch the Error from the recursive copytree so that we can # continue with other files @@ -346,8 +216,8 @@ def copydir(src, dst, _filter=None, ignore=None, force=True, recursive=True, _va except EnvironmentError as why: copy_errors.append((srcname, dstname, str(why))) try: - if os.path.exists(dst): - shutil.copystat(src, dst) + if dst.exists(): + shutil.copystat(str(src), str(dst)) except OSError as why: if WindowsError is not None and isinstance(why, WindowsError): # Copying file access times may fail on Windows @@ -394,7 +264,7 @@ def run_process(args, initial_env=None): No output is captured. """ command = " ".join([(" " in x and f'"{x}"' or x) for x in args]) - log.info(f"In directory {os.getcwd()}:\n\tRunning command: {command}") + log.debug(f"In directory {Path.cwd()}:\n\tRunning command: {command}") if initial_env is None: initial_env = os.environ @@ -406,62 +276,10 @@ def run_process(args, initial_env=None): return exit_code -def get_environment_from_batch_command(env_cmd, initial=None): - """ - Take a command (either a single command or list of arguments) - and return the environment created after running that command. - Note that if the command must be a batch file or .cmd file, or the - changes to the environment will not be captured. - - If initial is supplied, it is used as the initial environment passed - to the child process. - """ - - def validate_pair(ob): - if len(ob) != 2: - log.error(f"Unexpected result: {ob}") - return False - return True - - def consume(it): - try: - while True: - next(it) - except StopIteration: - pass - - if not isinstance(env_cmd, (list, tuple)): - env_cmd = [env_cmd] - # construct the command that will alter the environment - env_cmd = subprocess.list2cmdline(env_cmd) - # create a tag so we can tell in the output when the proc is done - tag = 'Done running command' - # construct a cmd.exe command to do accomplish this - cmd = f'cmd.exe /E:ON /V:ON /s /c "{env_cmd} && echo "{tag}" && set"' - # launch the process - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=initial) - # parse the output sent to stdout - lines = proc.stdout - # make sure the lines are strings - lines = [s.decode() for s in lines] - # consume whatever output occurs until the tag is reached - consume(itertools.takewhile(lambda l: tag not in l, lines)) - # define a way to handle each KEY=VALUE line - # parse key/values into pairs - pairs = [l.rstrip().split('=', 1) for l in lines] - # make sure the pairs are valid - valid_pairs = filter(validate_pair, pairs) - # construct a dictionary of the pairs - result = dict(valid_pairs) - # let the process finish - proc.communicate() - return result - - def back_tick(cmd, ret_err=False): """ - Run command `cmd`, return stdout, or stdout, stderr, - return_code if `ret_err` is True. + Run command `cmd`, return stdout, or (stdout, stderr, + return_code) if `ret_err` is True. Parameters ---------- @@ -485,22 +303,20 @@ def back_tick(cmd, ret_err=False): Raises RuntimeError if command returns non-zero exit code when ret_err isn't set. """ - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - out, err = proc.communicate() - if not isinstance(out, str): - # python 3 - out = out.decode() - err = err.decode() - retcode = proc.returncode - if retcode is None and not ret_err: - proc.terminate() - raise RuntimeError(f"{cmd} process did not terminate") - if retcode != 0 and not ret_err: - raise RuntimeError(f"{cmd} process returned code {retcode}\n*** {err}") - out = out.strip() + with subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) as proc: + out_bytes, err_bytes = proc.communicate() + out = out_bytes.decode().strip() + err = err_bytes.decode().strip() + retcode = proc.returncode + if retcode is None and not ret_err: + proc.terminate() + raise RuntimeError(f"{cmd} process did not terminate") + if retcode != 0 and not ret_err: + raise RuntimeError(f"{cmd} process returned code {retcode}\n*** {err}") if not ret_err: return out - return out, err.strip(), retcode + return out, err, retcode MACOS_OUTNAME_RE = re.compile(r'\(compatibility version [\d.]+, current version [\d.]+\)') @@ -657,7 +473,7 @@ def find_glob_in_path(pattern): pattern += '.exe' for path in os.environ.get('PATH', '').split(os.pathsep): - for match in glob.glob(os.path.join(path, pattern)): + for match in glob.glob(str(Path(path) / pattern)): result.append(match) return result @@ -682,7 +498,7 @@ def detect_clang(): clang_dir = os.environ.get(source, None) if not clang_dir: raise OSError("clang not found") - return (clang_dir, source) + return (Path(clang_dir), source) _7z_binary = None @@ -708,8 +524,8 @@ def download_and_extract_7z(fileurl, target): outputDir = f"-o{target}" if not _7z_binary: if sys.platform == "win32": - candidate = "c:\\Program Files\\7-Zip\\7z.exe" - if os.path.exists(candidate): + candidate = Path("c:\\Program Files\\7-Zip\\7z.exe") + if candidate.exists(): _7z_binary = candidate if not _7z_binary: _7z_binary = '7z' @@ -839,7 +655,8 @@ def _ldd_ldso(executable_path): # Choose appropriate runtime dynamic linker. for rtld in rtld_list: - if os.path.isfile(rtld) and os.access(rtld, os.X_OK): + rtld = Path(rtld) + if rtld.is_file() and os.access(rtld, os.X_OK): (_, _, code) = back_tick(rtld, True) # Code 127 is returned by ld.so when called without any # arguments (some kind of sanity check I guess). @@ -885,7 +702,7 @@ def ldd(executable_path): result = _ldd_ldd(executable_path) except RuntimeError as e: message = f"ldd: Falling back to ld.so ({str(e)})" - log.warn(message) + log.warning(message) if not result: result = _ldd_ldso(executable_path) return result @@ -893,8 +710,8 @@ def ldd(executable_path): def find_files_using_glob(path, pattern): """ Returns list of files that matched glob `pattern` in `path`. """ - final_pattern = os.path.join(path, pattern) - maybe_files = glob.glob(final_pattern) + final_pattern = Path(path) / pattern + maybe_files = glob.glob(str(final_pattern)) return maybe_files @@ -911,14 +728,16 @@ def find_qt_core_library_glob(lib_dir): # ldd for the specified platforms. # This has less priority because ICU libs are not used in the default # Qt configuration build. +# Note: Uses ldd to query shared library dependencies and thus does not +# work for cross builds. def copy_icu_libs(patchelf, destination_lib_dir): """ Copy ICU libraries that QtCore depends on, to given `destination_lib_dir`. """ - qt_core_library_path = find_qt_core_library_glob(destination_lib_dir) + qt_core_library_path = Path(find_qt_core_library_glob(destination_lib_dir)) - if not qt_core_library_path or not os.path.exists(qt_core_library_path): + if not qt_core_library_path or not qt_core_library_path.exists(): raise RuntimeError(f"QtCore library does not exist at path: {qt_core_library_path}. " "Failed to copy ICU libraries.") @@ -937,14 +756,15 @@ def copy_icu_libs(patchelf, destination_lib_dir): paths = ldd_get_paths_for_dependencies(icu_regex, dependencies=dependencies) if not paths: raise RuntimeError("Failed to find the necessary ICU libraries required by QtCore.") - log.info('Copying the detected ICU libraries required by QtCore.') + log.debug('Copying the detected ICU libraries required by QtCore.') - if not os.path.exists(destination_lib_dir): - os.makedirs(destination_lib_dir) + destination_lib_dir = Path(destination_lib_dir) + if not destination_lib_dir.exists(): + destination_lib_dir.mkdir(parents=True) for path in paths: - basename = os.path.basename(path) - destination = os.path.join(destination_lib_dir, basename) + basename = Path(path).name + destination = destination_lib_dir / basename copyfile(path, destination, force_copy_symlink=True) # Patch the ICU libraries to contain the $ORIGIN rpath # value, so that only the local package libraries are used. @@ -969,7 +789,7 @@ def linux_run_read_elf(executable_path): def linux_set_rpaths(patchelf, executable_path, rpath_string): """ Patches the `executable_path` with a new rpath string. """ - cmd = [patchelf, '--set-rpath', rpath_string, executable_path] + cmd = [str(patchelf), '--set-rpath', str(rpath_string), str(executable_path)] if run_process(cmd) != 0: raise RuntimeError(f"Error patching rpath in {executable_path}") @@ -1073,6 +893,7 @@ def linux_fix_rpaths_for_library(patchelf, executable_path, qt_rpath, override=F existing_rpaths = linux_get_rpaths(executable_path) rpaths.extend(existing_rpaths) + qt_rpath = str(qt_rpath) if linux_needs_qt_rpath(executable_path) and qt_rpath not in existing_rpaths: rpaths.append(qt_rpath) @@ -1111,26 +932,9 @@ def get_python_dict(python_script_path): raise -def install_pip_package_from_url_specifier(env_pip, url, upgrade=True): - args = [env_pip, "install", url] - if upgrade: - args.append("--upgrade") - args.append(url) - run_instruction(args, f"Failed to install {url}") - - -def install_pip_dependencies(env_pip, packages, upgrade=True): - for p in packages: - args = [env_pip, "install"] - if upgrade: - args.append("--upgrade") - args.append(p) - run_instruction(args, f"Failed to install {p}") - - def get_qtci_virtualEnv(python_ver, host, hostArch, targetArch): _pExe = "python" - _env = f"env{python_ver}" + _env = f"{os.environ.get('PYSIDE_VIRTUALENV') or 'env'+python_ver}" env_python = f"{_env}/bin/python" env_pip = f"{_env}/bin/pip" @@ -1142,22 +946,22 @@ def get_qtci_virtualEnv(python_ver, host, hostArch, targetArch): if python_ver.startswith("3"): var = f"PYTHON{python_ver}-32_PATH" log.info(f"Try to find python from {var} env variable") - _path = os.getenv(var, "") - _pExe = os.path.join(_path, "python.exe") - if not os.path.isfile(_pExe): - log.warn(f"Can't find python.exe from {_pExe}, using default python3") - _pExe = os.path.join(os.getenv("PYTHON3_32_PATH"), "python.exe") + _path = Path(os.getenv(var, "")) + _pExe = _path / "python.exe" + if not _pExe.is_file(): + log.warning(f"Can't find python.exe from {_pExe}, using default python3") + _pExe = Path(os.getenv("PYTHON3_32_PATH")) / "python.exe" else: - _pExe = os.path.join(os.getenv("PYTHON2_32_PATH"), "python.exe") + _pExe = Path(os.getenv("PYTHON2_32_PATH")) / "python.exe" else: if python_ver.startswith("3"): var = f"PYTHON{python_ver}-64_PATH" log.info(f"Try to find python from {var} env variable") - _path = os.getenv(var, "") - _pExe = os.path.join(_path, "python.exe") - if not os.path.isfile(_pExe): - log.warn(f"Can't find python.exe from {_pExe}, using default python3") - _pExe = os.path.join(os.getenv("PYTHON3_PATH"), "python.exe") + _path = Path(os.getenv(var, "")) + _pExe = _path / "python.exe" + if not _pExe.is_file(): + log.warning(f"Can't find python.exe from {_pExe}, using default python3") + _pExe = Path(os.getenv("PYTHON3_PATH")) / "python.exe" env_python = f"{_env}\\Scripts\\python.exe" env_pip = f"{_env}\\Scripts\\pip.exe" else: @@ -1167,7 +971,7 @@ def get_qtci_virtualEnv(python_ver, host, hostArch, targetArch): except Exception as e: print(f"Exception {type(e).__name__}: {e}") _pExe = "python3" - return(_pExe, _env, env_pip, env_python) + return (_pExe, _env, env_pip, env_python) def run_instruction(instruction, error, initial_env=None): @@ -1180,23 +984,6 @@ def run_instruction(instruction, error, initial_env=None): exit(result) -def acceptCITestConfiguration(hostOS, hostOSVer, targetArch, compiler): - # Disable unsupported CI configs for now - # NOTE: String must match with QT CI's storagestruct thrift - if (hostOSVer in ["WinRT_10", "WebAssembly", "Ubuntu_18_04", "Android_ANY"] - or hostOSVer.startswith("SLES_")): - log.info("Disabled {hostOSVer} from Coin configuration") - return False - # With 5.11 CI will create two sets of release binaries, - # one with msvc 2015 and one with msvc 2017 - # we shouldn't release the 2015 version. - # BUT, 32 bit build is done only on msvc 2015... - if compiler in ["MSVC2015"] and targetArch in ["X86_64"]: - log.warn(f"Disabled {compiler} to {targetArch} from Coin configuration") - return False - return True - - def get_ci_qtpaths_path(ci_install_dir, ci_host_os): qtpaths_path = f"--qtpaths={ci_install_dir}" if ci_host_os == "MacOS": @@ -1230,6 +1017,40 @@ def parse_cmake_conf_assignments_by_key(source_dir): return d +def _configure_failure_message(project_path, cmd, return_code, output, error, env): + """Format a verbose message about configure_cmake_project() failures.""" + cmd_string = ' '.join(cmd) + error_text = indent(error.strip(), " ") + output_text = indent(output.strip(), " ") + result = dedent(f""" + Failed to configure CMake project: '{project_path}' + Configure args were: + {cmd_string} + Return code: {return_code} + """) + + first = True + for k, v in env.items(): + if k.startswith("CMAKE"): + if first: + result += "Environment:\n" + first = False + result += f" {k}={v}\n" + + result += f"\nwith error:\n{error_text}\n" + + CMAKE_CMAKEOUTPUT_LOG_PATTERN = r'See also "([^"]+CMakeOutput\.log)"\.' + cmakeoutput_log_match = re.search(CMAKE_CMAKEOUTPUT_LOG_PATTERN, output) + if cmakeoutput_log_match: + cmakeoutput_log = Path(cmakeoutput_log_match.group(1)) + if cmakeoutput_log.is_file(): + log = indent(cmakeoutput_log.read_text().strip(), " ") + result += f"CMakeOutput.log:\n{log}\n" + + result += f"Output:\n{output_text}\n" + return result + + def configure_cmake_project(project_path, cmake_path, build_path=None, @@ -1257,23 +1078,18 @@ def configure_cmake_project(project_path, for arg, value in cmake_cache_args: cmd.extend([f'-D{arg}={value}']) - cmd_string = ' '.join(cmd) - # FIXME Python 3.7: Use subprocess.run() - proc = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False, - cwd=build_path, - universal_newlines=True) - output, error = proc.communicate() - proc.wait() + cmd = [str(i) for i in cmd] + + proc = subprocess.run(cmd, shell=False, cwd=build_path, + capture_output=True, universal_newlines=True) return_code = proc.returncode + output = proc.stdout + error = proc.stderr if return_code != 0: - raise RuntimeError(f"\nFailed to configure CMake project \n " - f"'{project_path}' \n with error: \n {error}\n " - f"Return code: {return_code}\n" - f"Configure args were:\n {cmd_string}") + m = _configure_failure_message(project_path, cmd, return_code, + output, error, os.environ) + raise RuntimeError(m) if clean_temp_dir: remove_tree(build_path) @@ -1295,3 +1111,53 @@ def parse_cmake_project_message_info(output): value = found.group(3).strip() result[category][key] = str(value) return result + + +def available_pyside_tools(qt_tools_path: Path, package_for_wheels: bool = False): + pyside_tools = PYSIDE_PYTHON_TOOLS.copy() + + if package_for_wheels: + # Qt wrappers in build/{python_env_name}/package_for_wheels/PySide6 + bin_path = qt_tools_path + else: + bin_path = qt_tools_path / "bin" + + def tool_exist(tool_path: Path): + if tool_path.exists(): + return True + else: + log.warning(f"{tool_path} not found. pyside-{tool_path.name} not included.") + return False + + if sys.platform == 'win32': + pyside_tools.extend([tool for tool in PYSIDE_WINDOWS_BIN_TOOLS + if tool_exist(bin_path / f"{tool}.exe")]) + else: + lib_exec_path = qt_tools_path / "Qt" / "libexec" if package_for_wheels \ + else qt_tools_path / "libexec" + pyside_tools.extend([tool for tool in PYSIDE_UNIX_LIBEXEC_TOOLS + if tool_exist(lib_exec_path / tool)]) + if sys.platform == 'darwin': + def name_to_path(name): + return f"{name.capitalize()}.app/Contents/MacOS/{name.capitalize()}" + + pyside_tools.extend([tool for tool in PYSIDE_UNIX_BIN_TOOLS + if tool_exist(bin_path / tool)]) + pyside_tools.extend([tool for tool in PYSIDE_UNIX_BUNDLED_TOOLS + if tool_exist(bin_path / name_to_path(tool))]) + else: + pyside_tools.extend([tool for tool in PYSIDE_LINUX_BIN_TOOLS + if tool_exist(bin_path / tool)]) + + return pyside_tools + + +def copy_qt_metatypes(destination_qt_dir, _vars): + """Copy the Qt metatypes files which changed location in 6.5""" + # <qt>/[lib]?/metatypes/* -> <setup>/{st_package_name}/Qt/[lib]?/metatypes + qt_meta_types_dir = "{qt_metatypes_dir}".format(**_vars) + qt_prefix_dir = "{qt_prefix_dir}".format(**_vars) + rel_meta_data_dir = os.fspath(Path(qt_meta_types_dir).relative_to(qt_prefix_dir)) + copydir(qt_meta_types_dir, destination_qt_dir / rel_meta_data_dir, + _filter=["*.json"], + recursive=False, _vars=_vars, force_copy_symlinks=True) |