diff options
Diffstat (limited to 'build_scripts/utils.py')
-rw-r--r-- | build_scripts/utils.py | 784 |
1 files changed, 361 insertions, 423 deletions
diff --git a/build_scripts/utils.py b/build_scripts/utils.py index 4f22f7d7b..74d9e6fc5 100644 --- a/build_scripts/utils.py +++ b/build_scripts/utils.py @@ -1,58 +1,25 @@ -############################################################################# -## -## Copyright (C) 2018 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -import sys -from pathlib import Path +import errno +import fnmatch +import glob import os import re -import stat -import errno import shutil +import stat import subprocess -import fnmatch -import itertools -import glob - +import sys +import tempfile import urllib.request as urllib +from collections import defaultdict +from pathlib import Path +from textwrap import dedent, indent + +from .log import log +from . import (PYSIDE_PYTHON_TOOLS, PYSIDE_LINUX_BIN_TOOLS, PYSIDE_UNIX_LIBEXEC_TOOLS, + PYSIDE_WINDOWS_BIN_TOOLS, PYSIDE_UNIX_BIN_TOOLS, PYSIDE_UNIX_BUNDLED_TOOLS) -import distutils.log as log -from distutils.errors import DistutilsSetupError try: WindowsError @@ -60,6 +27,27 @@ except NameError: WindowsError = None +def which(name): + """ + Like shutil.which, but accepts a string or a PathLike and returns a Path + """ + path = None + try: + if isinstance(name, Path): + name = str(name) + path = shutil.which(name) + if path is None: + raise TypeError("None was returned") + path = Path(path) + except TypeError as e: + log.error(f"{name} was not found in PATH: {e}") + return path + + +def is_64bit(): + return sys.maxsize > 2147483647 + + def filter_match(name, patterns): for pattern in patterns: if pattern is None: @@ -72,7 +60,7 @@ def filter_match(name, patterns): def update_env_path(newpaths): paths = os.environ['PATH'].lower().split(os.pathsep) for path in newpaths: - if not path.lower() in paths: + if str(path).lower() not in paths: log.info(f"Inserting path '{path}' to environment") paths.insert(0, path) os.environ['PATH'] = f"{path}{os.pathsep}{os.environ['PATH']}" @@ -83,206 +71,72 @@ def get_numpy_location(): if 'site-' in p: numpy = Path(p).resolve() / 'numpy' if numpy.is_dir(): - return os.fspath(numpy / 'core' / 'include') + return os.fspath(numpy / 'core' / 'include') return None -def winsdk_setenv(platform_arch, build_type): - from distutils.msvc9compiler import VERSION as MSVC_VERSION - from distutils.msvc9compiler import Reg - from distutils.msvc9compiler import HKEYS - from distutils.msvc9compiler import WINSDK_BASE - - sdk_version_map = { - "v6.0a": 9.0, - "v6.1": 9.0, - "v7.0": 9.0, - "v7.0a": 10.0, - "v7.1": 10.0 - } - - log.info(f"Searching Windows SDK with MSVC compiler version {MSVC_VERSION}") - setenv_paths = [] - for base in HKEYS: - sdk_versions = Reg.read_keys(base, WINSDK_BASE) - if sdk_versions: - for sdk_version in sdk_versions: - installationfolder = Reg.get_value(f"{WINSDK_BASE}\\{sdk_version}", - "installationfolder") - # productversion = Reg.get_value("{}\\{}".format(WINSDK_BASE, sdk_version), - # "productversion") - setenv_path = os.path.join(installationfolder, os.path.join('bin', 'SetEnv.cmd')) - if not os.path.exists(setenv_path): - continue - if sdk_version not in sdk_version_map: - continue - if sdk_version_map[sdk_version] != MSVC_VERSION: - continue - setenv_paths.append(setenv_path) - if len(setenv_paths) == 0: - raise DistutilsSetupError("Failed to find the Windows SDK with MSVC compiler " - f"version {MSVC_VERSION}") - for setenv_path in setenv_paths: - log.info(f"Found {setenv_path}") - - # Get SDK env (use latest SDK version installed on system) - setenv_path = setenv_paths[-1] - log.info(f"Using {setenv_path} ") - build_arch = "/x86" if platform_arch.startswith("32") else "/x64" - build_type = "/Debug" if build_type.lower() == "debug" else "/Release" - setenv_cmd = [setenv_path, build_arch, build_type] - setenv_env = get_environment_from_batch_command(setenv_cmd) - setenv_env_paths = os.pathsep.join([setenv_env[k] for k in setenv_env if k.upper() == 'PATH']).split(os.pathsep) - setenv_env_without_paths = dict([(k, setenv_env[k]) for k in setenv_env if k.upper() != 'PATH']) - - # Extend os.environ with SDK env - log.info("Initializing Windows SDK env...") - update_env_path(setenv_env_paths) - for k in sorted(setenv_env_without_paths): - v = setenv_env_without_paths[k] - log.info(f"Inserting '{k} = {v}' to environment") - os.environ[k] = v - log.info("Done initializing Windows SDK env") - - -def find_vcdir(version): - """ - This is the customized version of - distutils.msvc9compiler.find_vcvarsall method - """ - from distutils.msvc9compiler import VS_BASE - from distutils.msvc9compiler import Reg - vsbase = VS_BASE % version - try: - productdir = Reg.get_value(rf"{vsbase}\Setup\VC", "productdir") - except KeyError: - productdir = None - - # trying Express edition - if productdir is None: - try: - from distutils.msvc9compiler import VSEXPRESS_BASE - except ImportError: - pass - else: - vsbase = VSEXPRESS_BASE % version - try: - productdir = Reg.get_value(rf"{vsbase}\Setup\VC", "productdir") - except KeyError: - productdir = None - log.debug("Unable to find productdir in registry") - - if not productdir or not os.path.isdir(productdir): - toolskey = f"VS{version:0.0f}0COMNTOOLS" - toolsdir = os.environ.get(toolskey, None) - - if toolsdir and os.path.isdir(toolsdir): - productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") - productdir = os.path.abspath(productdir) - if not os.path.isdir(productdir): - log.debug(f"{productdir} is not a valid directory") - return None +def platform_cmake_options(as_tuple_list=False): + result = [] + if sys.platform == 'win32': + # Prevent cmake from auto-detecting clang if it is in path. + if as_tuple_list: + result.append(("CMAKE_C_COMPILER", "cl.exe")) + result.append(("CMAKE_CXX_COMPILER", "cl.exe")) else: - log.debug(f"Env var {toolskey} is not set or invalid") - if not productdir: - log.debug("No productdir found") - return None - return productdir - - -def init_msvc_env(platform_arch, build_type): - from distutils.msvc9compiler import VERSION as MSVC_VERSION + result.append("-DCMAKE_C_COMPILER=cl.exe") + result.append("-DCMAKE_CXX_COMPILER=cl.exe") + return result - log.info(f"Searching MSVC compiler version {MSVC_VERSION}") - vcdir_path = find_vcdir(MSVC_VERSION) - if not vcdir_path: - raise DistutilsSetupError(f"Failed to find the MSVC compiler version {MSVC_VERSION} on " - "your system.") - else: - log.info(f"Found {vcdir_path}") - log.info(f"Searching MSVC compiler {MSVC_VERSION} environment init script") - if platform_arch.startswith("32"): - vcvars_path = os.path.join(vcdir_path, "bin", "vcvars32.bat") - else: - vcvars_path = os.path.join(vcdir_path, "bin", "vcvars64.bat") - if not os.path.exists(vcvars_path): - vcvars_path = os.path.join(vcdir_path, "bin", "amd64", "vcvars64.bat") - if not os.path.exists(vcvars_path): - vcvars_path = os.path.join(vcdir_path, "bin", "amd64", "vcvarsamd64.bat") - - if not os.path.exists(vcvars_path): - # MSVC init script not found, try to find and init Windows SDK env - log.error("Failed to find the MSVC compiler environment init script " - "(vcvars.bat) on your system.") - winsdk_setenv(platform_arch, build_type) - return - else: - log.info(f"Found {vcvars_path}") - - # Get MSVC env - log.info(f"Using MSVC {MSVC_VERSION} in {vcvars_path}") - msvc_arch = "x86" if platform_arch.startswith("32") else "amd64" - log.info(f"Getting MSVC env for {msvc_arch} architecture") - vcvars_cmd = [vcvars_path, msvc_arch] - msvc_env = get_environment_from_batch_command(vcvars_cmd) - msvc_env_paths = os.pathsep.join([msvc_env[k] for k in msvc_env if k.upper() == 'PATH']).split(os.pathsep) - msvc_env_without_paths = dict([(k, msvc_env[k]) for k in msvc_env if k.upper() != 'PATH']) - - # Extend os.environ with MSVC env - log.info("Initializing MSVC env...") - update_env_path(msvc_env_paths) - for k in sorted(msvc_env_without_paths): - v = msvc_env_without_paths[k] - log.info(f"Inserting '{k} = {v}' to environment") - os.environ[k] = v - log.info("Done initializing MSVC env") - - -def copyfile(src, dst, force=True, vars=None, force_copy_symlink=False, +def copyfile(src, dst, force=True, _vars=None, force_copy_symlink=False, make_writable_by_owner=False): - if vars is not None: - src = src.format(**vars) - dst = dst.format(**vars) - - if not os.path.exists(src) and not force: - log.info(f"**Skipping copy file {src} to {dst}. Source does not exists.") + if isinstance(src, str): + src = Path(src.format(**_vars)) if _vars else Path(src) + if isinstance(dst, str): + dst = Path(dst.format(**_vars)) if _vars else Path(dst) + assert (isinstance(src, Path)) + assert (isinstance(dst, Path)) + + if not src.exists() and not force: + log.info(f"**Skipping copy file\n {src} to\n {dst}\n Source does not exist") return - if not os.path.islink(src) or force_copy_symlink: - if os.path.isfile(dst): + if not src.is_symlink() or force_copy_symlink: + if dst.is_file(): src_stat = os.stat(src) dst_stat = os.stat(dst) - if (src_stat.st_size == dst_stat.st_size and - src_stat.st_mtime <= dst_stat.st_mtime): + if (src_stat.st_size == dst_stat.st_size + and src_stat.st_mtime <= dst_stat.st_mtime): log.info(f"{dst} is up to date.") return dst - log.info(f"Copying file {src} to {dst}.") + log.debug(f"Copying file\n {src} to\n {dst}.") shutil.copy2(src, dst) if make_writable_by_owner: make_file_writable_by_owner(dst) return dst - link_target_path = os.path.realpath(src) - if os.path.dirname(link_target_path) == os.path.dirname(src): - link_target = os.path.basename(link_target_path) - link_name = os.path.basename(src) - current_directory = os.getcwd() + # We use 'strict=False' to mimic os.path.realpath in case + # the directory doesn't exist. + link_target_path = src.resolve(strict=False) + if link_target_path.parent == src.parent: + link_target = Path(link_target_path.name) + link_name = Path(src.name) + current_directory = Path.cwd() try: - target_dir = dst if os.path.isdir(dst) else os.path.dirname(dst) + target_dir = dst if dst.is_dir() else dst.parent os.chdir(target_dir) - if os.path.exists(link_name): - if (os.path.islink(link_name) and - os.readlink(link_name) == link_target): - log.info(f"Symlink {link_name} -> {link_target} already exists.") + if link_name.exists(): + if (link_name.is_symlink() + and os.readlink(link_name) == link_target): + log.info(f"Symlink already exists\n {link_name} ->\n {link_target}") return dst os.remove(link_name) - log.info(f"Symlinking {link_name} -> {link_target} in {target_dir}.") + log.info(f"Symlinking\n {link_name} ->\n {link_target} in\n {target_dir}") os.symlink(link_target, link_name) except OSError: - log.error(f"{link_name} -> {link_target}: Error creating symlink") + log.error(f"Error creating symlink\n {link_name} ->\n {link_target}") finally: os.chdir(current_directory) else: @@ -292,83 +146,86 @@ def copyfile(src, dst, force=True, vars=None, force_copy_symlink=False, return dst -def makefile(dst, content=None, vars=None): - if vars is not None: +def makefile(dst, content=None, _vars=None): + if _vars is not None: if content is not None: - content = content.format(**vars) - dst = dst.format(**vars) + content = content.format(**_vars) + dst = Path(dst.format(**_vars)) log.info(f"Making file {dst}.") - dstdir = os.path.dirname(dst) - if not os.path.exists(dstdir): - os.makedirs(dstdir) + dstdir = dst.parent + if not dstdir.exists(): + dstdir.mkdir(parents=True) with open(dst, "wt") as f: if content is not None: f.write(content) -def copydir(src, dst, filter=None, ignore=None, force=True, recursive=True, vars=None, +def copydir(src, dst, _filter=None, ignore=None, force=True, recursive=True, _vars=None, dir_filter_function=None, file_filter_function=None, force_copy_symlinks=False): - if vars is not None: - src = src.format(**vars) - dst = dst.format(**vars) - if filter is not None: - for i in range(len(filter)): - filter[i] = filter[i].format(**vars) + if isinstance(src, str): + src = Path(src.format(**_vars)) if _vars else Path(src) + if isinstance(dst, str): + dst = Path(dst.format(**_vars)) if _vars else Path(dst) + assert (isinstance(src, Path)) + assert (isinstance(dst, Path)) + + if _vars is not None: + if _filter is not None: + _filter = [i.format(**_vars) for i in _filter] if ignore is not None: - for i in range(len(ignore)): - ignore[i] = ignore[i].format(**vars) + ignore = [i.format(**_vars) for i in ignore] - if not os.path.exists(src) and not force: - log.info(f"**Skipping copy tree {src} to {dst}. Source does not exists. " - f"filter={filter}. ignore={ignore}.") + if not src.exists() and not force: + log.info(f"**Skipping copy tree\n {src} to\n {dst}\n Source does not exist. " + f"filter={_filter}. ignore={ignore}.") return [] - log.info(f"Copying tree {src} to {dst}. filter={filter}. ignore={ignore}.") + log.debug(f"Copying tree\n {src} to\n {dst}. filter={_filter}. ignore={ignore}.") names = os.listdir(src) results = [] - errors = [] + copy_errors = [] for name in names: - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) + srcname = src / name + dstname = dst / name try: - if os.path.isdir(srcname): + if srcname.is_dir(): if (dir_filter_function and not dir_filter_function(name, src, srcname)): continue if recursive: - results.extend(copydir(srcname, dstname, filter, ignore, force, recursive, - vars, dir_filter_function, file_filter_function, + results.extend(copydir(srcname, dstname, _filter, ignore, force, recursive, + _vars, dir_filter_function, file_filter_function, force_copy_symlinks)) else: if ((file_filter_function is not None and not file_filter_function(name, srcname)) - or (filter is not None and not filter_match(name, filter)) + or (_filter is not None and not filter_match(name, _filter)) or (ignore is not None and filter_match(name, ignore))): continue - if not os.path.exists(dst): - os.makedirs(dst) - results.append(copyfile(srcname, dstname, True, vars, force_copy_symlinks)) + if not dst.is_dir(): + dst.mkdir(parents=True) + results.append(copyfile(srcname, dstname, True, _vars, force_copy_symlinks)) # catch the Error from the recursive copytree so that we can # continue with other files except shutil.Error as err: - errors.extend(err.args[0]) + copy_errors.extend(err.args[0]) except EnvironmentError as why: - errors.append((srcname, dstname, str(why))) + copy_errors.append((srcname, dstname, str(why))) try: - if os.path.exists(dst): - shutil.copystat(src, dst) + if dst.exists(): + shutil.copystat(str(src), str(dst)) except OSError as why: if WindowsError is not None and isinstance(why, WindowsError): # Copying file access times may fail on Windows pass else: - errors.extend((src, dst, str(why))) - if errors: - raise EnvironmentError(errors) + copy_errors.extend((src, dst, str(why))) + if copy_errors: + raise EnvironmentError(copy_errors) return results @@ -377,9 +234,10 @@ def make_file_writable_by_owner(path): os.chmod(path, current_permissions | stat.S_IWUSR) -def rmtree(dirname, ignore=False): +def remove_tree(dirname, ignore=False): def handle_remove_readonly(func, path, exc): - excvalue = exc[1] + # exc returns like 'sys.exc_info()': type, value, traceback + _, excvalue, _ = exc if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES: os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 func(path) @@ -391,13 +249,12 @@ def rmtree(dirname, ignore=False): def run_process_output(args, initial_env=None): if initial_env is None: initial_env = os.environ - std_out = subprocess.Popen(args, env=initial_env, universal_newlines=1, - stdout=subprocess.PIPE).stdout result = [] - for raw_line in std_out.readlines(): - line = raw_line - result.append(line.rstrip()) - std_out.close() + with subprocess.Popen(args, env=initial_env, universal_newlines=1, + stdout=subprocess.PIPE) as p: + for raw_line in p.stdout.readlines(): + result.append(raw_line.rstrip()) + p.stdout.close() return result @@ -407,7 +264,7 @@ def run_process(args, initial_env=None): No output is captured. """ command = " ".join([(" " in x and f'"{x}"' or x) for x in args]) - log.info(f"In directory {os.getcwd()}:\n\tRunning command: {command}") + log.debug(f"In directory {Path.cwd()}:\n\tRunning command: {command}") if initial_env is None: initial_env = os.environ @@ -419,80 +276,10 @@ def run_process(args, initial_env=None): return exit_code -def get_environment_from_batch_command(env_cmd, initial=None): - """ - Take a command (either a single command or list of arguments) - and return the environment created after running that command. - Note that if the command must be a batch file or .cmd file, or the - changes to the environment will not be captured. - - If initial is supplied, it is used as the initial environment passed - to the child process. - """ - - def validate_pair(ob): - try: - if not (len(ob) == 2): - log.error(f"Unexpected result: {ob}") - raise ValueError - except: - return False - return True - - def consume(iter): - try: - while True: - next(iter) - except StopIteration: - pass - - if not isinstance(env_cmd, (list, tuple)): - env_cmd = [env_cmd] - # construct the command that will alter the environment - env_cmd = subprocess.list2cmdline(env_cmd) - # create a tag so we can tell in the output when the proc is done - tag = 'Done running command' - # construct a cmd.exe command to do accomplish this - cmd = f'cmd.exe /E:ON /V:ON /s /c "{env_cmd} && echo "{tag}" && set"' - # launch the process - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=initial) - # parse the output sent to stdout - lines = proc.stdout - # make sure the lines are strings - lines = map(lambda s: s.decode(), lines) - # consume whatever output occurs until the tag is reached - consume(itertools.takewhile(lambda l: tag not in l, lines)) - # define a way to handle each KEY=VALUE line - # parse key/values into pairs - pairs = map(lambda l: l.rstrip().split('=', 1), lines) - # make sure the pairs are valid - valid_pairs = filter(validate_pair, pairs) - # construct a dictionary of the pairs - result = dict(valid_pairs) - # let the process finish - proc.communicate() - return result - - -def regenerate_qt_resources(src, pyside_rcc_path, pyside_rcc_options): - names = os.listdir(src) - for name in names: - srcname = os.path.join(src, name) - if os.path.isdir(srcname): - regenerate_qt_resources(srcname, pyside_rcc_path, pyside_rcc_options) - elif srcname.endswith('.qrc'): - # Replace last occurence of '.qrc' in srcname - srcname_split = srcname.rsplit('.qrc', 1) - dstname = '_rc.py'.join(srcname_split) - if os.path.exists(dstname): - log.info(f"Regenerating {dstname} from {os.path.basename(srcname)}") - run_process([pyside_rcc_path] + pyside_rcc_options + [srcname, '-o', dstname]) - - def back_tick(cmd, ret_err=False): """ - Run command `cmd`, return stdout, or stdout, stderr, - return_code if `ret_err` is True. + Run command `cmd`, return stdout, or (stdout, stderr, + return_code) if `ret_err` is True. Parameters ---------- @@ -516,22 +303,20 @@ def back_tick(cmd, ret_err=False): Raises RuntimeError if command returns non-zero exit code when ret_err isn't set. """ - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - out, err = proc.communicate() - if not isinstance(out, str): - # python 3 - out = out.decode() - err = err.decode() - retcode = proc.returncode - if retcode is None and not ret_err: - proc.terminate() - raise RuntimeError(f"{cmd} process did not terminate") - if retcode != 0 and not ret_err: - raise RuntimeError(f"{cmd} process returned code {retcode}\n*** {err}") - out = out.strip() + with subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) as proc: + out_bytes, err_bytes = proc.communicate() + out = out_bytes.decode().strip() + err = err_bytes.decode().strip() + retcode = proc.returncode + if retcode is None and not ret_err: + proc.terminate() + raise RuntimeError(f"{cmd} process did not terminate") + if retcode != 0 and not ret_err: + raise RuntimeError(f"{cmd} process returned code {retcode}\n*** {err}") if not ret_err: return out - return out, err.strip(), retcode + return out, err, retcode MACOS_OUTNAME_RE = re.compile(r'\(compatibility version [\d.]+, current version [\d.]+\)') @@ -595,7 +380,10 @@ def macos_get_rpaths(libpath): def macos_add_rpath(rpath, library_path): - back_tick(f"install_name_tool -add_rpath {rpath} {library_path}") + try: + back_tick(f"install_name_tool -add_rpath {rpath} {library_path}") + except RuntimeError as e: + print(f"Exception {type(e).__name__}: {e}") def macos_fix_rpaths_for_library(library_path, qt_lib_dir): @@ -642,8 +430,8 @@ def macos_fix_rpaths_for_library(library_path, qt_lib_dir): macos_add_qt_rpath(library_path, qt_lib_dir, existing_rpath_commands, install_names) -def macos_add_qt_rpath(library_path, qt_lib_dir, existing_rpath_commands=[], - library_dependencies=[]): +def macos_add_qt_rpath(library_path, qt_lib_dir, existing_rpath_commands=None, + library_dependencies=None): """ Adds an rpath load command to the Qt lib directory if necessary @@ -651,6 +439,12 @@ def macos_add_qt_rpath(library_path, qt_lib_dir, existing_rpath_commands=[], and adds an rpath load command that points to the Qt lib directory (qt_lib_dir). """ + if existing_rpath_commands is None: + existing_rpath_commands = [] + + if library_dependencies is None: + library_dependencies = [] + if not existing_rpath_commands: existing_rpath_commands = macos_get_rpaths(library_path) @@ -679,7 +473,7 @@ def find_glob_in_path(pattern): pattern += '.exe' for path in os.environ.get('PATH', '').split(os.pathsep): - for match in glob.glob(os.path.join(path, pattern)): + for match in glob.glob(str(Path(path) / pattern)): result.append(match) return result @@ -704,7 +498,7 @@ def detect_clang(): clang_dir = os.environ.get(source, None) if not clang_dir: raise OSError("clang not found") - return (clang_dir, source) + return (Path(clang_dir), source) _7z_binary = None @@ -730,8 +524,8 @@ def download_and_extract_7z(fileurl, target): outputDir = f"-o{target}" if not _7z_binary: if sys.platform == "win32": - candidate = "c:\\Program Files\\7-Zip\\7z.exe" - if os.path.exists(candidate): + candidate = Path("c:\\Program Files\\7-Zip\\7z.exe") + if candidate.exists(): _7z_binary = candidate if not _7z_binary: _7z_binary = '7z' @@ -820,7 +614,8 @@ def _ldd_ldd(executable_path): except Exception as e: error = str(e) if not output: - message = f"ldd failed to query for dependent shared libraries of {executable_path}: {error}" + message = (f"ldd failed to query for dependent shared libraries of {executable_path}: " + f"{error}") raise RuntimeError(message) return output @@ -860,7 +655,8 @@ def _ldd_ldso(executable_path): # Choose appropriate runtime dynamic linker. for rtld in rtld_list: - if os.path.isfile(rtld) and os.access(rtld, os.X_OK): + rtld = Path(rtld) + if rtld.is_file() and os.access(rtld, os.X_OK): (_, _, code) = back_tick(rtld, True) # Code 127 is returned by ld.so when called without any # arguments (some kind of sanity check I guess). @@ -906,7 +702,7 @@ def ldd(executable_path): result = _ldd_ldd(executable_path) except RuntimeError as e: message = f"ldd: Falling back to ld.so ({str(e)})" - log.warn(message) + log.warning(message) if not result: result = _ldd_ldso(executable_path) return result @@ -914,8 +710,8 @@ def ldd(executable_path): def find_files_using_glob(path, pattern): """ Returns list of files that matched glob `pattern` in `path`. """ - final_pattern = os.path.join(path, pattern) - maybe_files = glob.glob(final_pattern) + final_pattern = Path(path) / pattern + maybe_files = glob.glob(str(final_pattern)) return maybe_files @@ -932,16 +728,18 @@ def find_qt_core_library_glob(lib_dir): # ldd for the specified platforms. # This has less priority because ICU libs are not used in the default # Qt configuration build. +# Note: Uses ldd to query shared library dependencies and thus does not +# work for cross builds. def copy_icu_libs(patchelf, destination_lib_dir): """ Copy ICU libraries that QtCore depends on, to given `destination_lib_dir`. """ - qt_core_library_path = find_qt_core_library_glob(destination_lib_dir) + qt_core_library_path = Path(find_qt_core_library_glob(destination_lib_dir)) - if not qt_core_library_path or not os.path.exists(qt_core_library_path): + if not qt_core_library_path or not qt_core_library_path.exists(): raise RuntimeError(f"QtCore library does not exist at path: {qt_core_library_path}. " - "Failed to copy ICU libraries.") + "Failed to copy ICU libraries.") dependencies = ldd_get_dependencies(qt_core_library_path) @@ -958,14 +756,15 @@ def copy_icu_libs(patchelf, destination_lib_dir): paths = ldd_get_paths_for_dependencies(icu_regex, dependencies=dependencies) if not paths: raise RuntimeError("Failed to find the necessary ICU libraries required by QtCore.") - log.info('Copying the detected ICU libraries required by QtCore.') + log.debug('Copying the detected ICU libraries required by QtCore.') - if not os.path.exists(destination_lib_dir): - os.makedirs(destination_lib_dir) + destination_lib_dir = Path(destination_lib_dir) + if not destination_lib_dir.exists(): + destination_lib_dir.mkdir(parents=True) for path in paths: - basename = os.path.basename(path) - destination = os.path.join(destination_lib_dir, basename) + basename = Path(path).name + destination = destination_lib_dir / basename copyfile(path, destination, force_copy_symlink=True) # Patch the ICU libraries to contain the $ORIGIN rpath # value, so that only the local package libraries are used. @@ -990,7 +789,7 @@ def linux_run_read_elf(executable_path): def linux_set_rpaths(patchelf, executable_path, rpath_string): """ Patches the `executable_path` with a new rpath string. """ - cmd = [patchelf, '--set-rpath', rpath_string, executable_path] + cmd = [str(patchelf), '--set-rpath', str(rpath_string), str(executable_path)] if run_process(cmd) != 0: raise RuntimeError(f"Error patching rpath in {executable_path}") @@ -1003,7 +802,6 @@ def linux_prepend_rpath(patchelf, executable_path, new_path): rpaths.insert(0, new_path) new_rpaths_string = ":".join(rpaths) linux_set_rpaths(patchelf, executable_path, new_rpaths_string) - result = True def linux_patch_executable(patchelf, executable_path): @@ -1095,6 +893,7 @@ def linux_fix_rpaths_for_library(patchelf, executable_path, qt_rpath, override=F existing_rpaths = linux_get_rpaths(executable_path) rpaths.extend(existing_rpaths) + qt_rpath = str(qt_rpath) if linux_needs_qt_rpath(executable_path) and qt_rpath not in existing_rpaths: rpaths.append(qt_rpath) @@ -1133,26 +932,9 @@ def get_python_dict(python_script_path): raise -def install_pip_package_from_url_specifier(env_pip, url, upgrade=True): - args = [env_pip, "install", url] - if upgrade: - args.append("--upgrade") - args.append(url) - run_instruction(args, f"Failed to install {url}") - - -def install_pip_dependencies(env_pip, packages, upgrade=True): - for p in packages: - args = [env_pip, "install"] - if upgrade: - args.append("--upgrade") - args.append(p) - run_instruction(args, f"Failed to install {p}") - - def get_qtci_virtualEnv(python_ver, host, hostArch, targetArch): _pExe = "python" - _env = f"env{python_ver}" + _env = f"{os.environ.get('PYSIDE_VIRTUALENV') or 'env'+python_ver}" env_python = f"{_env}/bin/python" env_pip = f"{_env}/bin/pip" @@ -1164,28 +946,32 @@ def get_qtci_virtualEnv(python_ver, host, hostArch, targetArch): if python_ver.startswith("3"): var = f"PYTHON{python_ver}-32_PATH" log.info(f"Try to find python from {var} env variable") - _path = os.getenv(var, "") - _pExe = os.path.join(_path, "python.exe") - if not os.path.isfile(_pExe): - log.warn(f"Can't find python.exe from {_pExe}, using default python3") - _pExe = os.path.join(os.getenv("PYTHON3_32_PATH"), "python.exe") + _path = Path(os.getenv(var, "")) + _pExe = _path / "python.exe" + if not _pExe.is_file(): + log.warning(f"Can't find python.exe from {_pExe}, using default python3") + _pExe = Path(os.getenv("PYTHON3_32_PATH")) / "python.exe" else: - _pExe = os.path.join(os.getenv("PYTHON2_32_PATH"), "python.exe") + _pExe = Path(os.getenv("PYTHON2_32_PATH")) / "python.exe" else: if python_ver.startswith("3"): var = f"PYTHON{python_ver}-64_PATH" log.info(f"Try to find python from {var} env variable") - _path = os.getenv(var, "") - _pExe = os.path.join(_path, "python.exe") - if not os.path.isfile(_pExe): - log.warn(f"Can't find python.exe from {_pExe}, using default python3") - _pExe = os.path.join(os.getenv("PYTHON3_PATH"), "python.exe") + _path = Path(os.getenv(var, "")) + _pExe = _path / "python.exe" + if not _pExe.is_file(): + log.warning(f"Can't find python.exe from {_pExe}, using default python3") + _pExe = Path(os.getenv("PYTHON3_PATH")) / "python.exe" env_python = f"{_env}\\Scripts\\python.exe" env_pip = f"{_env}\\Scripts\\pip.exe" else: - if python_ver == "3": + _pExe = f"python{python_ver}" + try: + run_instruction([_pExe, "--version"], f"Failed to guess python version {_pExe}") + except Exception as e: + print(f"Exception {type(e).__name__}: {e}") _pExe = "python3" - return(_pExe, _env, env_pip, env_python) + return (_pExe, _env, env_pip, env_python) def run_instruction(instruction, error, initial_env=None): @@ -1198,21 +984,14 @@ def run_instruction(instruction, error, initial_env=None): exit(result) -def acceptCITestConfiguration(hostOS, hostOSVer, targetArch, compiler): - # Disable unsupported CI configs for now - # NOTE: String must match with QT CI's storagestruct thrift - if (hostOSVer in ["WinRT_10", "WebAssembly", "Ubuntu_18_04", "Android_ANY"] - or hostOSVer.startswith("SLES_")): - log.info("Disabled {hostOSVer} from Coin configuration") - return False - # With 5.11 CI will create two sets of release binaries, - # one with msvc 2015 and one with msvc 2017 - # we shouldn't release the 2015 version. - # BUT, 32 bit build is done only on msvc 2015... - if compiler in ["MSVC2015"] and targetArch in ["X86_64"]: - log.warn(f"Disabled {compiler} to {targetArch} from Coin configuration") - return False - return True +def get_ci_qtpaths_path(ci_install_dir, ci_host_os): + qtpaths_path = f"--qtpaths={ci_install_dir}" + if ci_host_os == "MacOS": + return f"{qtpaths_path}/bin/qtpaths" + elif ci_host_os == "Windows": + return f"{qtpaths_path}\\bin\\qtpaths.exe" + else: + return f"{qtpaths_path}/bin/qtpaths" def get_ci_qmake_path(ci_install_dir, ci_host_os): @@ -1223,3 +1002,162 @@ def get_ci_qmake_path(ci_install_dir, ci_host_os): return f"{qmake_path}\\bin\\qmake.exe" else: return f"{qmake_path}/bin/qmake" + + +def parse_cmake_conf_assignments_by_key(source_dir): + """ + Parses a .cmake.conf file that contains set(foo "bar") assignments + and returns a dict with those assignments transformed to keys and + values. + """ + + contents = (Path(source_dir) / ".cmake.conf").read_text() + matches = re.findall(r'set\((.+?) "(.*?)"\)', contents) + d = {key: value for key, value in matches} + return d + + +def _configure_failure_message(project_path, cmd, return_code, output, error, env): + """Format a verbose message about configure_cmake_project() failures.""" + cmd_string = ' '.join(cmd) + error_text = indent(error.strip(), " ") + output_text = indent(output.strip(), " ") + result = dedent(f""" + Failed to configure CMake project: '{project_path}' + Configure args were: + {cmd_string} + Return code: {return_code} + """) + + first = True + for k, v in env.items(): + if k.startswith("CMAKE"): + if first: + result += "Environment:\n" + first = False + result += f" {k}={v}\n" + + result += f"\nwith error:\n{error_text}\n" + + CMAKE_CMAKEOUTPUT_LOG_PATTERN = r'See also "([^"]+CMakeOutput\.log)"\.' + cmakeoutput_log_match = re.search(CMAKE_CMAKEOUTPUT_LOG_PATTERN, output) + if cmakeoutput_log_match: + cmakeoutput_log = Path(cmakeoutput_log_match.group(1)) + if cmakeoutput_log.is_file(): + log = indent(cmakeoutput_log.read_text().strip(), " ") + result += f"CMakeOutput.log:\n{log}\n" + + result += f"Output:\n{output_text}\n" + return result + + +def configure_cmake_project(project_path, + cmake_path, + build_path=None, + temp_prefix_build_path=None, + cmake_args=None, + cmake_cache_args=None, + ): + clean_temp_dir = False + if not build_path: + # Ensure parent dir exists. + if temp_prefix_build_path: + os.makedirs(temp_prefix_build_path, exist_ok=True) + + project_name = Path(project_path).name + build_path = tempfile.mkdtemp(prefix=f"{project_name}_", dir=temp_prefix_build_path) + + if 'QFP_SETUP_KEEP_TEMP_FILES' not in os.environ: + clean_temp_dir = True + + cmd = [cmake_path, '-G', 'Ninja', '-S', project_path, '-B', build_path] + + if cmake_args: + cmd.extend(cmake_args) + + for arg, value in cmake_cache_args: + cmd.extend([f'-D{arg}={value}']) + + cmd = [str(i) for i in cmd] + + proc = subprocess.run(cmd, shell=False, cwd=build_path, + capture_output=True, universal_newlines=True) + return_code = proc.returncode + output = proc.stdout + error = proc.stderr + + if return_code != 0: + m = _configure_failure_message(project_path, cmd, return_code, + output, error, os.environ) + raise RuntimeError(m) + + if clean_temp_dir: + remove_tree(build_path) + + return output + + +def parse_cmake_project_message_info(output): + # Parse the output for anything prefixed + # '-- qfp:<category>:<key>: <value>' as created by the message() + # calls in a given CMake project and store it in a python dict. + result = defaultdict(lambda: defaultdict(str)) + pattern = re.compile(r"^-- qfp:(.+?):(.+?):(.*)$") + for line in output.splitlines(): + found = pattern.search(line) + if found: + category = found.group(1).strip() + key = found.group(2).strip() + value = found.group(3).strip() + result[category][key] = str(value) + return result + + +def available_pyside_tools(qt_tools_path: Path, package_for_wheels: bool = False): + pyside_tools = PYSIDE_PYTHON_TOOLS.copy() + + if package_for_wheels: + # Qt wrappers in build/{python_env_name}/package_for_wheels/PySide6 + bin_path = qt_tools_path + else: + bin_path = qt_tools_path / "bin" + + def tool_exist(tool_path: Path): + if tool_path.exists(): + return True + else: + log.warning(f"{tool_path} not found. pyside-{tool_path.name} not included.") + return False + + if sys.platform == 'win32': + pyside_tools.extend([tool for tool in PYSIDE_WINDOWS_BIN_TOOLS + if tool_exist(bin_path / f"{tool}.exe")]) + else: + lib_exec_path = qt_tools_path / "Qt" / "libexec" if package_for_wheels \ + else qt_tools_path / "libexec" + pyside_tools.extend([tool for tool in PYSIDE_UNIX_LIBEXEC_TOOLS + if tool_exist(lib_exec_path / tool)]) + if sys.platform == 'darwin': + def name_to_path(name): + return f"{name.capitalize()}.app/Contents/MacOS/{name.capitalize()}" + + pyside_tools.extend([tool for tool in PYSIDE_UNIX_BIN_TOOLS + if tool_exist(bin_path / tool)]) + pyside_tools.extend([tool for tool in PYSIDE_UNIX_BUNDLED_TOOLS + if tool_exist(bin_path / name_to_path(tool))]) + else: + pyside_tools.extend([tool for tool in PYSIDE_LINUX_BIN_TOOLS + if tool_exist(bin_path / tool)]) + + return pyside_tools + + +def copy_qt_metatypes(destination_qt_dir, _vars): + """Copy the Qt metatypes files which changed location in 6.5""" + # <qt>/[lib]?/metatypes/* -> <setup>/{st_package_name}/Qt/[lib]?/metatypes + qt_meta_types_dir = "{qt_metatypes_dir}".format(**_vars) + qt_prefix_dir = "{qt_prefix_dir}".format(**_vars) + rel_meta_data_dir = os.fspath(Path(qt_meta_types_dir).relative_to(qt_prefix_dir)) + copydir(qt_meta_types_dir, destination_qt_dir / rel_meta_data_dir, + _filter=["*.json"], + recursive=False, _vars=_vars, force_copy_symlinks=True) |