diff options
Diffstat (limited to 'sources/pyside-tools/deploy_lib/dependency_util.py')
| -rw-r--r-- | sources/pyside-tools/deploy_lib/dependency_util.py | 319 |
1 files changed, 319 insertions, 0 deletions
diff --git a/sources/pyside-tools/deploy_lib/dependency_util.py b/sources/pyside-tools/deploy_lib/dependency_util.py new file mode 100644 index 000000000..2d5b188d3 --- /dev/null +++ b/sources/pyside-tools/deploy_lib/dependency_util.py @@ -0,0 +1,319 @@ +# Copyright (C) 2024 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import ast +import re +import os +import site +import json +import warnings +import logging +import shutil +import sys +from pathlib import Path +from typing import List, Set +from functools import lru_cache + +from . import IMPORT_WARNING_PYSIDE, run_command + + +@lru_cache(maxsize=None) +def get_py_files(project_dir: Path, extra_ignore_dirs: List[Path] = None, project_data=None): + """Finds and returns all the Python files in the project + """ + py_candidates = [] + ignore_dirs = ["__pycache__", "env", "venv", "deployment"] + + if project_data: + py_candidates = project_data.python_files + ui_candidates = project_data.ui_files + qrc_candidates = project_data.qrc_files + + def add_uic_qrc_candidates(candidates, candidate_type): + possible_py_candidates = [(file.parent / f"{candidate_type}_{file.stem}.py") + for file in candidates + if (file.parent / f"{candidate_type}_{file.stem}.py").exists() + ] + + if len(possible_py_candidates) != len(candidates): + warnings.warn(f"[DEPLOY] The number of {candidate_type} files and their " + "corresponding Python files don't match.", + category=RuntimeWarning) + + py_candidates.extend(possible_py_candidates) + + if ui_candidates: + add_uic_qrc_candidates(ui_candidates, "ui") + + if qrc_candidates: + add_uic_qrc_candidates(qrc_candidates, "qrc") + + return py_candidates + + # incase there is not .pyproject file, search all python files in project_dir, except + # ignore_dirs + if extra_ignore_dirs: + ignore_dirs.extend(extra_ignore_dirs) + + # find relevant .py files + _walk = os.walk(project_dir) + for root, dirs, files in _walk: + dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")] + for py_file in files: + if py_file.endswith(".py"): + py_candidates.append(Path(root) / py_file) + + return py_candidates + + +@lru_cache(maxsize=None) +def get_ast(py_file: Path): + """Given a Python file returns the abstract syntax tree + """ + contents = py_file.read_text(encoding="utf-8") + try: + tree = ast.parse(contents) + except SyntaxError: + print(f"[DEPLOY] Unable to parse {py_file}") + return tree + + +def find_permission_categories(project_dir: Path, extra_ignore_dirs: List[Path] = None, + project_data=None): + """Given the project directory, finds all the permission categories required by the + project. eg: Camera, Bluetooth, Contacts etc. + + Note: This function is only relevant for mac0S deployment. + """ + all_perm_categories = set() + mod_pattern = re.compile("Q(?P<mod_name>.*)Permission") + + def pyside_permission_imports(py_file: Path): + perm_categories = [] + try: + tree = get_ast(py_file) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + main_mod_name = node.module + if main_mod_name == "PySide6.QtCore": + # considers 'from PySide6.QtCore import QtMicrophonePermission' + for imported_module in node.names: + full_mod_name = imported_module.name + match = mod_pattern.search(full_mod_name) + if match: + mod_name = match.group("mod_name") + perm_categories.append(mod_name) + continue + + if isinstance(node, ast.Import): + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name == "PySide6": + logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file))) + except Exception as e: + raise RuntimeError(f"[DEPLOY] Finding permission categories failed on file " + f"{str(py_file)} with error {e}") + + return set(perm_categories) + + py_candidates = get_py_files(project_dir, extra_ignore_dirs, project_data) + for py_candidate in py_candidates: + all_perm_categories = all_perm_categories.union(pyside_permission_imports(py_candidate)) + + if not all_perm_categories: + ValueError("[DEPLOY] No permission categories were found for macOS app bundle creation.") + + return all_perm_categories + + +def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None, + project_data=None): + """ + Searches all the python files in the project to find all the PySide modules used by + the application. + """ + all_modules = set() + mod_pattern = re.compile("PySide6.Qt(?P<mod_name>.*)") + + def pyside_module_imports(py_file: Path): + modules = [] + try: + tree = get_ast(py_file) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + main_mod_name = node.module + if main_mod_name.startswith("PySide6"): + if main_mod_name == "PySide6": + # considers 'from PySide6 import QtCore' + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name.startswith("Qt"): + modules.append(full_mod_name[2:]) + continue + + # considers 'from PySide6.QtCore import Qt' + match = mod_pattern.search(main_mod_name) + if match: + mod_name = match.group("mod_name") + modules.append(mod_name) + else: + logging.warning(( + f"[DEPLOY] Unable to find module name from {ast.dump(node)}")) + + if isinstance(node, ast.Import): + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name == "PySide6": + logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file))) + except Exception as e: + raise RuntimeError(f"[DEPLOY] Finding module import failed on file {str(py_file)} with " + f"error {e}") + + return set(modules) + + py_candidates = get_py_files(project_dir, extra_ignore_dirs, project_data) + for py_candidate in py_candidates: + all_modules = all_modules.union(pyside_module_imports(py_candidate)) + + if not all_modules: + ValueError("[DEPLOY] No PySide6 modules were found") + + return list(all_modules) + + +class QtDependencyReader: + def __init__(self, dry_run: bool = False) -> None: + self.dry_run = dry_run + self.lib_reader_name = None + self.qt_module_path_pattern = None + self.lib_pattern = None + self.command = None + self.qt_libs_dir = None + + if sys.platform == "linux": + self.lib_reader_name = "readelf" + self.qt_module_path_pattern = "libQt6{module}.so.6" + self.lib_pattern = re.compile("libQt6(?P<mod_name>.*).so.6") + self.command_args = "-d" + elif sys.platform == "darwin": + self.lib_reader_name = "dyld_info" + self.qt_module_path_pattern = "Qt{module}.framework/Versions/A/Qt{module}" + self.lib_pattern = re.compile("@rpath/Qt(?P<mod_name>.*).framework/Versions/A/") + self.command_args = "-dependents" + elif sys.platform == "win32": + self.lib_reader_name = "dumpbin" + self.qt_module_path_pattern = "Qt6{module}.dll" + self.lib_pattern = re.compile("Qt6(?P<mod_name>.*).dll") + self.command_args = "/dependents" + else: + print(f"[DEPLOY] Deployment on unsupported platfrom {sys.platform}") + sys.exit(1) + + self.pyside_install_dir = None + self.qt_libs_dir = self.get_qt_libs_dir() + self._lib_reader = shutil.which(self.lib_reader_name) + + def get_qt_libs_dir(self): + """ + Finds the path to the Qt libs directory inside PySide6 package installation + """ + for possible_site_package in site.getsitepackages(): + if possible_site_package.endswith("site-packages"): + self.pyside_install_dir = Path(possible_site_package) / "PySide6" + + if not self.pyside_install_dir: + print("Unable to find site-packages. Exiting ...") + sys.exit(-1) + + if sys.platform == "win32": + return self.pyside_install_dir + + return self.pyside_install_dir / "Qt" / "lib" # for linux and macOS + + @property + def lib_reader(self): + return self._lib_reader + + def find_dependencies(self, module: str, used_modules: Set[str] = None): + """ + Given a Qt module, find all the other Qt modules it is dependent on and add it to the + 'used_modules' set + """ + qt_module_path = self.qt_libs_dir / self.qt_module_path_pattern.format(module=module) + if not qt_module_path.exists(): + warnings.warn(f"[DEPLOY] {qt_module_path.name} not found in {str(qt_module_path)}." + "Skipping finding its dependencies.", category=RuntimeWarning) + return + + lib_pattern = re.compile(self.lib_pattern) + command = [self.lib_reader, self.command_args, str(qt_module_path)] + # print the command if dry_run is True. + # Normally run_command is going to print the command in dry_run mode. But, this is a + # special case where we need to print the command as well as to run it. + if self.dry_run: + command_str = " ".join(command) + print(command_str + "\n") + + # We need to run this even for dry run, to see the full Nuitka command being executed + _, output = run_command(command=command, dry_run=False, fetch_output=True) + + dependent_modules = set() + for line in output.splitlines(): + line = line.decode("utf-8").lstrip() + if sys.platform == "darwin": + if line.endswith(f"Qt{module} [arm64]:"): + # macOS Qt frameworks bundles have both x86_64 and arm64 architectures + # We only need to consider one as the dependencies are redundant + break + elif line.endswith(f"Qt{module} [X86_64]:"): + # this line needs to be skipped because it matches with the pattern + # and is related to the module itself, not the dependencies of the module + continue + elif sys.platform == "win32" and line.startswith("Summary"): + # the dependencies would be found before the `Summary` line + break + match = lib_pattern.search(line) + if match: + dep_module = match.group("mod_name") + dependent_modules.add(dep_module) + if dep_module not in used_modules: + used_modules.add(dep_module) + self.find_dependencies(module=dep_module, used_modules=used_modules) + + if dependent_modules: + logging.info(f"[DEPLOY] Following dependencies found for {module}: {dependent_modules}") + else: + logging.info(f"[DEPLOY] No Qt dependencies found for {module}") + + def find_plugin_dependencies(self, used_modules: List[str], python_exe: Path) -> List[str]: + """ + Given the modules used by the application, returns all the required plugins + """ + plugins = set() + pyside_wheels = ["PySide6_Essentials", "PySide6_Addons"] + # TODO from 3.12 use list(dist.name for dist in importlib.metadata.distributions()) + _, installed_packages = run_command(command=[str(python_exe), "-m", "pip", "freeze"], + dry_run=False, fetch_output=True) + installed_packages = [p.decode().split('==')[0] for p in installed_packages.split()] + for pyside_wheel in pyside_wheels: + if pyside_wheel not in installed_packages: + # the wheel is not installed and hence no plugins are checked for its modules + logging.warning((f"[DEPLOY] The package {pyside_wheel} is not installed. ")) + continue + pyside_mod_plugin_json_name = f"{pyside_wheel}.json" + pyside_mod_plugin_json_file = self.pyside_install_dir / pyside_mod_plugin_json_name + if not pyside_mod_plugin_json_file.exists(): + warnings.warn(f"[DEPLOY] Unable to find {pyside_mod_plugin_json_file}.", + category=RuntimeWarning) + continue + + # convert the json to dict + pyside_mod_dict = {} + with open(pyside_mod_plugin_json_file) as pyside_json: + pyside_mod_dict = json.load(pyside_json) + + # find all the plugins in the modules + for module in used_modules: + plugins.update(pyside_mod_dict.get(module, [])) + + return list(plugins) |