From 7526d9c4aa884a9d03700a76751158fd9c8bfece Mon Sep 17 00:00:00 2001 From: Shyamnath Premnadh Date: Wed, 31 Jan 2024 16:33:05 +0100 Subject: Deployment: Find dependent modules - Based on the desktop platform, find all the Qt module dependencies of the application just like Android. These dependencies can help in optimizing the plugins packaged with the application. - Desktop deployment has new cl arguments: --extra-ignore-dirs and --extra-modules that further complements finding the Qt modules used by the application. - Since the Qt dependencies are also required for desktop deployment, 'modules' field in pysidedeploy.spec is moved from under 'buildozer' key to 'qt' key. - dependency finding code moved to dependency_util.py. This also helps in list the imports without conflicts in deploy_lib/__init__.py. - Fix tests. Skip the deploy tests for macOS 11 as the CI does not include dyld_info either via XCode or CommandLineTools. Task-number: PYSIDE-1612 Change-Id: I3524e1996bfec76c5635d1b35ccbc4ecd6ba7b8d Reviewed-by: Adrian Herrmann --- sources/pyside-tools/android_deploy.py | 24 +-- sources/pyside-tools/android_deploy.pyproject | 2 +- sources/pyside-tools/deploy.py | 32 ++- sources/pyside-tools/deploy.pyproject | 3 +- sources/pyside-tools/deploy_lib/__init__.py | 28 ++- .../deploy_lib/android/android_config.py | 26 +-- sources/pyside-tools/deploy_lib/config.py | 79 +++++++- sources/pyside-tools/deploy_lib/default.spec | 6 +- sources/pyside-tools/deploy_lib/dependency_util.py | 218 +++++++++++++++++++++ sources/pyside-tools/deploy_lib/deploy_util.py | 109 +---------- 10 files changed, 355 insertions(+), 172 deletions(-) create mode 100644 sources/pyside-tools/deploy_lib/dependency_util.py (limited to 'sources/pyside-tools') diff --git a/sources/pyside-tools/android_deploy.py b/sources/pyside-tools/android_deploy.py index 362fb3766..75269d622 100644 --- a/sources/pyside-tools/android_deploy.py +++ b/sources/pyside-tools/android_deploy.py @@ -9,7 +9,7 @@ from pathlib import Path from textwrap import dedent from deploy_lib import (create_config_file, cleanup, config_option_exists, PythonExecutable, - MAJOR_VERSION) + MAJOR_VERSION, HELP_EXTRA_IGNORE_DIRS, HELP_EXTRA_MODULES) from deploy_lib.android import AndroidData, AndroidConfig from deploy_lib.android.buildozer import Buildozer @@ -46,26 +46,6 @@ from deploy_lib.android.buildozer import Buildozer Note: This file is used by both pyside6-deploy and pyside6-android-deploy """ -HELP_EXTRA_IGNORE_DIRS = dedent(""" - Comma separated directory names inside the project dir. These - directories will be skipped when searching for python files - relevant to the project. - - Example usage: --extra-ignore-dirs=doc,translations - """) - -HELP_EXTRA_MODULES = dedent(""" - Comma separated list of Qt modules to be added to the application, - in case they are not found automatically. - - This occurs when you have 'import PySide6' in your code instead - 'from PySide6 import '. The module name is specified - with either omitting the prefix of Qt or with it. - - Example usage 1: --extra-modules=Network,Svg - Example usage 2: --extra-modules=QtNetwork,QtSvg - """) - def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = None, ndk_path: Path = None, sdk_path: Path = None, config_file: Path = None, init: bool = False, @@ -124,7 +104,7 @@ def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = Non config.title = name try: - config.modules += extra_modules + config.modules += list(set(extra_modules).difference(set(config.modules))) # this cannot be done when config file is initialized because cleanup() removes it # so this can only be done after the cleanup() diff --git a/sources/pyside-tools/android_deploy.pyproject b/sources/pyside-tools/android_deploy.pyproject index f976cb5a6..bc6347243 100644 --- a/sources/pyside-tools/android_deploy.pyproject +++ b/sources/pyside-tools/android_deploy.pyproject @@ -4,6 +4,6 @@ "deploy_lib/android/recipes/PySide6/__init__.tmpl.py", "deploy_lib/android/recipes/shiboken6/__init__.tmpl.py", "deploy_lib/android/__init__.py", "deploy_lib/android/android_helper.py", - "deploy_lib/android/buildozer.py" + "deploy_lib/android/buildozer.py", "deploy_lib/dependency_util.py" ] } diff --git a/sources/pyside-tools/deploy.py b/sources/pyside-tools/deploy.py index 576c01f9d..6cb6d4d9c 100644 --- a/sources/pyside-tools/deploy.py +++ b/sources/pyside-tools/deploy.py @@ -34,13 +34,14 @@ import traceback from pathlib import Path from textwrap import dedent -from deploy_lib import (MAJOR_VERSION, Config, cleanup, config_option_exists, - finalize, create_config_file, PythonExecutable, Nuitka) +from deploy_lib import (MAJOR_VERSION, DesktopConfig, cleanup, config_option_exists, + finalize, create_config_file, PythonExecutable, Nuitka, + HELP_EXTRA_MODULES, HELP_EXTRA_IGNORE_DIRS) def main(main_file: Path = None, name: str = None, config_file: Path = None, init: bool = False, loglevel=logging.WARNING, dry_run: bool = False, keep_deployment_files: bool = False, - force: bool = False): + force: bool = False, extra_ignore_dirs: str = None, extra_modules_grouped: str = None): logging.basicConfig(level=loglevel) if config_file and not config_file.exists() and not main_file.exists(): @@ -56,6 +57,18 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini config = None logging.info("[DEPLOY] Start") + if extra_ignore_dirs: + extra_ignore_dirs = extra_ignore_dirs.split(",") + + extra_modules = [] + if extra_modules_grouped: + tmp_extra_modules = extra_modules_grouped.split(",") + for extra_module in tmp_extra_modules: + if extra_module.startswith("Qt"): + extra_modules.append(extra_module[2:]) + else: + extra_modules.append(extra_module) + python = PythonExecutable(dry_run=dry_run, init=init, force=force) config_file_exists = config_file and Path(config_file).exists() @@ -65,8 +78,9 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini config_file = create_config_file(dry_run=dry_run, config_file=config_file, main_file=main_file) - config = Config(config_file=config_file, source_file=main_file, python_exe=python.exe, - dry_run=dry_run, existing_config_file=config_file_exists) + config = DesktopConfig(config_file=config_file, source_file=main_file, python_exe=python.exe, + dry_run=dry_run, existing_config_file=config_file_exists, + extra_ignore_dirs=extra_ignore_dirs) # set application name if name: @@ -81,6 +95,8 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini if python.is_pyenv_python() and add_arg not in config.extra_args: config.extra_args += add_arg + config.modules += list(set(extra_modules).difference(set(config.modules))) + # writing config file # in the case of --dry-run, we use default.spec as reference. Do not save the changes # for --dry-run @@ -153,7 +169,11 @@ if __name__ == "__main__": parser.add_argument("--name", type=str, help="Application name") + parser.add_argument("--extra-ignore-dirs", type=str, help=HELP_EXTRA_IGNORE_DIRS) + + parser.add_argument("--extra-modules", type=str, help=HELP_EXTRA_MODULES) + args = parser.parse_args() main(args.main_file, args.name, args.config_file, args.init, args.loglevel, args.dry_run, - args.keep_deployment_files, args.force) + args.keep_deployment_files, args.force, args.extra_ignore_dirs, args.extra_modules) diff --git a/sources/pyside-tools/deploy.pyproject b/sources/pyside-tools/deploy.pyproject index eef1668c5..0e6ca8251 100644 --- a/sources/pyside-tools/deploy.pyproject +++ b/sources/pyside-tools/deploy.pyproject @@ -2,6 +2,7 @@ "files": ["deploy.py", "deploy_lib/__init__.py", "deploy_lib/commands.py", "deploy_lib/config.py", "deploy_lib/default.spec", "deploy_lib/nuitka_helper.py", "deploy_lib/pyside_icon.ico", "deploy_lib/pyside_icon.icns","deploy_lib/pyside_icon.jpg", - "deploy_lib/python_helper.py", "deploy_lib/deploy_util.py" + "deploy_lib/python_helper.py", "deploy_lib/deploy_util.py", + "deploy_lib/dependency_util.py" ] } diff --git a/sources/pyside-tools/deploy_lib/__init__.py b/sources/pyside-tools/deploy_lib/__init__.py index 27d178eee..b3dedd40c 100644 --- a/sources/pyside-tools/deploy_lib/__init__.py +++ b/sources/pyside-tools/deploy_lib/__init__.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import sys from pathlib import Path +from textwrap import dedent MAJOR_VERSION = 6 @@ -19,21 +20,40 @@ DEFAULT_APP_ICON = str((Path(__file__).parent / f"pyside_icon{IMAGE_FORMAT}").re IMPORT_WARNING_PYSIDE = (f"[DEPLOY] Found 'import PySide6' in file {0}" ". Use 'from PySide6 import ' or pass the module" " needed using --extra-modules command line argument") +HELP_EXTRA_IGNORE_DIRS = dedent(""" + Comma-separated directory names inside the project dir. These + directories will be skipped when searching for Python files + relevant to the project. + + Example usage: --extra-ignore-dirs=doc,translations + """) + +HELP_EXTRA_MODULES = dedent(""" + Comma-separated list of Qt modules to be added to the application, + in case they are not found automatically. + + This occurs when you have 'import PySide6' in your code instead + 'from PySide6 import '. The module name is specified + by either omitting the prefix of Qt or including it. + + Example usage 1: --extra-modules=Network,Svg + Example usage 2: --extra-modules=QtNetwork,QtSvg + """) def get_all_pyside_modules(): """ Returns all the modules installed with PySide6 """ + import PySide6 # They all start with `Qt` as the prefix. Removing this prefix and getting the actual # module name - import PySide6 return [module[2:] for module in PySide6.__all__] from .commands import run_command, run_qmlimportscanner +from .dependency_util import find_pyside_modules, QtDependencyReader from .nuitka_helper import Nuitka -from .config import BaseConfig, Config +from .config import BaseConfig, Config, DesktopConfig from .python_helper import PythonExecutable -from .deploy_util import (cleanup, finalize, create_config_file, - config_option_exists, find_pyside_modules) +from .deploy_util import cleanup, finalize, create_config_file, config_option_exists diff --git a/sources/pyside-tools/deploy_lib/android/android_config.py b/sources/pyside-tools/deploy_lib/android/android_config.py index 8054ce373..417656bb0 100644 --- a/sources/pyside-tools/deploy_lib/android/android_config.py +++ b/sources/pyside-tools/deploy_lib/android/android_config.py @@ -12,8 +12,7 @@ from pkginfo import Wheel from . import (extract_and_copy_jar, get_wheel_android_arch, find_lib_dependencies, get_llvm_readobj, find_qtlibs_in_wheel, platform_map, create_recipe) -from .. import (Config, find_pyside_modules, run_qmlimportscanner, get_all_pyside_modules, - MAJOR_VERSION) +from .. import (Config, find_pyside_modules, get_all_pyside_modules, MAJOR_VERSION) ANDROID_NDK_VERSION = "25c" ANDROID_DEPLOY_CACHE = Path.home() / ".pyside6_android_deploy" @@ -107,9 +106,8 @@ class AndroidConfig(Config): self.qt_libs_path: zipfile.Path = find_qtlibs_in_wheel(wheel_pyside=self.wheel_pyside) logging.info(f"[DEPLOY] Qt libs path inside wheel: {str(self.qt_libs_path)}") - self._modules = [] - if self.get_value("buildozer", "modules"): - self.modules = self.get_value("buildozer", "modules").split(",") + if self.get_value("qt", "modules"): + self.modules = self.get_value("qt", "modules").split(",") else: self._find_and_set_pysidemodules() self._find_and_set_qtquick_modules() @@ -190,7 +188,7 @@ class AndroidConfig(Config): @modules.setter def modules(self, modules): self._modules = modules - self.set_value("buildozer", "modules", ",".join(modules)) + self.set_value("qt", "modules", ",".join(modules)) @property def local_libs(self): @@ -282,22 +280,6 @@ class AndroidConfig(Config): raise RuntimeError("[DEPLOY] PySide wheel corrupted. Wheel name should end with" "platform name") - def _find_and_set_qtquick_modules(self): - """Identify if QtQuick is used in QML files and add them as dependency - """ - extra_modules = [] - if not self.qml_modules: - self.qml_modules = set(run_qmlimportscanner(qml_files=self.qml_files, - dry_run=self.dry_run)) - - if "QtQuick" in self.qml_modules: - extra_modules.append("Quick") - - if "QtQuick.Controls" in self.qml_modules: - extra_modules.append("QuickControls2") - - self.modules += extra_modules - def _find_dependent_qt_modules(self): """ Given pysidedeploy_config.modules, find all the other dependent Qt modules. This is diff --git a/sources/pyside-tools/deploy_lib/config.py b/sources/pyside-tools/deploy_lib/config.py index 61b2ebec1..44b4ded06 100644 --- a/sources/pyside-tools/deploy_lib/config.py +++ b/sources/pyside-tools/deploy_lib/config.py @@ -1,15 +1,14 @@ # Copyright (C) 2022 The Qt Company Ltd. # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only - import configparser import logging import warnings from configparser import ConfigParser +from typing import List from pathlib import Path from project import ProjectData -from .commands import run_qmlimportscanner -from . import DEFAULT_APP_ICON +from . import DEFAULT_APP_ICON, find_pyside_modules, run_qmlimportscanner, QtDependencyReader # Some QML plugins like QtCore are excluded from this list as they don't contribute much to # executable size. Excluding them saves the extra processing of checking for them in files @@ -17,7 +16,8 @@ EXCLUDED_QML_PLUGINS = {"QtQuick", "QtQuick3D", "QtCharts", "QtWebEngine", "QtTe class BaseConfig: - + """Wrapper class around any .spec file with function to read and set values for the .spec file + """ def __init__(self, config_file: Path, comment_prefixes: str = "/", existing_config_file: bool = False) -> None: self.config_file = config_file @@ -60,9 +60,10 @@ class Config(BaseConfig): """ def __init__(self, config_file: Path, source_file: Path, python_exe: Path, dry_run: bool, - existing_config_file: bool = False): + existing_config_file: bool = False, extra_ignore_dirs: List[str] = None): super().__init__(config_file=config_file, existing_config_file=existing_config_file) + self.extra_ignore_dirs = extra_ignore_dirs self._dry_run = dry_run self.qml_modules = set() # set source_file @@ -122,6 +123,8 @@ class Config(BaseConfig): self._generated_files_path = self.project_dir / "deployment" + self.modules = [] + def set_or_fetch(self, config_property_val, config_property_key, config_property_group="app"): """ Write to config_file if 'config_property_key' is known without config_file @@ -221,6 +224,15 @@ class Config(BaseConfig): def exe_dir(self, exe_dir: Path): self._exe_dir = exe_dir + @property + def modules(self): + return self._modules + + @modules.setter + def modules(self, modules): + self._modules = modules + self.set_value("qt", "modules", ",".join(modules)) + def _find_and_set_qml_files(self): """Fetches all the qml_files in the folder and sets them if the field qml_files is empty in the config_dir""" @@ -321,3 +333,60 @@ class Config(BaseConfig): config_property_val=self.exe_dir, config_property_key="exec_directory" ) ).absolute() + + def _find_and_set_pysidemodules(self): + self.modules = find_pyside_modules(project_dir=self.project_dir, + extra_ignore_dirs=self.extra_ignore_dirs, + project_data=self.project_data) + logging.info("The following PySide modules were found from the Python files of " + f"the project {self.modules}") + + def _find_and_set_qtquick_modules(self): + """Identify if QtQuick is used in QML files and add them as dependency + """ + extra_modules = [] + if not self.qml_modules: + self.qml_modules = set(run_qmlimportscanner(qml_files=self.qml_files, + dry_run=self.dry_run)) + + if "QtQuick" in self.qml_modules: + extra_modules.append("Quick") + + if "QtQuick.Controls" in self.qml_modules: + extra_modules.append("QuickControls2") + + self.modules += extra_modules + + +class DesktopConfig(Config): + """Wrapper class around pysidedeploy.spec, but specific to Desktop deployment + """ + def __init__(self, config_file: Path, source_file: Path, python_exe: Path, dry_run: bool, + existing_config_file: bool = False, extra_ignore_dirs: List[str] = None): + super().__init__(config_file, source_file, python_exe, dry_run, existing_config_file, + extra_ignore_dirs) + + if self.get_value("qt", "modules"): + self.modules = self.get_value("qt", "modules").split(",") + else: + self._find_and_set_pysidemodules() + self._find_and_set_qtquick_modules() + self._find_dependent_qt_modules() + + def _find_dependent_qt_modules(self): + """ + Given pysidedeploy_config.modules, find all the other dependent Qt modules. + """ + dependency_reader = QtDependencyReader(dry_run=self.dry_run) + all_modules = set(self.modules) + + if not dependency_reader.lib_reader: + warnings.warn(f"[DEPLOY] Unable to find {dependency_reader.lib_reader_name}. This tool" + " helps to find the Qt module dependencies of the application. Skipping " + " checking for dependencies.", category=RuntimeWarning) + return + + for module_name in self.modules: + dependency_reader.find_dependencies(module=module_name, used_modules=all_modules) + + self.modules = list(all_modules) diff --git a/sources/pyside-tools/deploy_lib/default.spec b/sources/pyside-tools/deploy_lib/default.spec index ffbaa5532..2276fa496 100644 --- a/sources/pyside-tools/deploy_lib/default.spec +++ b/sources/pyside-tools/deploy_lib/default.spec @@ -41,6 +41,9 @@ qml_files = # excluded qml plugin binaries excluded_qml_plugins = +# Qt modules used. Comma separated +modules = + [android] # path to PySide wheel @@ -77,9 +80,6 @@ ndk_path = # if empty uses default sdk path downloaded by buildozer sdk_path = -# modules used. Comma separated -modules = - # other libraries to be loaded. Comma separated. # loaded at app startup local_libs = diff --git a/sources/pyside-tools/deploy_lib/dependency_util.py b/sources/pyside-tools/deploy_lib/dependency_util.py new file mode 100644 index 000000000..c7821794f --- /dev/null +++ b/sources/pyside-tools/deploy_lib/dependency_util.py @@ -0,0 +1,218 @@ +# Copyright (C) 2024 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import ast +import re +import os +import site +import warnings +import logging +import shutil +import sys +from pathlib import Path +from typing import List, Set + +from . import IMPORT_WARNING_PYSIDE, run_command + + +def get_qt_libs_dir(): + """ + Finds the path to the Qt libs directory inside PySide6 package installation + """ + pyside_install_dir = None + for possible_site_package in site.getsitepackages(): + if possible_site_package.endswith("site-packages"): + pyside_install_dir = Path(possible_site_package) / "PySide6" + + if not pyside_install_dir: + print("Unable to find site-packages. Exiting ...") + sys.exit(-1) + + if sys.platform == "win32": + return pyside_install_dir + + return pyside_install_dir / "Qt" / "lib" # for linux and macOS + + +def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None, + project_data=None): + """ + Searches all the python files in the project to find all the PySide modules used by + the application. + """ + all_modules = set() + mod_pattern = re.compile("PySide6.Qt(?P.*)") + + def pyside_imports(py_file: Path): + modules = [] + contents = py_file.read_text(encoding="utf-8") + try: + tree = ast.parse(contents) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + main_mod_name = node.module + if main_mod_name.startswith("PySide6"): + if main_mod_name == "PySide6": + # considers 'from PySide6 import QtCore' + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name.startswith("Qt"): + modules.append(full_mod_name[2:]) + continue + + # considers 'from PySide6.QtCore import Qt' + match = mod_pattern.search(main_mod_name) + if match: + mod_name = match.group("mod_name") + modules.append(mod_name) + else: + logging.warning(( + f"[DEPLOY] Unable to find module name from {ast.dump(node)}")) + + if isinstance(node, ast.Import): + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name == "PySide6": + logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file))) + except Exception as e: + raise RuntimeError(f"[DEPLOY] Finding module import failed on file {str(py_file)} with " + f"error {e}") + + return set(modules) + + py_candidates = [] + ignore_dirs = ["__pycache__", "env", "venv", "deployment"] + + if project_data: + py_candidates = project_data.python_files + ui_candidates = project_data.ui_files + qrc_candidates = project_data.qrc_files + ui_py_candidates = None + qrc_ui_candidates = None + + if ui_candidates: + ui_py_candidates = [(file.parent / f"ui_{file.stem}.py") for file in ui_candidates + if (file.parent / f"ui_{file.stem}.py").exists()] + + if len(ui_py_candidates) != len(ui_candidates): + warnings.warn("[DEPLOY] The number of uic files and their corresponding Python" + " files don't match.", category=RuntimeWarning) + + py_candidates.extend(ui_py_candidates) + + if qrc_candidates: + qrc_ui_candidates = [(file.parent / f"rc_{file.stem}.py") for file in qrc_candidates + if (file.parent / f"rc_{file.stem}.py").exists()] + + if len(qrc_ui_candidates) != len(qrc_candidates): + warnings.warn("[DEPLOY] The number of qrc files and their corresponding Python" + " files don't match.", category=RuntimeWarning) + + py_candidates.extend(qrc_ui_candidates) + + for py_candidate in py_candidates: + all_modules = all_modules.union(pyside_imports(py_candidate)) + return list(all_modules) + + # incase there is not .pyproject file, search all python files in project_dir, except + # ignore_dirs + if extra_ignore_dirs: + ignore_dirs.extend(extra_ignore_dirs) + + # find relevant .py files + _walk = os.walk(project_dir) + for root, dirs, files in _walk: + dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")] + for py_file in files: + if py_file.endswith(".py"): + py_candidates.append(Path(root) / py_file) + + for py_candidate in py_candidates: + all_modules = all_modules.union(pyside_imports(py_candidate)) + + if not all_modules: + ValueError("[DEPLOY] No PySide6 modules were found") + + return list(all_modules) + + +class QtDependencyReader: + def __init__(self, dry_run: bool = False) -> None: + self.dry_run = dry_run + self.lib_reader_name = None + self.qt_module_path_pattern = None + self.lib_pattern = None + self.command = None + self.qt_libs_dir = None + + if sys.platform == "linux": + self.lib_reader_name = "readelf" + self.qt_module_path_pattern = "libQt6{module}.so.6" + self.lib_pattern = re.compile("libQt6(?P.*).so.6") + self.command_args = "-d" + elif sys.platform == "darwin": + self.lib_reader_name = "dyld_info" + self.qt_module_path_pattern = "Qt{module}.framework/Versions/A/Qt{module}" + self.lib_pattern = re.compile("@rpath/Qt(?P.*).framework/Versions/A/") + self.command_args = "-dependents" + elif sys.platform == "win32": + self.lib_reader_name = "dumpbin" + self.qt_module_path_pattern = "Qt6{module}.dll" + self.lib_pattern = re.compile("Qt6(?P.*).dll") + self.command_args = "/dependents" + else: + print(f"[DEPLOY] Deployment on unsupported platfrom {sys.platform}") + sys.exit(1) + + self.qt_libs_dir = get_qt_libs_dir() + self._lib_reader = shutil.which(self.lib_reader_name) + + @property + def lib_reader(self): + return self._lib_reader + + def find_dependencies(self, module: str, used_modules: Set[str] = None): + """ + Given a Qt module, find all the other Qt modules it is dependent on and add it to the + 'used_modules' set + """ + qt_module_path = self.qt_libs_dir / self.qt_module_path_pattern.format(module=module) + if not qt_module_path.exists(): + warnings.warn(f"[DEPLOY] {qt_module_path.name} not found in {str(qt_module_path)}." + "Skipping finding its dependencies.", category=RuntimeWarning) + return + + lib_pattern = re.compile(self.lib_pattern) + command = [self.lib_reader, self.command_args, str(qt_module_path)] + # print the command if dry_run is True. + # Normally run_command is going to print the command in dry_run mode. But, this is a + # special case where we need to print the command as well as to run it. + if self.dry_run: + command_str = " ".join(command) + print(command_str + "\n") + + # We need to run this even for dry run, to see the full Nuitka command being executed + _, output = run_command(command=command, dry_run=False, fetch_output=True) + + dependent_modules = set() + for line in output.splitlines(): + line = line.decode("utf-8").lstrip() + if sys.platform == "darwin" and line.startswith(f"Qt{module} [arm64]"): + # macOS Qt frameworks bundles have both x86_64 and arm64 architectures + # We only need to consider one as the dependencies are redundant + break + elif sys.platform == "win32" and line.startswith("Summary"): + # the dependencies would be found before the `Summary` line + break + match = lib_pattern.search(line) + if match: + dep_module = match.group("mod_name") + dependent_modules.add(dep_module) + if dep_module not in used_modules: + used_modules.add(dep_module) + self.find_dependencies(module=dep_module, used_modules=used_modules) + + if dependent_modules: + logging.info(f"[DEPLOY] Following dependencies found for {module}: {dependent_modules}") + else: + logging.info(f"[DEPLOY] No Qt dependencies found for {module}") diff --git a/sources/pyside-tools/deploy_lib/deploy_util.py b/sources/pyside-tools/deploy_lib/deploy_util.py index b20c9c8cb..274b41905 100644 --- a/sources/pyside-tools/deploy_lib/deploy_util.py +++ b/sources/pyside-tools/deploy_lib/deploy_util.py @@ -1,17 +1,12 @@ # Copyright (C) 2023 The Qt Company Ltd. # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -import ast -import re -import os -import warnings import logging import shutil import sys from pathlib import Path -from typing import List -from . import EXE_FORMAT, IMPORT_WARNING_PYSIDE +from . import EXE_FORMAT from .config import Config @@ -76,105 +71,3 @@ def finalize(config: Config): shutil.copy(generated_exec_path, config.exe_dir) print("[DEPLOY] Executed file created in " f"{str(config.exe_dir / (config.source_file.stem + EXE_FORMAT))}") - - -def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None, - project_data=None): - """ - Searches all the python files in the project to find all the PySide modules used by - the application. - """ - all_modules = set() - mod_pattern = re.compile("PySide6.Qt(?P.*)") - - def pyside_imports(py_file: Path): - modules = [] - contents = py_file.read_text(encoding="utf-8") - try: - tree = ast.parse(contents) - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom): - main_mod_name = node.module - if main_mod_name.startswith("PySide6"): - if main_mod_name == "PySide6": - # considers 'from PySide6 import QtCore' - for imported_module in node.names: - full_mod_name = imported_module.name - if full_mod_name.startswith("Qt"): - modules.append(full_mod_name[2:]) - continue - - # considers 'from PySide6.QtCore import Qt' - match = mod_pattern.search(main_mod_name) - if match: - mod_name = match.group("mod_name") - modules.append(mod_name) - else: - logging.warning(( - f"[DEPLOY] Unable to find module name from{ast.dump(node)}")) - - if isinstance(node, ast.Import): - for imported_module in node.names: - full_mod_name = imported_module.name - if full_mod_name == "PySide6": - logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file))) - except Exception as e: - raise RuntimeError(f"[DEPLOY] Finding module import failed on file {str(py_file)} with " - f"error {e}") - - return set(modules) - - py_candidates = [] - ignore_dirs = ["__pycache__", "env", "venv", "deployment"] - - if project_data: - py_candidates = project_data.python_files - ui_candidates = project_data.ui_files - qrc_candidates = project_data.qrc_files - ui_py_candidates = None - qrc_ui_candidates = None - - if ui_candidates: - ui_py_candidates = [(file.parent / f"ui_{file.stem}.py") for file in ui_candidates - if (file.parent / f"ui_{file.stem}.py").exists()] - - if len(ui_py_candidates) != len(ui_candidates): - warnings.warn("[DEPLOY] The number of uic files and their corresponding Python" - " files don't match.", category=RuntimeWarning) - - py_candidates.extend(ui_py_candidates) - - if qrc_candidates: - qrc_ui_candidates = [(file.parent / f"rc_{file.stem}.py") for file in qrc_candidates - if (file.parent / f"rc_{file.stem}.py").exists()] - - if len(qrc_ui_candidates) != len(qrc_candidates): - warnings.warn("[DEPLOY] The number of qrc files and their corresponding Python" - " files don't match.", category=RuntimeWarning) - - py_candidates.extend(qrc_ui_candidates) - - for py_candidate in py_candidates: - all_modules = all_modules.union(pyside_imports(py_candidate)) - return list(all_modules) - - # incase there is not .pyproject file, search all python files in project_dir, except - # ignore_dirs - if extra_ignore_dirs: - ignore_dirs.extend(extra_ignore_dirs) - - # find relevant .py files - _walk = os.walk(project_dir) - for root, dirs, files in _walk: - dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")] - for py_file in files: - if py_file.endswith(".py"): - py_candidates.append(Path(root) / py_file) - - for py_candidate in py_candidates: - all_modules = all_modules.union(pyside_imports(py_candidate)) - - if not all_modules: - ValueError("[DEPLOY] No PySide6 modules were found") - - return list(all_modules) -- cgit v1.2.3