diff options
-rw-r--r-- | sources/pyside-tools/android_deploy.py | 24 | ||||
-rw-r--r-- | sources/pyside-tools/android_deploy.pyproject | 2 | ||||
-rw-r--r-- | sources/pyside-tools/deploy.py | 32 | ||||
-rw-r--r-- | sources/pyside-tools/deploy.pyproject | 3 | ||||
-rw-r--r-- | sources/pyside-tools/deploy_lib/__init__.py | 28 | ||||
-rw-r--r-- | sources/pyside-tools/deploy_lib/android/android_config.py | 26 | ||||
-rw-r--r-- | sources/pyside-tools/deploy_lib/config.py | 79 | ||||
-rw-r--r-- | sources/pyside-tools/deploy_lib/default.spec | 6 | ||||
-rw-r--r-- | sources/pyside-tools/deploy_lib/dependency_util.py | 218 | ||||
-rw-r--r-- | sources/pyside-tools/deploy_lib/deploy_util.py | 109 | ||||
-rw-r--r-- | sources/pyside6/tests/CMakeLists.txt | 2 | ||||
-rw-r--r-- | sources/pyside6/tests/tools/pyside6-android-deploy/test_pyside6_android_deploy.py | 4 | ||||
-rw-r--r-- | sources/pyside6/tests/tools/pyside6-deploy/test_pyside6_deploy.py | 41 |
13 files changed, 393 insertions, 181 deletions
diff --git a/sources/pyside-tools/android_deploy.py b/sources/pyside-tools/android_deploy.py index 362fb3766..75269d622 100644 --- a/sources/pyside-tools/android_deploy.py +++ b/sources/pyside-tools/android_deploy.py @@ -9,7 +9,7 @@ from pathlib import Path from textwrap import dedent from deploy_lib import (create_config_file, cleanup, config_option_exists, PythonExecutable, - MAJOR_VERSION) + MAJOR_VERSION, HELP_EXTRA_IGNORE_DIRS, HELP_EXTRA_MODULES) from deploy_lib.android import AndroidData, AndroidConfig from deploy_lib.android.buildozer import Buildozer @@ -46,26 +46,6 @@ from deploy_lib.android.buildozer import Buildozer Note: This file is used by both pyside6-deploy and pyside6-android-deploy """ -HELP_EXTRA_IGNORE_DIRS = dedent(""" - Comma separated directory names inside the project dir. These - directories will be skipped when searching for python files - relevant to the project. - - Example usage: --extra-ignore-dirs=doc,translations - """) - -HELP_EXTRA_MODULES = dedent(""" - Comma separated list of Qt modules to be added to the application, - in case they are not found automatically. - - This occurs when you have 'import PySide6' in your code instead - 'from PySide6 import <module>'. The module name is specified - with either omitting the prefix of Qt or with it. - - Example usage 1: --extra-modules=Network,Svg - Example usage 2: --extra-modules=QtNetwork,QtSvg - """) - def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = None, ndk_path: Path = None, sdk_path: Path = None, config_file: Path = None, init: bool = False, @@ -124,7 +104,7 @@ def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = Non config.title = name try: - config.modules += extra_modules + config.modules += list(set(extra_modules).difference(set(config.modules))) # this cannot be done when config file is initialized because cleanup() removes it # so this can only be done after the cleanup() diff --git a/sources/pyside-tools/android_deploy.pyproject b/sources/pyside-tools/android_deploy.pyproject index f976cb5a6..bc6347243 100644 --- a/sources/pyside-tools/android_deploy.pyproject +++ b/sources/pyside-tools/android_deploy.pyproject @@ -4,6 +4,6 @@ "deploy_lib/android/recipes/PySide6/__init__.tmpl.py", "deploy_lib/android/recipes/shiboken6/__init__.tmpl.py", "deploy_lib/android/__init__.py", "deploy_lib/android/android_helper.py", - "deploy_lib/android/buildozer.py" + "deploy_lib/android/buildozer.py", "deploy_lib/dependency_util.py" ] } diff --git a/sources/pyside-tools/deploy.py b/sources/pyside-tools/deploy.py index 576c01f9d..6cb6d4d9c 100644 --- a/sources/pyside-tools/deploy.py +++ b/sources/pyside-tools/deploy.py @@ -34,13 +34,14 @@ import traceback from pathlib import Path from textwrap import dedent -from deploy_lib import (MAJOR_VERSION, Config, cleanup, config_option_exists, - finalize, create_config_file, PythonExecutable, Nuitka) +from deploy_lib import (MAJOR_VERSION, DesktopConfig, cleanup, config_option_exists, + finalize, create_config_file, PythonExecutable, Nuitka, + HELP_EXTRA_MODULES, HELP_EXTRA_IGNORE_DIRS) def main(main_file: Path = None, name: str = None, config_file: Path = None, init: bool = False, loglevel=logging.WARNING, dry_run: bool = False, keep_deployment_files: bool = False, - force: bool = False): + force: bool = False, extra_ignore_dirs: str = None, extra_modules_grouped: str = None): logging.basicConfig(level=loglevel) if config_file and not config_file.exists() and not main_file.exists(): @@ -56,6 +57,18 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini config = None logging.info("[DEPLOY] Start") + if extra_ignore_dirs: + extra_ignore_dirs = extra_ignore_dirs.split(",") + + extra_modules = [] + if extra_modules_grouped: + tmp_extra_modules = extra_modules_grouped.split(",") + for extra_module in tmp_extra_modules: + if extra_module.startswith("Qt"): + extra_modules.append(extra_module[2:]) + else: + extra_modules.append(extra_module) + python = PythonExecutable(dry_run=dry_run, init=init, force=force) config_file_exists = config_file and Path(config_file).exists() @@ -65,8 +78,9 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini config_file = create_config_file(dry_run=dry_run, config_file=config_file, main_file=main_file) - config = Config(config_file=config_file, source_file=main_file, python_exe=python.exe, - dry_run=dry_run, existing_config_file=config_file_exists) + config = DesktopConfig(config_file=config_file, source_file=main_file, python_exe=python.exe, + dry_run=dry_run, existing_config_file=config_file_exists, + extra_ignore_dirs=extra_ignore_dirs) # set application name if name: @@ -81,6 +95,8 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini if python.is_pyenv_python() and add_arg not in config.extra_args: config.extra_args += add_arg + config.modules += list(set(extra_modules).difference(set(config.modules))) + # writing config file # in the case of --dry-run, we use default.spec as reference. Do not save the changes # for --dry-run @@ -153,7 +169,11 @@ if __name__ == "__main__": parser.add_argument("--name", type=str, help="Application name") + parser.add_argument("--extra-ignore-dirs", type=str, help=HELP_EXTRA_IGNORE_DIRS) + + parser.add_argument("--extra-modules", type=str, help=HELP_EXTRA_MODULES) + args = parser.parse_args() main(args.main_file, args.name, args.config_file, args.init, args.loglevel, args.dry_run, - args.keep_deployment_files, args.force) + args.keep_deployment_files, args.force, args.extra_ignore_dirs, args.extra_modules) diff --git a/sources/pyside-tools/deploy.pyproject b/sources/pyside-tools/deploy.pyproject index eef1668c5..0e6ca8251 100644 --- a/sources/pyside-tools/deploy.pyproject +++ b/sources/pyside-tools/deploy.pyproject @@ -2,6 +2,7 @@ "files": ["deploy.py", "deploy_lib/__init__.py", "deploy_lib/commands.py", "deploy_lib/config.py", "deploy_lib/default.spec", "deploy_lib/nuitka_helper.py", "deploy_lib/pyside_icon.ico", "deploy_lib/pyside_icon.icns","deploy_lib/pyside_icon.jpg", - "deploy_lib/python_helper.py", "deploy_lib/deploy_util.py" + "deploy_lib/python_helper.py", "deploy_lib/deploy_util.py", + "deploy_lib/dependency_util.py" ] } diff --git a/sources/pyside-tools/deploy_lib/__init__.py b/sources/pyside-tools/deploy_lib/__init__.py index 27d178eee..b3dedd40c 100644 --- a/sources/pyside-tools/deploy_lib/__init__.py +++ b/sources/pyside-tools/deploy_lib/__init__.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import sys from pathlib import Path +from textwrap import dedent MAJOR_VERSION = 6 @@ -19,21 +20,40 @@ DEFAULT_APP_ICON = str((Path(__file__).parent / f"pyside_icon{IMAGE_FORMAT}").re IMPORT_WARNING_PYSIDE = (f"[DEPLOY] Found 'import PySide6' in file {0}" ". Use 'from PySide6 import <module>' or pass the module" " needed using --extra-modules command line argument") +HELP_EXTRA_IGNORE_DIRS = dedent(""" + Comma-separated directory names inside the project dir. These + directories will be skipped when searching for Python files + relevant to the project. + + Example usage: --extra-ignore-dirs=doc,translations + """) + +HELP_EXTRA_MODULES = dedent(""" + Comma-separated list of Qt modules to be added to the application, + in case they are not found automatically. + + This occurs when you have 'import PySide6' in your code instead + 'from PySide6 import <module>'. The module name is specified + by either omitting the prefix of Qt or including it. + + Example usage 1: --extra-modules=Network,Svg + Example usage 2: --extra-modules=QtNetwork,QtSvg + """) def get_all_pyside_modules(): """ Returns all the modules installed with PySide6 """ + import PySide6 # They all start with `Qt` as the prefix. Removing this prefix and getting the actual # module name - import PySide6 return [module[2:] for module in PySide6.__all__] from .commands import run_command, run_qmlimportscanner +from .dependency_util import find_pyside_modules, QtDependencyReader from .nuitka_helper import Nuitka -from .config import BaseConfig, Config +from .config import BaseConfig, Config, DesktopConfig from .python_helper import PythonExecutable -from .deploy_util import (cleanup, finalize, create_config_file, - config_option_exists, find_pyside_modules) +from .deploy_util import cleanup, finalize, create_config_file, config_option_exists diff --git a/sources/pyside-tools/deploy_lib/android/android_config.py b/sources/pyside-tools/deploy_lib/android/android_config.py index 8054ce373..417656bb0 100644 --- a/sources/pyside-tools/deploy_lib/android/android_config.py +++ b/sources/pyside-tools/deploy_lib/android/android_config.py @@ -12,8 +12,7 @@ from pkginfo import Wheel from . import (extract_and_copy_jar, get_wheel_android_arch, find_lib_dependencies, get_llvm_readobj, find_qtlibs_in_wheel, platform_map, create_recipe) -from .. import (Config, find_pyside_modules, run_qmlimportscanner, get_all_pyside_modules, - MAJOR_VERSION) +from .. import (Config, find_pyside_modules, get_all_pyside_modules, MAJOR_VERSION) ANDROID_NDK_VERSION = "25c" ANDROID_DEPLOY_CACHE = Path.home() / ".pyside6_android_deploy" @@ -107,9 +106,8 @@ class AndroidConfig(Config): self.qt_libs_path: zipfile.Path = find_qtlibs_in_wheel(wheel_pyside=self.wheel_pyside) logging.info(f"[DEPLOY] Qt libs path inside wheel: {str(self.qt_libs_path)}") - self._modules = [] - if self.get_value("buildozer", "modules"): - self.modules = self.get_value("buildozer", "modules").split(",") + if self.get_value("qt", "modules"): + self.modules = self.get_value("qt", "modules").split(",") else: self._find_and_set_pysidemodules() self._find_and_set_qtquick_modules() @@ -190,7 +188,7 @@ class AndroidConfig(Config): @modules.setter def modules(self, modules): self._modules = modules - self.set_value("buildozer", "modules", ",".join(modules)) + self.set_value("qt", "modules", ",".join(modules)) @property def local_libs(self): @@ -282,22 +280,6 @@ class AndroidConfig(Config): raise RuntimeError("[DEPLOY] PySide wheel corrupted. Wheel name should end with" "platform name") - def _find_and_set_qtquick_modules(self): - """Identify if QtQuick is used in QML files and add them as dependency - """ - extra_modules = [] - if not self.qml_modules: - self.qml_modules = set(run_qmlimportscanner(qml_files=self.qml_files, - dry_run=self.dry_run)) - - if "QtQuick" in self.qml_modules: - extra_modules.append("Quick") - - if "QtQuick.Controls" in self.qml_modules: - extra_modules.append("QuickControls2") - - self.modules += extra_modules - def _find_dependent_qt_modules(self): """ Given pysidedeploy_config.modules, find all the other dependent Qt modules. This is diff --git a/sources/pyside-tools/deploy_lib/config.py b/sources/pyside-tools/deploy_lib/config.py index 61b2ebec1..44b4ded06 100644 --- a/sources/pyside-tools/deploy_lib/config.py +++ b/sources/pyside-tools/deploy_lib/config.py @@ -1,15 +1,14 @@ # Copyright (C) 2022 The Qt Company Ltd. # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only - import configparser import logging import warnings from configparser import ConfigParser +from typing import List from pathlib import Path from project import ProjectData -from .commands import run_qmlimportscanner -from . import DEFAULT_APP_ICON +from . import DEFAULT_APP_ICON, find_pyside_modules, run_qmlimportscanner, QtDependencyReader # Some QML plugins like QtCore are excluded from this list as they don't contribute much to # executable size. Excluding them saves the extra processing of checking for them in files @@ -17,7 +16,8 @@ EXCLUDED_QML_PLUGINS = {"QtQuick", "QtQuick3D", "QtCharts", "QtWebEngine", "QtTe class BaseConfig: - + """Wrapper class around any .spec file with function to read and set values for the .spec file + """ def __init__(self, config_file: Path, comment_prefixes: str = "/", existing_config_file: bool = False) -> None: self.config_file = config_file @@ -60,9 +60,10 @@ class Config(BaseConfig): """ def __init__(self, config_file: Path, source_file: Path, python_exe: Path, dry_run: bool, - existing_config_file: bool = False): + existing_config_file: bool = False, extra_ignore_dirs: List[str] = None): super().__init__(config_file=config_file, existing_config_file=existing_config_file) + self.extra_ignore_dirs = extra_ignore_dirs self._dry_run = dry_run self.qml_modules = set() # set source_file @@ -122,6 +123,8 @@ class Config(BaseConfig): self._generated_files_path = self.project_dir / "deployment" + self.modules = [] + def set_or_fetch(self, config_property_val, config_property_key, config_property_group="app"): """ Write to config_file if 'config_property_key' is known without config_file @@ -221,6 +224,15 @@ class Config(BaseConfig): def exe_dir(self, exe_dir: Path): self._exe_dir = exe_dir + @property + def modules(self): + return self._modules + + @modules.setter + def modules(self, modules): + self._modules = modules + self.set_value("qt", "modules", ",".join(modules)) + def _find_and_set_qml_files(self): """Fetches all the qml_files in the folder and sets them if the field qml_files is empty in the config_dir""" @@ -321,3 +333,60 @@ class Config(BaseConfig): config_property_val=self.exe_dir, config_property_key="exec_directory" ) ).absolute() + + def _find_and_set_pysidemodules(self): + self.modules = find_pyside_modules(project_dir=self.project_dir, + extra_ignore_dirs=self.extra_ignore_dirs, + project_data=self.project_data) + logging.info("The following PySide modules were found from the Python files of " + f"the project {self.modules}") + + def _find_and_set_qtquick_modules(self): + """Identify if QtQuick is used in QML files and add them as dependency + """ + extra_modules = [] + if not self.qml_modules: + self.qml_modules = set(run_qmlimportscanner(qml_files=self.qml_files, + dry_run=self.dry_run)) + + if "QtQuick" in self.qml_modules: + extra_modules.append("Quick") + + if "QtQuick.Controls" in self.qml_modules: + extra_modules.append("QuickControls2") + + self.modules += extra_modules + + +class DesktopConfig(Config): + """Wrapper class around pysidedeploy.spec, but specific to Desktop deployment + """ + def __init__(self, config_file: Path, source_file: Path, python_exe: Path, dry_run: bool, + existing_config_file: bool = False, extra_ignore_dirs: List[str] = None): + super().__init__(config_file, source_file, python_exe, dry_run, existing_config_file, + extra_ignore_dirs) + + if self.get_value("qt", "modules"): + self.modules = self.get_value("qt", "modules").split(",") + else: + self._find_and_set_pysidemodules() + self._find_and_set_qtquick_modules() + self._find_dependent_qt_modules() + + def _find_dependent_qt_modules(self): + """ + Given pysidedeploy_config.modules, find all the other dependent Qt modules. + """ + dependency_reader = QtDependencyReader(dry_run=self.dry_run) + all_modules = set(self.modules) + + if not dependency_reader.lib_reader: + warnings.warn(f"[DEPLOY] Unable to find {dependency_reader.lib_reader_name}. This tool" + " helps to find the Qt module dependencies of the application. Skipping " + " checking for dependencies.", category=RuntimeWarning) + return + + for module_name in self.modules: + dependency_reader.find_dependencies(module=module_name, used_modules=all_modules) + + self.modules = list(all_modules) diff --git a/sources/pyside-tools/deploy_lib/default.spec b/sources/pyside-tools/deploy_lib/default.spec index ffbaa5532..2276fa496 100644 --- a/sources/pyside-tools/deploy_lib/default.spec +++ b/sources/pyside-tools/deploy_lib/default.spec @@ -41,6 +41,9 @@ qml_files = # excluded qml plugin binaries excluded_qml_plugins = +# Qt modules used. Comma separated +modules = + [android] # path to PySide wheel @@ -77,9 +80,6 @@ ndk_path = # if empty uses default sdk path downloaded by buildozer sdk_path = -# modules used. Comma separated -modules = - # other libraries to be loaded. Comma separated. # loaded at app startup local_libs = diff --git a/sources/pyside-tools/deploy_lib/dependency_util.py b/sources/pyside-tools/deploy_lib/dependency_util.py new file mode 100644 index 000000000..c7821794f --- /dev/null +++ b/sources/pyside-tools/deploy_lib/dependency_util.py @@ -0,0 +1,218 @@ +# Copyright (C) 2024 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import ast +import re +import os +import site +import warnings +import logging +import shutil +import sys +from pathlib import Path +from typing import List, Set + +from . import IMPORT_WARNING_PYSIDE, run_command + + +def get_qt_libs_dir(): + """ + Finds the path to the Qt libs directory inside PySide6 package installation + """ + pyside_install_dir = None + for possible_site_package in site.getsitepackages(): + if possible_site_package.endswith("site-packages"): + pyside_install_dir = Path(possible_site_package) / "PySide6" + + if not pyside_install_dir: + print("Unable to find site-packages. Exiting ...") + sys.exit(-1) + + if sys.platform == "win32": + return pyside_install_dir + + return pyside_install_dir / "Qt" / "lib" # for linux and macOS + + +def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None, + project_data=None): + """ + Searches all the python files in the project to find all the PySide modules used by + the application. + """ + all_modules = set() + mod_pattern = re.compile("PySide6.Qt(?P<mod_name>.*)") + + def pyside_imports(py_file: Path): + modules = [] + contents = py_file.read_text(encoding="utf-8") + try: + tree = ast.parse(contents) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + main_mod_name = node.module + if main_mod_name.startswith("PySide6"): + if main_mod_name == "PySide6": + # considers 'from PySide6 import QtCore' + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name.startswith("Qt"): + modules.append(full_mod_name[2:]) + continue + + # considers 'from PySide6.QtCore import Qt' + match = mod_pattern.search(main_mod_name) + if match: + mod_name = match.group("mod_name") + modules.append(mod_name) + else: + logging.warning(( + f"[DEPLOY] Unable to find module name from {ast.dump(node)}")) + + if isinstance(node, ast.Import): + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name == "PySide6": + logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file))) + except Exception as e: + raise RuntimeError(f"[DEPLOY] Finding module import failed on file {str(py_file)} with " + f"error {e}") + + return set(modules) + + py_candidates = [] + ignore_dirs = ["__pycache__", "env", "venv", "deployment"] + + if project_data: + py_candidates = project_data.python_files + ui_candidates = project_data.ui_files + qrc_candidates = project_data.qrc_files + ui_py_candidates = None + qrc_ui_candidates = None + + if ui_candidates: + ui_py_candidates = [(file.parent / f"ui_{file.stem}.py") for file in ui_candidates + if (file.parent / f"ui_{file.stem}.py").exists()] + + if len(ui_py_candidates) != len(ui_candidates): + warnings.warn("[DEPLOY] The number of uic files and their corresponding Python" + " files don't match.", category=RuntimeWarning) + + py_candidates.extend(ui_py_candidates) + + if qrc_candidates: + qrc_ui_candidates = [(file.parent / f"rc_{file.stem}.py") for file in qrc_candidates + if (file.parent / f"rc_{file.stem}.py").exists()] + + if len(qrc_ui_candidates) != len(qrc_candidates): + warnings.warn("[DEPLOY] The number of qrc files and their corresponding Python" + " files don't match.", category=RuntimeWarning) + + py_candidates.extend(qrc_ui_candidates) + + for py_candidate in py_candidates: + all_modules = all_modules.union(pyside_imports(py_candidate)) + return list(all_modules) + + # incase there is not .pyproject file, search all python files in project_dir, except + # ignore_dirs + if extra_ignore_dirs: + ignore_dirs.extend(extra_ignore_dirs) + + # find relevant .py files + _walk = os.walk(project_dir) + for root, dirs, files in _walk: + dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")] + for py_file in files: + if py_file.endswith(".py"): + py_candidates.append(Path(root) / py_file) + + for py_candidate in py_candidates: + all_modules = all_modules.union(pyside_imports(py_candidate)) + + if not all_modules: + ValueError("[DEPLOY] No PySide6 modules were found") + + return list(all_modules) + + +class QtDependencyReader: + def __init__(self, dry_run: bool = False) -> None: + self.dry_run = dry_run + self.lib_reader_name = None + self.qt_module_path_pattern = None + self.lib_pattern = None + self.command = None + self.qt_libs_dir = None + + if sys.platform == "linux": + self.lib_reader_name = "readelf" + self.qt_module_path_pattern = "libQt6{module}.so.6" + self.lib_pattern = re.compile("libQt6(?P<mod_name>.*).so.6") + self.command_args = "-d" + elif sys.platform == "darwin": + self.lib_reader_name = "dyld_info" + self.qt_module_path_pattern = "Qt{module}.framework/Versions/A/Qt{module}" + self.lib_pattern = re.compile("@rpath/Qt(?P<mod_name>.*).framework/Versions/A/") + self.command_args = "-dependents" + elif sys.platform == "win32": + self.lib_reader_name = "dumpbin" + self.qt_module_path_pattern = "Qt6{module}.dll" + self.lib_pattern = re.compile("Qt6(?P<mod_name>.*).dll") + self.command_args = "/dependents" + else: + print(f"[DEPLOY] Deployment on unsupported platfrom {sys.platform}") + sys.exit(1) + + self.qt_libs_dir = get_qt_libs_dir() + self._lib_reader = shutil.which(self.lib_reader_name) + + @property + def lib_reader(self): + return self._lib_reader + + def find_dependencies(self, module: str, used_modules: Set[str] = None): + """ + Given a Qt module, find all the other Qt modules it is dependent on and add it to the + 'used_modules' set + """ + qt_module_path = self.qt_libs_dir / self.qt_module_path_pattern.format(module=module) + if not qt_module_path.exists(): + warnings.warn(f"[DEPLOY] {qt_module_path.name} not found in {str(qt_module_path)}." + "Skipping finding its dependencies.", category=RuntimeWarning) + return + + lib_pattern = re.compile(self.lib_pattern) + command = [self.lib_reader, self.command_args, str(qt_module_path)] + # print the command if dry_run is True. + # Normally run_command is going to print the command in dry_run mode. But, this is a + # special case where we need to print the command as well as to run it. + if self.dry_run: + command_str = " ".join(command) + print(command_str + "\n") + + # We need to run this even for dry run, to see the full Nuitka command being executed + _, output = run_command(command=command, dry_run=False, fetch_output=True) + + dependent_modules = set() + for line in output.splitlines(): + line = line.decode("utf-8").lstrip() + if sys.platform == "darwin" and line.startswith(f"Qt{module} [arm64]"): + # macOS Qt frameworks bundles have both x86_64 and arm64 architectures + # We only need to consider one as the dependencies are redundant + break + elif sys.platform == "win32" and line.startswith("Summary"): + # the dependencies would be found before the `Summary` line + break + match = lib_pattern.search(line) + if match: + dep_module = match.group("mod_name") + dependent_modules.add(dep_module) + if dep_module not in used_modules: + used_modules.add(dep_module) + self.find_dependencies(module=dep_module, used_modules=used_modules) + + if dependent_modules: + logging.info(f"[DEPLOY] Following dependencies found for {module}: {dependent_modules}") + else: + logging.info(f"[DEPLOY] No Qt dependencies found for {module}") diff --git a/sources/pyside-tools/deploy_lib/deploy_util.py b/sources/pyside-tools/deploy_lib/deploy_util.py index b20c9c8cb..274b41905 100644 --- a/sources/pyside-tools/deploy_lib/deploy_util.py +++ b/sources/pyside-tools/deploy_lib/deploy_util.py @@ -1,17 +1,12 @@ # Copyright (C) 2023 The Qt Company Ltd. # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -import ast -import re -import os -import warnings import logging import shutil import sys from pathlib import Path -from typing import List -from . import EXE_FORMAT, IMPORT_WARNING_PYSIDE +from . import EXE_FORMAT from .config import Config @@ -76,105 +71,3 @@ def finalize(config: Config): shutil.copy(generated_exec_path, config.exe_dir) print("[DEPLOY] Executed file created in " f"{str(config.exe_dir / (config.source_file.stem + EXE_FORMAT))}") - - -def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None, - project_data=None): - """ - Searches all the python files in the project to find all the PySide modules used by - the application. - """ - all_modules = set() - mod_pattern = re.compile("PySide6.Qt(?P<mod_name>.*)") - - def pyside_imports(py_file: Path): - modules = [] - contents = py_file.read_text(encoding="utf-8") - try: - tree = ast.parse(contents) - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom): - main_mod_name = node.module - if main_mod_name.startswith("PySide6"): - if main_mod_name == "PySide6": - # considers 'from PySide6 import QtCore' - for imported_module in node.names: - full_mod_name = imported_module.name - if full_mod_name.startswith("Qt"): - modules.append(full_mod_name[2:]) - continue - - # considers 'from PySide6.QtCore import Qt' - match = mod_pattern.search(main_mod_name) - if match: - mod_name = match.group("mod_name") - modules.append(mod_name) - else: - logging.warning(( - f"[DEPLOY] Unable to find module name from{ast.dump(node)}")) - - if isinstance(node, ast.Import): - for imported_module in node.names: - full_mod_name = imported_module.name - if full_mod_name == "PySide6": - logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file))) - except Exception as e: - raise RuntimeError(f"[DEPLOY] Finding module import failed on file {str(py_file)} with " - f"error {e}") - - return set(modules) - - py_candidates = [] - ignore_dirs = ["__pycache__", "env", "venv", "deployment"] - - if project_data: - py_candidates = project_data.python_files - ui_candidates = project_data.ui_files - qrc_candidates = project_data.qrc_files - ui_py_candidates = None - qrc_ui_candidates = None - - if ui_candidates: - ui_py_candidates = [(file.parent / f"ui_{file.stem}.py") for file in ui_candidates - if (file.parent / f"ui_{file.stem}.py").exists()] - - if len(ui_py_candidates) != len(ui_candidates): - warnings.warn("[DEPLOY] The number of uic files and their corresponding Python" - " files don't match.", category=RuntimeWarning) - - py_candidates.extend(ui_py_candidates) - - if qrc_candidates: - qrc_ui_candidates = [(file.parent / f"rc_{file.stem}.py") for file in qrc_candidates - if (file.parent / f"rc_{file.stem}.py").exists()] - - if len(qrc_ui_candidates) != len(qrc_candidates): - warnings.warn("[DEPLOY] The number of qrc files and their corresponding Python" - " files don't match.", category=RuntimeWarning) - - py_candidates.extend(qrc_ui_candidates) - - for py_candidate in py_candidates: - all_modules = all_modules.union(pyside_imports(py_candidate)) - return list(all_modules) - - # incase there is not .pyproject file, search all python files in project_dir, except - # ignore_dirs - if extra_ignore_dirs: - ignore_dirs.extend(extra_ignore_dirs) - - # find relevant .py files - _walk = os.walk(project_dir) - for root, dirs, files in _walk: - dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")] - for py_file in files: - if py_file.endswith(".py"): - py_candidates.append(Path(root) / py_file) - - for py_candidate in py_candidates: - all_modules = all_modules.union(pyside_imports(py_candidate)) - - if not all_modules: - ValueError("[DEPLOY] No PySide6 modules were found") - - return list(all_modules) diff --git a/sources/pyside6/tests/CMakeLists.txt b/sources/pyside6/tests/CMakeLists.txt index c581a8d0f..539e1aea8 100644 --- a/sources/pyside6/tests/CMakeLists.txt +++ b/sources/pyside6/tests/CMakeLists.txt @@ -17,7 +17,7 @@ endif() # the path to the testbinding module get_filename_component(BUILD_DIR "${CMAKE_BINARY_DIR}" DIRECTORY) get_filename_component(BUILD_DIR "${CMAKE_BINARY_DIR}" DIRECTORY) -set(QT_DIR "${_qt5Core_install_prefix}") +set(QT_DIR "${QT6_INSTALL_PREFIX}") macro(TEST_QT_MODULE var name) if(NOT DISABLE_${name} AND ${var}) diff --git a/sources/pyside6/tests/tools/pyside6-android-deploy/test_pyside6_android_deploy.py b/sources/pyside6/tests/tools/pyside6-android-deploy/test_pyside6_android_deploy.py index 44faaba86..ec575e923 100644 --- a/sources/pyside6/tests/tools/pyside6-android-deploy/test_pyside6_android_deploy.py +++ b/sources/pyside6/tests/tools/pyside6-android-deploy/test_pyside6_android_deploy.py @@ -138,7 +138,7 @@ class TestPySide6AndroidDeployWidgets(DeployTestBase): self.assertIn(str(self.ndk_path), config_obj.get_value("buildozer", "ndk_path")) self.assertEqual(config_obj.get_value("buildozer", "sdk_path"), '') expected_modules = {"Core", "Gui"} - obtained_modules = set(config_obj.get_value("buildozer", "modules").split(",")) + obtained_modules = set(config_obj.get_value("qt", "modules").split(",")) self.assertEqual(obtained_modules, expected_modules) expected_local_libs = "" self.assertEqual(config_obj.get_value("buildozer", "local_libs"), @@ -241,7 +241,7 @@ class TestPySide6AndroidDeployQml(DeployTestBase): config_obj = self.deploy_lib.BaseConfig(config_file=self.config_file) expected_modules = {"Quick", "Core", "Gui", "Network", "Qml", "QmlModels", "OpenGL"} - obtained_modules = set(config_obj.get_value("buildozer", "modules").split(",")) + obtained_modules = set(config_obj.get_value("qt", "modules").split(",")) self.assertEqual(obtained_modules, expected_modules) expected_local_libs = "" self.assertEqual(config_obj.get_value("buildozer", "local_libs"), diff --git a/sources/pyside6/tests/tools/pyside6-deploy/test_pyside6_deploy.py b/sources/pyside6/tests/tools/pyside6-deploy/test_pyside6_deploy.py index 074cfbdda..c79a633e1 100644 --- a/sources/pyside6/tests/tools/pyside6-deploy/test_pyside6_deploy.py +++ b/sources/pyside6/tests/tools/pyside6-deploy/test_pyside6_deploy.py @@ -7,12 +7,13 @@ import shutil import sys import os import importlib +import platform from pathlib import Path from unittest.mock import patch from unittest import mock sys.path.append(os.fspath(Path(__file__).resolve().parents[2])) -from init_paths import init_test_paths # noqa: E402 +from init_paths import init_test_paths, _get_qt_lib_dir # noqa: E402 init_test_paths(False) @@ -78,6 +79,8 @@ class DeployTestBase(LongSortedOptionTest): os.chdir(self.current_dir) +@unittest.skipIf(sys.platform == "darwin" and int(platform.mac_ver()[0].split('.')[0]) <= 11, + "Test only works on macOS version 12+") class TestPySide6DeployWidgets(DeployTestBase): @classmethod def setUpClass(cls): @@ -114,9 +117,10 @@ class TestPySide6DeployWidgets(DeployTestBase): original_output = self.deploy.main(self.main_file, dry_run=True, force=True) self.assertEqual(original_output, self.expected_run_cmd) - def testWidgetConfigFile(self): + @patch("deploy_lib.dependency_util.get_qt_libs_dir") + def testWidgetConfigFile(self, mock_sitepackages): + mock_sitepackages.return_value = Path(_get_qt_lib_dir()) # includes both dry run and config_file tests - # init init_result = self.deploy.main(self.main_file, init=True, force=True) self.assertEqual(init_result, None) @@ -137,6 +141,11 @@ class TestPySide6DeployWidgets(DeployTestBase): equ_value = equ_base + " --static-libpython=no" if is_pyenv_python() else equ_base self.assertEqual(config_obj.get_value("nuitka", "extra_args"), equ_value) self.assertEqual(config_obj.get_value("qt", "excluded_qml_plugins"), "") + expected_modules = {"Core", "Gui", "Widgets"} + if sys.platform != "win32": + expected_modules.add("DBus") + obtained_modules = set(config_obj.get_value("qt", "modules").split(",")) + self.assertEqual(obtained_modules, expected_modules) self.config_file.unlink() def testErrorReturns(self): @@ -147,6 +156,8 @@ class TestPySide6DeployWidgets(DeployTestBase): self.assertTrue("Directory does not contain main.py file." in str(context.exception)) +@unittest.skipIf(sys.platform == "darwin" and int(platform.mac_ver()[0].split('.')[0]) <= 11, + "Test only works on macOS version 12+") class TestPySide6DeployQml(DeployTestBase): @classmethod def setUpClass(cls): @@ -195,7 +206,9 @@ class TestPySide6DeployQml(DeployTestBase): self.expected_run_cmd += " --static-libpython=no" self.config_file = self.temp_example_qml / "pysidedeploy.spec" - def testQmlConfigFile(self): + @patch("deploy_lib.dependency_util.get_qt_libs_dir") + def testQmlConfigFile(self, mock_sitepackages): + mock_sitepackages.return_value = Path(_get_qt_lib_dir()) # create config file with patch("deploy_lib.config.run_qmlimportscanner") as mock_qmlimportscanner: mock_qmlimportscanner.return_value = ["QtQuick"] @@ -217,6 +230,11 @@ class TestPySide6DeployQml(DeployTestBase): config_obj.get_value("qt", "excluded_qml_plugins"), "QtCharts,QtQuick3D,QtSensors,QtTest,QtWebEngine", ) + expected_modules = {"Core", "Gui", "Qml", "Quick", "Network", "OpenGL", "QmlModels"} + if sys.platform != "win32": + expected_modules.add("DBus") + obtained_modules = set(config_obj.get_value("qt", "modules").split(",")) + self.assertEqual(obtained_modules, expected_modules) self.config_file.unlink() def testQmlDryRun(self): @@ -234,6 +252,8 @@ class TestPySide6DeployQml(DeployTestBase): self.assertEqual(mock_qmlimportscanner.call_count, 1) +@unittest.skipIf(sys.platform == "darwin" and int(platform.mac_ver()[0].split('.')[0]) <= 11, + "Test only works on macOS version 12+") class TestPySide6DeployWebEngine(DeployTestBase): @classmethod def setUpClass(cls): @@ -243,8 +263,10 @@ class TestPySide6DeployWebEngine(DeployTestBase): shutil.copytree(example_webenginequick, Path(cls.temp_dir) / "nanobrowser") ).resolve() - # this test case retains the QtWebEngine dlls - def testWebEngineQuickDryRun(self): + @patch("deploy_lib.dependency_util.get_qt_libs_dir") + def testWebEngineQuickDryRun(self, mock_sitepackages): + mock_sitepackages.return_value = Path(_get_qt_lib_dir()) + # this test case retains the QtWebEngine dlls # setup os.chdir(self.temp_example_webenginequick) main_file = self.temp_example_webenginequick / "quicknanobrowser.py" @@ -311,6 +333,13 @@ class TestPySide6DeployWebEngine(DeployTestBase): config_obj.get_value("qt", "excluded_qml_plugins"), "QtCharts,QtQuick3D,QtSensors,QtTest", ) + expected_modules = {"Core", "Gui", "Quick", "Qml", "WebEngineQuick", "Network", "OpenGL", + "Positioning", "WebEngineCore", "WebChannel", "WebChannelQuick", + "QmlModels"} + if sys.platform != "win32": + expected_modules.add("DBus") + obtained_modules = set(config_obj.get_value("qt", "modules").split(",")) + self.assertEqual(obtained_modules, expected_modules) if __name__ == "__main__": |