diff options
author | Shyamnath Premnadh <Shyamnath.Premnadh@qt.io> | 2024-01-30 12:09:04 +0100 |
---|---|---|
committer | Shyamnath Premnadh <Shyamnath.Premnadh@qt.io> | 2024-03-01 14:36:12 +0100 |
commit | 9948f7fd34b268cffaf8cb06d6925f59ce0c538f (patch) | |
tree | 531f8ceb14ade275523d61816a63dab6fbd357eb /sources/pyside-tools/deploy_lib | |
parent | 019a1932c559f0d73d2d8bcd4b3b26ba03dbccb8 (diff) |
Deployment: More Refactoring and minor bug fixes
- setup_python() moved to constructor of PythonExecutable.
-install_python_dependencies() moved under PythonExecutable in
python_helper.py.
- create_executable() of PythonExecutable removed. Instead, we call
Nuitka.create_executable() directly. This removes unncessary import
problems when using PythonExecutable class for Android Deployment.
- nuitka==1.8.0 changed to Nuitka=1.8 in default.spec to match with
the installed version. Otherwise, it forces the reinstall of
Nuitka==1.8 every time (bug).
- Remove recomputation of qt_plugins and local_libs. If the values
exist in pysidedeploy.spec, then they should not be computed again.
This serves the purposes of speeding up the deployment and also
to no modifying the already existing pysidedeploy.spec.
- find_pyside_modules() moved from python_helper.py to deploy_util.py.
- Adapt tests.
- Remove os.fspath wrapping from python.exe. This is not needed as
python.exe is already pathlib.Path.
Pick-to: 6.5 6.6
Task-number: PYSIDE-1612
Change-Id: Ic598e57cd2f2779c410b12fc9584cf60c5e94505
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Diffstat (limited to 'sources/pyside-tools/deploy_lib')
-rw-r--r-- | sources/pyside-tools/deploy_lib/__init__.py | 9 | ||||
-rw-r--r-- | sources/pyside-tools/deploy_lib/android/android_config.py | 28 | ||||
-rw-r--r-- | sources/pyside-tools/deploy_lib/default.spec | 6 | ||||
-rw-r--r-- | sources/pyside-tools/deploy_lib/deploy_util.py | 153 | ||||
-rw-r--r-- | sources/pyside-tools/deploy_lib/python_helper.py | 168 |
5 files changed, 170 insertions, 194 deletions
diff --git a/sources/pyside-tools/deploy_lib/__init__.py b/sources/pyside-tools/deploy_lib/__init__.py index 40a7535db..27d178eee 100644 --- a/sources/pyside-tools/deploy_lib/__init__.py +++ b/sources/pyside-tools/deploy_lib/__init__.py @@ -16,6 +16,9 @@ else: EXE_FORMAT = ".bin" DEFAULT_APP_ICON = str((Path(__file__).parent / f"pyside_icon{IMAGE_FORMAT}").resolve()) +IMPORT_WARNING_PYSIDE = (f"[DEPLOY] Found 'import PySide6' in file {0}" + ". Use 'from PySide6 import <module>' or pass the module" + " needed using --extra-modules command line argument") def get_all_pyside_modules(): @@ -30,7 +33,7 @@ def get_all_pyside_modules(): from .commands import run_command, run_qmlimportscanner from .nuitka_helper import Nuitka -from .python_helper import PythonExecutable, find_pyside_modules from .config import BaseConfig, Config -from .deploy_util import (cleanup, finalize, create_config_file, setup_python, - install_python_dependencies, config_option_exists) +from .python_helper import PythonExecutable +from .deploy_util import (cleanup, finalize, create_config_file, + config_option_exists, find_pyside_modules) diff --git a/sources/pyside-tools/deploy_lib/android/android_config.py b/sources/pyside-tools/deploy_lib/android/android_config.py index 1ea99411f..8054ce373 100644 --- a/sources/pyside-tools/deploy_lib/android/android_config.py +++ b/sources/pyside-tools/deploy_lib/android/android_config.py @@ -121,20 +121,21 @@ class AndroidConfig(Config): self._dependency_files = [] self._find_and_set_dependency_files() - self._qt_plugins = [] - if self.get_value("android", "plugins"): - self._qt_plugins = self.get_value("android", "plugins").split(",") - + dependent_plugins = [] self._local_libs = [] if self.get_value("buildozer", "local_libs"): - self.local_libs = self.get_value("buildozer", "local_libs").split(",") + self._local_libs = self.get_value("buildozer", "local_libs").split(",") + else: + # the local_libs can also store dependent plugins + local_libs, dependent_plugins = self._find_local_libs() + self.local_libs = list(set(local_libs)) - dependent_plugins = [] - # the local_libs can also store dependent plugins - local_libs, dependent_plugins = self._find_local_libs() - self._find_plugin_dependencies(dependent_plugins) - self.qt_plugins += dependent_plugins - self.local_libs += local_libs + self._qt_plugins = [] + if self.get_value("android", "plugins"): + self._qt_plugins = self.get_value("android", "plugins").split(",") + elif dependent_plugins: + self._find_plugin_dependencies(dependent_plugins) + self.qt_plugins = list(set(dependent_plugins)) recipe_dir_temp = self.get_value("buildozer", "recipe_dir") if recipe_dir_temp: @@ -382,11 +383,6 @@ class AndroidConfig(Config): # eg: lib<lib_name>_x86_64.so file_name = Path(file).stem - if file_name.startswith("libplugins_platforms_qtforandroid"): - # the platform library is a requisite and is already added from the - # configuration file - continue - # we only need lib_name, because lib and arch gets re-added by # python-for-android match = lib_pattern.search(file_name) diff --git a/sources/pyside-tools/deploy_lib/default.spec b/sources/pyside-tools/deploy_lib/default.spec index 7ca2edfe7..185a2cd5a 100644 --- a/sources/pyside-tools/deploy_lib/default.spec +++ b/sources/pyside-tools/deploy_lib/default.spec @@ -27,7 +27,7 @@ python_path = # python packages to install # ordered-set: increase compile time performance of nuitka packaging # zstandard: provides final executable size optimization -packages = nuitka==1.8.0,ordered_set,zstandard +packages = Nuitka==1.8,ordered_set,zstandard # buildozer: for deploying Android application android_packages = buildozer==1.5.0,cython==0.29.33 @@ -50,7 +50,7 @@ wheel_pyside = wheel_shiboken = # plugins to be copied to libs folder of the packaged application. Comma separated -plugins = platforms_qtforandroid +plugins = [nuitka] @@ -82,7 +82,7 @@ modules = # other libraries to be loaded. Comma separated. # loaded at app startup -local_libs = plugins_platforms_qtforandroid +local_libs = # architecture of deployed platform # possible values: ["aarch64", "armv7a", "i686", "x86_64"] diff --git a/sources/pyside-tools/deploy_lib/deploy_util.py b/sources/pyside-tools/deploy_lib/deploy_util.py index a8ca58611..b20c9c8cb 100644 --- a/sources/pyside-tools/deploy_lib/deploy_util.py +++ b/sources/pyside-tools/deploy_lib/deploy_util.py @@ -1,14 +1,18 @@ # Copyright (C) 2023 The Qt Company Ltd. # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +import ast +import re +import os +import warnings import logging import shutil import sys from pathlib import Path +from typing import List -from . import EXE_FORMAT +from . import EXE_FORMAT, IMPORT_WARNING_PYSIDE from .config import Config -from .python_helper import PythonExecutable def config_option_exists(): @@ -62,49 +66,6 @@ def create_config_file(dry_run: bool = False, config_file: Path = None, main_fil return config_file -def setup_python(dry_run: bool, force: bool, init: bool): - """ - Sets up Python venv for deployment, and return a wrapper around the venv environment - """ - python = None - response = "yes" - # checking if inside virtual environment - if not PythonExecutable.is_venv() and not force and not dry_run and not init: - response = input(("You are not using a virtual environment. pyside6-deploy needs to install" - " a few Python packages for deployment to work seamlessly. \n" - "Proceed? [Y/n]")) - - if response.lower() in ["no", "n"]: - print("[DEPLOY] Exiting ...") - sys.exit(0) - - python = PythonExecutable(dry_run=dry_run) - logging.info(f"[DEPLOY] Using python at {sys.executable}") - - return python - - -def install_python_dependencies(config: Config, python: PythonExecutable, init: bool, - packages: str, is_android: bool = False): - """ - Installs the python package dependencies for the target deployment platform - """ - packages = config.get_value("python", packages).split(",") - if not init: - # install packages needed for deployment - logging.info("[DEPLOY] Installing dependencies") - python.install(packages=packages) - # nuitka requires patchelf to make patchelf rpath changes for some Qt files - if sys.platform.startswith("linux") and not is_android: - python.install(packages=["patchelf"]) - elif is_android: - # install only buildozer - logging.info("[DEPLOY] Installing buildozer") - buildozer_package_with_version = ([package for package in packages - if package.startswith("buildozer")]) - python.install(packages=list(buildozer_package_with_version)) - - def finalize(config: Config): """ Copy the executable into the final location @@ -115,3 +76,105 @@ def finalize(config: Config): shutil.copy(generated_exec_path, config.exe_dir) print("[DEPLOY] Executed file created in " f"{str(config.exe_dir / (config.source_file.stem + EXE_FORMAT))}") + + +def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None, + project_data=None): + """ + Searches all the python files in the project to find all the PySide modules used by + the application. + """ + all_modules = set() + mod_pattern = re.compile("PySide6.Qt(?P<mod_name>.*)") + + def pyside_imports(py_file: Path): + modules = [] + contents = py_file.read_text(encoding="utf-8") + try: + tree = ast.parse(contents) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + main_mod_name = node.module + if main_mod_name.startswith("PySide6"): + if main_mod_name == "PySide6": + # considers 'from PySide6 import QtCore' + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name.startswith("Qt"): + modules.append(full_mod_name[2:]) + continue + + # considers 'from PySide6.QtCore import Qt' + match = mod_pattern.search(main_mod_name) + if match: + mod_name = match.group("mod_name") + modules.append(mod_name) + else: + logging.warning(( + f"[DEPLOY] Unable to find module name from{ast.dump(node)}")) + + if isinstance(node, ast.Import): + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name == "PySide6": + logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file))) + except Exception as e: + raise RuntimeError(f"[DEPLOY] Finding module import failed on file {str(py_file)} with " + f"error {e}") + + return set(modules) + + py_candidates = [] + ignore_dirs = ["__pycache__", "env", "venv", "deployment"] + + if project_data: + py_candidates = project_data.python_files + ui_candidates = project_data.ui_files + qrc_candidates = project_data.qrc_files + ui_py_candidates = None + qrc_ui_candidates = None + + if ui_candidates: + ui_py_candidates = [(file.parent / f"ui_{file.stem}.py") for file in ui_candidates + if (file.parent / f"ui_{file.stem}.py").exists()] + + if len(ui_py_candidates) != len(ui_candidates): + warnings.warn("[DEPLOY] The number of uic files and their corresponding Python" + " files don't match.", category=RuntimeWarning) + + py_candidates.extend(ui_py_candidates) + + if qrc_candidates: + qrc_ui_candidates = [(file.parent / f"rc_{file.stem}.py") for file in qrc_candidates + if (file.parent / f"rc_{file.stem}.py").exists()] + + if len(qrc_ui_candidates) != len(qrc_candidates): + warnings.warn("[DEPLOY] The number of qrc files and their corresponding Python" + " files don't match.", category=RuntimeWarning) + + py_candidates.extend(qrc_ui_candidates) + + for py_candidate in py_candidates: + all_modules = all_modules.union(pyside_imports(py_candidate)) + return list(all_modules) + + # incase there is not .pyproject file, search all python files in project_dir, except + # ignore_dirs + if extra_ignore_dirs: + ignore_dirs.extend(extra_ignore_dirs) + + # find relevant .py files + _walk = os.walk(project_dir) + for root, dirs, files in _walk: + dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")] + for py_file in files: + if py_file.endswith(".py"): + py_candidates.append(Path(root) / py_file) + + for py_candidate in py_candidates: + all_modules = all_modules.union(pyside_imports(py_candidate)) + + if not all_modules: + ValueError("[DEPLOY] No PySide6 modules were found") + + return list(all_modules) diff --git a/sources/pyside-tools/deploy_lib/python_helper.py b/sources/pyside-tools/deploy_lib/python_helper.py index 6ec3b64f8..7cbf323ed 100644 --- a/sources/pyside-tools/deploy_lib/python_helper.py +++ b/sources/pyside-tools/deploy_lib/python_helper.py @@ -1,124 +1,15 @@ # Copyright (C) 2022 The Qt Company Ltd. # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -import ast import logging import os -import re import sys -import warnings -from typing import List + from importlib import util from importlib.metadata import version from pathlib import Path -from . import Nuitka, run_command - -IMPORT_WARNING_PYSIDE = (f"[DEPLOY] Found 'import PySide6' in file {0}" - ". Use 'from PySide6 import <module>' or pass the module" - " needed using --extra-modules command line argument") - - -def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None, - project_data=None): - """ - Searches all the python files in the project to find all the PySide modules used by - the application. - """ - all_modules = set() - mod_pattern = re.compile("PySide6.Qt(?P<mod_name>.*)") - - def pyside_imports(py_file: Path): - modules = [] - contents = py_file.read_text(encoding="utf-8") - try: - tree = ast.parse(contents) - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom): - main_mod_name = node.module - if main_mod_name.startswith("PySide6"): - if main_mod_name == "PySide6": - # considers 'from PySide6 import QtCore' - for imported_module in node.names: - full_mod_name = imported_module.name - if full_mod_name.startswith("Qt"): - modules.append(full_mod_name[2:]) - continue - - # considers 'from PySide6.QtCore import Qt' - match = mod_pattern.search(main_mod_name) - if match: - mod_name = match.group("mod_name") - modules.append(mod_name) - else: - logging.warning(( - f"[DEPLOY] Unable to find module name from{ast.dump(node)}")) - - if isinstance(node, ast.Import): - for imported_module in node.names: - full_mod_name = imported_module.name - if full_mod_name == "PySide6": - logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file))) - except Exception as e: - raise RuntimeError(f"[DEPLOY] Finding module import failed on file {str(py_file)} with " - f"error {e}") - - return set(modules) - - py_candidates = [] - ignore_dirs = ["__pycache__", "env", "venv", "deployment"] - - if project_data: - py_candidates = project_data.python_files - ui_candidates = project_data.ui_files - qrc_candidates = project_data.qrc_files - ui_py_candidates = None - qrc_ui_candidates = None - - if ui_candidates: - ui_py_candidates = [(file.parent / f"ui_{file.stem}.py") for file in ui_candidates - if (file.parent / f"ui_{file.stem}.py").exists()] - - if len(ui_py_candidates) != len(ui_candidates): - warnings.warn("[DEPLOY] The number of uic files and their corresponding Python" - " files don't match.", category=RuntimeWarning) - - py_candidates.extend(ui_py_candidates) - - if qrc_candidates: - qrc_ui_candidates = [(file.parent / f"rc_{file.stem}.py") for file in qrc_candidates - if (file.parent / f"rc_{file.stem}.py").exists()] - - if len(qrc_ui_candidates) != len(qrc_candidates): - warnings.warn("[DEPLOY] The number of qrc files and their corresponding Python" - " files don't match.", category=RuntimeWarning) - - py_candidates.extend(qrc_ui_candidates) - - for py_candidate in py_candidates: - all_modules = all_modules.union(pyside_imports(py_candidate)) - return list(all_modules) - - # incase there is not .pyproject file, search all python files in project_dir, except - # ignore_dirs - if extra_ignore_dirs: - ignore_dirs.extend(extra_ignore_dirs) - - # find relevant .py files - _walk = os.walk(project_dir) - for root, dirs, files in _walk: - dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")] - for py_file in files: - if py_file.endswith(".py"): - py_candidates.append(Path(root) / py_file) - - for py_candidate in py_candidates: - all_modules = all_modules.union(pyside_imports(py_candidate)) - - if not all_modules: - ValueError("[DEPLOY] No PySide6 modules were found") - - return list(all_modules) +from . import Config, run_command class PythonExecutable: @@ -126,10 +17,28 @@ class PythonExecutable: Wrapper class around Python executable """ - def __init__(self, python_path=None, dry_run=False): - self.exe = python_path if python_path else Path(sys.executable) + def __init__(self, python_path: Path = None, dry_run: bool = False, init: bool = False, + force: bool = False): + self.dry_run = dry_run - self.nuitka = Nuitka(nuitka=[os.fspath(self.exe), "-m", "nuitka"]) + self.init = init + if not python_path: + response = "yes" + # checking if inside virtual environment + if not self.is_venv() and not force and not self.dry_run and not self.init: + response = input(("You are not using a virtual environment. pyside6-deploy needs " + "to install a few Python packages for deployment to work " + "seamlessly. \n Proceed? [Y/n]")) + + if response.lower() in ["no", "n"]: + print("[DEPLOY] Exiting ...") + sys.exit(0) + + self.exe = Path(sys.executable) + else: + self.exe = python_path + + logging.info(f"[DEPLOY] Using Python at {str(self.exe)}") @property def exe(self): @@ -193,16 +102,21 @@ class PythonExecutable: def is_installed(self, package): return bool(util.find_spec(package)) - def create_executable(self, source_file: Path, extra_args: str, config): - if config.qml_files: - logging.info(f"[DEPLOY] Included QML files: {config.qml_files}") - - command_str = self.nuitka.create_executable(source_file=source_file, - extra_args=extra_args, - qml_files=config.qml_files, - excluded_qml_plugins=(config. - excluded_qml_plugins), - icon=config.icon, - dry_run=self.dry_run) - - return command_str + def install_dependencies(self, config: Config, packages: str, is_android: bool = False): + """ + Installs the python package dependencies for the target deployment platform + """ + packages = config.get_value("python", packages).split(",") + if not self.init: + # install packages needed for deployment + logging.info("[DEPLOY] Installing dependencies") + self.install(packages=packages) + # nuitka requires patchelf to make patchelf rpath changes for some Qt files + if sys.platform.startswith("linux") and not is_android: + self.install(packages=["patchelf"]) + elif is_android: + # install only buildozer + logging.info("[DEPLOY] Installing buildozer") + buildozer_package_with_version = ([package for package in packages + if package.startswith("buildozer")]) + self.install(packages=list(buildozer_package_with_version)) |