diff options
Diffstat (limited to 'sources/pyside-tools')
| -rw-r--r-- | sources/pyside-tools/android_deploy.py | 69 | ||||
| -rw-r--r-- | sources/pyside-tools/deploy_lib/__init__.py | 2 | ||||
| -rw-r--r-- | sources/pyside-tools/deploy_lib/android/__init__.py | 7 | ||||
| -rw-r--r-- | sources/pyside-tools/deploy_lib/android/android_helper.py | 73 | ||||
| -rw-r--r-- | sources/pyside-tools/deploy_lib/android/buildozer.py | 84 | ||||
| -rw-r--r-- | sources/pyside-tools/deploy_lib/python_helper.py | 87 |
6 files changed, 275 insertions, 47 deletions
diff --git a/sources/pyside-tools/android_deploy.py b/sources/pyside-tools/android_deploy.py index b1ea32064..97ac523b4 100644 --- a/sources/pyside-tools/android_deploy.py +++ b/sources/pyside-tools/android_deploy.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import argparse -import sys import logging import shutil import traceback @@ -12,10 +11,9 @@ from textwrap import dedent from pkginfo import Wheel from deploy_lib import (setup_python, get_config, cleanup, install_python_dependencies, - config_option_exists, MAJOR_VERSION) + config_option_exists, find_pyside_modules, MAJOR_VERSION) from deploy_lib.android import (create_recipe, extract_and_copy_jar, get_wheel_android_arch, - Buildozer, AndroidData, WIDGET_APPLICATION_MODULES, - QUICK_APPLICATION_MODULES) + Buildozer, AndroidData) """ pyside6-android-deploy deployment tool @@ -39,8 +37,6 @@ from deploy_lib.android import (create_recipe, extract_and_copy_jar, get_wheel_a Platforms Supported: aarch64, armv7a, i686, x86_64 - Supported Modules: Core, Gui, Widgets, Network, OpenGL, Qml, Quick, QuickControls2 - Config file: On the first run of the tool, it creates a config file called pysidedeploy.spec which controls the various characteristic of the deployment. Users can simply change the value @@ -50,13 +46,46 @@ from deploy_lib.android import (create_recipe, extract_and_copy_jar, get_wheel_a Note: This file is used by both pyside6-deploy and pyside6-android-deploy """ +HELP_EXTRA_IGNORE_DIRS = dedent(""" + Comma separated directory names inside the project dir. These + directories will be skipped when searching for python files + relevant to the project. + + Example usage: --extra-ignore-dirs=doc,translations + """) + +HELP_EXTRA_MODULES = dedent(""" + Comma separated list of Qt modules to be added to the application, + in case they are not found automatically. + + This occurs when you have 'import PySide6' in your code instead + 'from PySide6 import <module>'. The module name is specified + with either omitting the prefix of Qt or with it. + + Example usage 1: --extra-modules=Network,Svg + Example usage 2: --extra-modules=QtNetwork,QtSvg + """) + def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = None, ndk_path: Path = None, sdk_path: Path = None, config_file: Path = None, init: bool = False, loglevel=logging.WARNING, dry_run: bool = False, keep_deployment_files: bool = False, - force: bool = False): + force: bool = False, extra_ignore_dirs: str = None, extra_modules_grouped: str = None): logging.basicConfig(level=loglevel) + + if extra_ignore_dirs: + extra_ignore_dirs = extra_ignore_dirs.split(",") + + extra_modules = [] + if extra_modules_grouped: + tmp_extra_modules = extra_modules_grouped.split(",") + for extra_module in tmp_extra_modules: + if extra_module.startswith("Qt"): + extra_modules.append(extra_module[2:]) + else: + extra_modules.append(extra_module) + main_file = Path.cwd() / "main.py" generated_files_path = None if not main_file.exists(): @@ -65,12 +94,6 @@ def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = Non 'main.py' and it should be run from the application directory """)) - # check if ndk and sdk path given, else use default - if ndk_path and sdk_path: - logging.warning("[DEPLOY] May not work with custom Ndk and Sdk versions." - "Use the default by leaving out --ndk-path and --sdk-path cl" - "arguments") - android_data = AndroidData(wheel_pyside=pyside_wheel, wheel_shiboken=shiboken_wheel, ndk_path=ndk_path, sdk_path=sdk_path) @@ -121,8 +144,13 @@ def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = Non # TODO: Optimize this based on the modules needed # check if other modules not supported by Android used and raise error if not config.modules: - config.modules = (QUICK_APPLICATION_MODULES if config.qml_files else - WIDGET_APPLICATION_MODULES) + config.modules = find_pyside_modules(project_dir=config.project_dir, + extra_ignore_dirs=extra_ignore_dirs, + project_data=config.project_data) + logging.info("The following PySide modules were found from the python files of " + f"the project {config.modules}") + + config.modules.extend(extra_modules) # find architecture from wheel name if not config.arch: @@ -211,16 +239,21 @@ if __name__ == "__main__": help=f"Path to shiboken{MAJOR_VERSION} Android Wheel", required=not config_option_exists()) + #TODO: --ndk-path and --sdk-path will be removed when automatic download of sdk and ndk is added parser.add_argument("--ndk-path", type=lambda p: Path(p).resolve(), help=("Path to Android Ndk. If omitted, the default from buildozer is used") - , required="--sdk-path" in sys.argv) + , required=True) parser.add_argument("--sdk-path", type=lambda p: Path(p).resolve(), help=("Path to Android Sdk. If omitted, the default from buildozer is used") - , required="--ndk-path" in sys.argv) + , required=True) + + parser.add_argument("--extra-ignore-dirs", type=str, help=HELP_EXTRA_IGNORE_DIRS) + + parser.add_argument("--extra-modules", type=str, help=HELP_EXTRA_MODULES) args = parser.parse_args() main(args.name, args.wheel_pyside, args.wheel_shiboken, args.ndk_path, args.sdk_path, args.config_file, args.init, args.loglevel, args.dry_run, args.keep_deployment_files, - args.force) + args.force, args.extra_ignore_dirs, args.extra_modules) diff --git a/sources/pyside-tools/deploy_lib/__init__.py b/sources/pyside-tools/deploy_lib/__init__.py index 1aa7ef9cc..003bac5f4 100644 --- a/sources/pyside-tools/deploy_lib/__init__.py +++ b/sources/pyside-tools/deploy_lib/__init__.py @@ -8,6 +8,6 @@ EXE_FORMAT = ".exe" if sys.platform == "win32" else ".bin" from .commands import run_command from .nuitka_helper import Nuitka from .config import BaseConfig, Config -from .python_helper import PythonExecutable +from .python_helper import PythonExecutable, find_pyside_modules from .deploy_util import (cleanup, finalize, get_config, setup_python, install_python_dependencies, config_option_exists) diff --git a/sources/pyside-tools/deploy_lib/android/__init__.py b/sources/pyside-tools/deploy_lib/android/__init__.py index 12eb6830c..e53f41848 100644 --- a/sources/pyside-tools/deploy_lib/android/__init__.py +++ b/sources/pyside-tools/deploy_lib/android/__init__.py @@ -1,10 +1,7 @@ # Copyright (C) 2023 The Qt Company Ltd. # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -WIDGET_APPLICATION_MODULES = ["Core", "Gui", "Widgets"] -QUICK_APPLICATION_MODULES = ["Core", "Gui", "Widgets", "Network", "OpenGL", "Qml", "Quick", - "QuickControls2"] - from .android_helper import (create_recipe, extract_and_copy_jar, get_wheel_android_arch, - AndroidData) + AndroidData, get_llvm_readobj, find_lib_dependencies, + find_qtlibs_in_wheel) from .buildozer import Buildozer diff --git a/sources/pyside-tools/deploy_lib/android/android_helper.py b/sources/pyside-tools/deploy_lib/android/android_helper.py index 321fc3734..514ac8b5d 100644 --- a/sources/pyside-tools/deploy_lib/android/android_helper.py +++ b/sources/pyside-tools/deploy_lib/android/android_helper.py @@ -2,10 +2,15 @@ # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import logging -from pathlib import Path -from jinja2 import Environment, FileSystemLoader +import zipfile from zipfile import ZipFile from dataclasses import dataclass +from typing import Set + +from pathlib import Path +from jinja2 import Environment, FileSystemLoader + +from .. import run_command @dataclass @@ -62,3 +67,67 @@ def get_wheel_android_arch(wheel: Path): return arch return None + + +def get_llvm_readobj(ndk_path: Path) -> Path: + ''' + Return the path to llvm_readobj from the Android Ndk + ''' + if not ndk_path: + # fetch ndk path from buildozer + raise FileNotFoundError("[DEPLOY] Unable to find Ndk path. Please pass the Ndk path either" + " from the CLI or from pysidedeploy.spec") + + # TODO: Requires change if Windows platform supports Android Deployment or if we + # support host other than linux-x86_64 + return (ndk_path / "toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-readobj") + + +def find_lib_dependencies(llvm_readobj: Path, lib_path: Path, used_dependencies: Set[str] = None, + dry_run: bool = False): + """ + Find all the Qt dependencies of a library using llvm_readobj + """ + if lib_path.name in used_dependencies: + return + + command = [str(llvm_readobj), "--needed-libs", str(lib_path)] + _, output = run_command(command=command, dry_run=dry_run, fetch_output=True) + + dependencies = set() + neededlibraries_found = False + for line in output.splitlines(): + line = line.decode("utf-8").lstrip() + if line.startswith("NeededLibraries") and not neededlibraries_found: + neededlibraries_found = True + if neededlibraries_found and line.startswith("libQt"): + dependencies.add(line) + used_dependencies.add(line) + dependent_lib_path = lib_path.parent / line + find_lib_dependencies(llvm_readobj, dependent_lib_path, used_dependencies, dry_run) + + if dependencies: + logging.info(f"[DEPLOY] Following dependencies found for {lib_path.stem}: {dependencies}") + else: + logging.info(f"[DEPLOY] No Qt dependencies found for {lib_path.stem}") + + +def find_qtlibs_in_wheel(wheel_pyside: Path): + """ + Find the path to Qt/lib folder inside the wheel. + """ + archive = ZipFile(wheel_pyside) + qt_libs_path = wheel_pyside / "PySide6/Qt/lib" + qt_libs_path = zipfile.Path(archive, at=qt_libs_path) + if not qt_libs_path.exists(): + for file in archive.namelist(): + # the dependency files are inside the libs folder + if file.endswith("android-dependencies.xml"): + qt_libs_path = zipfile.Path(archive, at=file).parent + # all dependency files are in the same path + break + + if not qt_libs_path: + raise FileNotFoundError("[DEPLOY] Unable to find Qt libs folder inside the wheel") + + return qt_libs_path diff --git a/sources/pyside-tools/deploy_lib/android/buildozer.py b/sources/pyside-tools/deploy_lib/android/buildozer.py index 45c62075c..2eeb2d261 100644 --- a/sources/pyside-tools/deploy_lib/android/buildozer.py +++ b/sources/pyside-tools/deploy_lib/android/buildozer.py @@ -3,12 +3,15 @@ import re import logging +import tempfile import xml.etree.ElementTree as ET + import zipfile from pathlib import Path from typing import List -from .. import MAJOR_VERSION, BaseConfig, Config, run_command +from .. import run_command, BaseConfig, Config, MAJOR_VERSION +from . import get_llvm_readobj, find_lib_dependencies, find_qtlibs_in_wheel class BuildozerConfig(BaseConfig): @@ -47,10 +50,24 @@ class BuildozerConfig(BaseConfig): self.set_value('app', "p4a.local_recipes", str(pysidedeploy_config.recipe_dir)) self.set_value("app", "p4a.bootstrap", "qt") - # gets the xml dependency files from Qt installation path - dependency_files = self.__get_dependency_files(pysidedeploy_config) + self.qt_libs_path: zipfile.Path = ( + find_qtlibs_in_wheel(wheel_pyside=pysidedeploy_config.wheel_pyside)) + logging.info(f"[DEPLOY] Found Qt libs path inside wheel: {str(self.qt_libs_path)}") + + extra_modules = self.__find_dependent_qt_modules(pysidedeploy_config) + logging.info(f"[DEPLOY] Other dependent modules to be added: {extra_modules}") + pysidedeploy_config.modules = pysidedeploy_config.modules + extra_modules + + # update the config file with the extra modules + if extra_modules: + pysidedeploy_config.update_config() modules = ",".join(pysidedeploy_config.modules) + + # gets the xml dependency files from Qt installation path + dependency_files = self.__get_dependency_files(modules=pysidedeploy_config.modules, + arch=self.arch) + local_libs = self.__find_local_libs(dependency_files) pysidedeploy_config.local_libs += local_libs @@ -80,31 +97,18 @@ class BuildozerConfig(BaseConfig): self.update_config() - def __get_dependency_files(self, pysidedeploy_config: Config) -> List[zipfile.Path]: + def __get_dependency_files(self, modules: List[str], arch: str) -> List[zipfile.Path]: """ Based on pysidedeploy_config.modules, returns the Qt6{module}_{arch}-android-dependencies.xml file, which contains the various dependencies of the module, like permissions, plugins etc """ dependency_files = [] - needed_dependency_files = [(f"Qt{MAJOR_VERSION}{module}_{self.arch}" - "-android-dependencies.xml") for module in - pysidedeploy_config.modules] - archive = zipfile.ZipFile(pysidedeploy_config.wheel_pyside) - - # find parent path to dependency files in the wheel - dependency_parent_path = None - for file in archive.namelist(): - if file.endswith("android-dependencies.xml"): - dependency_parent_path = Path(file).parent - # all dependency files are in the same path - break + needed_dependency_files = [(f"Qt{MAJOR_VERSION}{module}_{arch}" + "-android-dependencies.xml") for module in modules] for dependency_file_name in needed_dependency_files: - dependency_file = dependency_parent_path / dependency_file_name - # convert from pathlib.Path to zipfile.Path - dependency_file = zipfile.Path(archive, at=str(dependency_file)) - + dependency_file = self.qt_libs_path / dependency_file_name if dependency_file.exists(): dependency_files.append(dependency_file) @@ -180,6 +184,46 @@ class BuildozerConfig(BaseConfig): return list(local_libs) + def __find_dependent_qt_modules(self, pysidedeploy_config: Config): + """ + Given pysidedeploy_config.modules, find all the other dependent Qt modules. This is + done by using llvm-readobj (readelf) to find the dependent libraries from the module + library. + """ + dependent_modules = set() + all_dependencies = set() + lib_pattern = re.compile(f"libQt6(?P<mod_name>.*)_{self.arch}") + + llvm_readobj = get_llvm_readobj(pysidedeploy_config.ndk_path) + if not llvm_readobj.exists(): + raise FileNotFoundError(f"[DEPLOY] {llvm_readobj} does not exist." + "Finding Qt dependencies failed") + + archive = zipfile.ZipFile(pysidedeploy_config.wheel_pyside) + lib_path_suffix = Path(str(self.qt_libs_path)).relative_to(pysidedeploy_config.wheel_pyside) + + with tempfile.TemporaryDirectory() as tmpdir: + archive.extractall(tmpdir) + qt_libs_tmpdir = Path(tmpdir) / lib_path_suffix + # find the lib folder where Qt libraries are stored + for module_name in pysidedeploy_config.modules: + qt_module_path = qt_libs_tmpdir / f"libQt6{module_name}_{self.arch}.so" + if not qt_module_path.exists(): + raise FileNotFoundError(f"[DEPLOY] libQt6{module_name}_{self.arch}.so not found" + " inside the wheel") + find_lib_dependencies(llvm_readobj=llvm_readobj, lib_path=qt_module_path, + dry_run=pysidedeploy_config.dry_run, + used_dependencies=all_dependencies) + + for dependency in all_dependencies: + match = lib_pattern.search(dependency) + if match: + module = match.group("mod_name") + if module not in pysidedeploy_config.modules: + dependent_modules.add(module) + + return list(dependent_modules) + class Buildozer: dry_run = False diff --git a/sources/pyside-tools/deploy_lib/python_helper.py b/sources/pyside-tools/deploy_lib/python_helper.py index e86ce2e1c..0174a8fcf 100644 --- a/sources/pyside-tools/deploy_lib/python_helper.py +++ b/sources/pyside-tools/deploy_lib/python_helper.py @@ -1,9 +1,12 @@ # Copyright (C) 2022 The Qt Company Ltd. # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -import sys +import ast import os +import re +import sys import logging +from typing import List from importlib import util if sys.version_info >= (3, 8): from importlib.metadata import version @@ -13,6 +16,88 @@ from pathlib import Path from . import Nuitka, run_command, Config +IMPORT_WARNING_PYSIDE = (f"[DEPLOY] Found 'import PySide6' in file {0}" + ". Use 'from PySide6 import <module>' or pass the module" + " needed using --extra-modules command line argument") + + +def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None, + project_data=None): + """ + Searches all the python files in the project to find all the PySide modules used by + the application. + """ + all_modules = set() + mod_pattern = re.compile("PySide6.Qt(?P<mod_name>.*)") + + def pyside_imports(py_file: Path): + modules = [] + contents = py_file.read_text(encoding="utf-8") + try: + tree = ast.parse(contents) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + main_mod_name = node.module + if main_mod_name.startswith("PySide6"): + if main_mod_name == "PySide6": + # considers 'from PySide6 import QtCore' + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name.startswith("Qt"): + modules.append(full_mod_name[2:]) + continue + + # considers 'from PySide6.QtCore import Qt' + match = mod_pattern.search(main_mod_name) + if match: + mod_name = match.group("mod_name") + modules.append(mod_name) + else: + logging.warning(( + f"[DEPLOY] Unable to find module name from{ast.dump(node)}")) + + if isinstance(node, ast.Import): + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name == "PySide6": + logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file))) + + except Exception as e: + logging.error(f"Finding module import failed on file {str(py_file)}") + raise e + + return set(modules) + + py_candidates = [] + ignore_dirs = ["__pycache__", "env", "venv", "deployment"] + + if project_data: + py_candidates = project_data.python_files + for py_candidate in py_candidates: + all_modules = all_modules.union(pyside_imports(py_candidate)) + return list(all_modules) + + # incase there is not .pyproject file, search all python files in project_dir, except + # ignore_dirs + if extra_ignore_dirs: + ignore_dirs.extend(extra_ignore_dirs) + + # find relevant .py files + _walk = os.walk(project_dir) + for root, dirs, files in _walk: + dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")] + for py_file in files: + if py_file.endswith(".py"): + py_candidates.append(Path(root) / py_file) + + for py_candidate in py_candidates: + all_modules = all_modules.union(pyside_imports(py_candidate)) + + if not all_modules: + ValueError("[DEPLOY] No PySide6 modules were found") + + return list(all_modules) + class PythonExecutable: """ |