diff options
Diffstat (limited to 'sources/pyside-tools')
47 files changed, 5841 insertions, 0 deletions
diff --git a/sources/pyside-tools/CMakeLists.txt b/sources/pyside-tools/CMakeLists.txt new file mode 100644 index 000000000..e629ec570 --- /dev/null +++ b/sources/pyside-tools/CMakeLists.txt @@ -0,0 +1,87 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +cmake_minimum_required(VERSION 3.18) +project(pyside-tools) + +include(cmake/PySideToolsSetup.cmake) + +if(${CMAKE_SYSTEM_NAME} STREQUAL "Android") + # create Qt6AndroidBindings.jar + if (NOT DEFINED ANDROID_SDK_ROOT) + message(FATAL_ERROR "Please provide the location of the Android SDK directory via " + "your toolchain file") + endif() + if (NOT DEFINED ANDROID_PLATFORM) + message(FATAL_ERROR "Please provide the location of the Android Platform API level via " + "your toolchain file eg: android-31") + endif() + include(cmake/PySideAndroid.cmake) + create_and_install_qt_javabindings() +else() + set(files ${CMAKE_CURRENT_SOURCE_DIR}/pyside_tool.py + ${CMAKE_CURRENT_SOURCE_DIR}/metaobjectdump.py + ${CMAKE_CURRENT_SOURCE_DIR}/project.py + ${CMAKE_CURRENT_SOURCE_DIR}/qml.py + ${CMAKE_CURRENT_SOURCE_DIR}/qtpy2cpp.py + ${CMAKE_CURRENT_SOURCE_DIR}/deploy.py + ${CMAKE_CURRENT_SOURCE_DIR}/android_deploy.py + ${CMAKE_CURRENT_SOURCE_DIR}/requirements-android.txt) + + set(directories ${CMAKE_CURRENT_SOURCE_DIR}/deploy_lib + ${CMAKE_CURRENT_SOURCE_DIR}/project) + + if(NOT NO_QT_TOOLS STREQUAL "yes") + set(TOOLS_PATH "${QT6_INSTALL_PREFIX}/${QT6_HOST_INFO_BINDIR}") + set(LIBEXEC_PATH "${QT6_INSTALL_PREFIX}/${QT6_HOST_INFO_LIBEXECDIR}") + + list(APPEND files "${LIBEXEC_PATH}/uic${CMAKE_EXECUTABLE_SUFFIX}" + "${LIBEXEC_PATH}/rcc${CMAKE_EXECUTABLE_SUFFIX}" + "${LIBEXEC_PATH}/qmltyperegistrar${CMAKE_EXECUTABLE_SUFFIX}" + "${LIBEXEC_PATH}/qmlimportscanner${CMAKE_EXECUTABLE_SUFFIX}" + "${LIBEXEC_PATH}/qmlcachegen${CMAKE_EXECUTABLE_SUFFIX}" + "${TOOLS_PATH}/lrelease${CMAKE_EXECUTABLE_SUFFIX}" + "${TOOLS_PATH}/lupdate${CMAKE_EXECUTABLE_SUFFIX}" + "${TOOLS_PATH}/qmllint${CMAKE_EXECUTABLE_SUFFIX}" + "${TOOLS_PATH}/qmlformat${CMAKE_EXECUTABLE_SUFFIX}" + "${TOOLS_PATH}/qmlls${CMAKE_EXECUTABLE_SUFFIX}" + "${TOOLS_PATH}/qsb${CMAKE_EXECUTABLE_SUFFIX}" + "${TOOLS_PATH}/balsam${CMAKE_EXECUTABLE_SUFFIX}" + "${TOOLS_PATH}/balsamui${CMAKE_EXECUTABLE_SUFFIX}") + + if (APPLE) + list(APPEND directories "${TOOLS_PATH}/Assistant.app" + "${TOOLS_PATH}/Designer.app" + "${TOOLS_PATH}/Linguist.app") + else() + list(APPEND files "${TOOLS_PATH}/assistant${CMAKE_EXECUTABLE_SUFFIX}" + "${TOOLS_PATH}/designer${CMAKE_EXECUTABLE_SUFFIX}" + "${TOOLS_PATH}/linguist${CMAKE_EXECUTABLE_SUFFIX}") + endif() + endif() + + list(APPEND directories ${CMAKE_CURRENT_SOURCE_DIR}/qtpy2cpp_lib) + + # pyside6-rcc, pyside6-uic, pyside6-designer, shiboken and pyside6-lupdate entrypoints + foreach(file ${files}) + if(EXISTS ${file}) + install(FILES "${file}" + DESTINATION bin + PERMISSIONS + OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ + WORLD_EXECUTE WORLD_READ) + else() + message(WARNING "${file} does not exist. Hence, pyside6-${file} will not work") + endif() + endforeach() + + foreach(directory ${directories}) + install(DIRECTORY "${directory}" + DESTINATION bin + FILE_PERMISSIONS + OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ + WORLD_EXECUTE WORLD_READ) + endforeach() +endif() diff --git a/sources/pyside-tools/__init__.py b/sources/pyside-tools/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/sources/pyside-tools/__init__.py diff --git a/sources/pyside-tools/android_deploy.py b/sources/pyside-tools/android_deploy.py new file mode 100644 index 000000000..75269d622 --- /dev/null +++ b/sources/pyside-tools/android_deploy.py @@ -0,0 +1,212 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import argparse +import logging +import shutil +import traceback +from pathlib import Path +from textwrap import dedent + +from deploy_lib import (create_config_file, cleanup, config_option_exists, PythonExecutable, + MAJOR_VERSION, HELP_EXTRA_IGNORE_DIRS, HELP_EXTRA_MODULES) +from deploy_lib.android import AndroidData, AndroidConfig +from deploy_lib.android.buildozer import Buildozer + + +""" pyside6-android-deploy deployment tool + + Deployment tool that uses buildozer (https://buildozer.readthedocs.io/en/latest/) and + python-for-android (https://python-for-android.readthedocs.io/en/latest/) to deploy PySide6 + applications to Android + + How does it work? + + Command: pyside6-android-deploy --wheel-pyside=<pyside_wheel_path> + --wheel-shiboken=<shiboken_wheel_path> + --ndk-path=<optional_ndk_path> + --sdk-path=<optional_sdk_path> + pyside6-android-deploy android -c /path/to/pysidedeploy.spec + + + Note: If --ndk-path and --sdk-path are not specified, the cache of the tool + `.pyside6_android_deploy` is checked in the user's HOME directory. If it is not found, the user + will have to manually download them. + + Prerequisities: Python main entrypoint file should be named "main.py" + + Platforms Supported: aarch64, armv7a, i686, x86_64 + + Config file: + On the first run of the tool, it creates a config file called pysidedeploy.spec which + controls the various characteristic of the deployment. Users can simply change the value + in this config file to achieve different properties ie. change the application name, + deployment platform etc. + + Note: This file is used by both pyside6-deploy and pyside6-android-deploy +""" + + +def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = None, + ndk_path: Path = None, sdk_path: Path = None, config_file: Path = None, init: bool = False, + loglevel=logging.WARNING, dry_run: bool = False, keep_deployment_files: bool = False, + force: bool = False, extra_ignore_dirs: str = None, extra_modules_grouped: str = None): + + logging.basicConfig(level=loglevel) + + if extra_ignore_dirs: + extra_ignore_dirs = extra_ignore_dirs.split(",") + + extra_modules = [] + if extra_modules_grouped: + tmp_extra_modules = extra_modules_grouped.split(",") + for extra_module in tmp_extra_modules: + if extra_module.startswith("Qt"): + extra_modules.append(extra_module[2:]) + else: + extra_modules.append(extra_module) + + main_file = Path.cwd() / "main.py" + if not main_file.exists(): + raise RuntimeError(("[DEPLOY] For Android deployment to work, the main" + " entrypoint Python file should be named 'main.py'" + " and it should be run from the application" + " directory")) + + android_data = AndroidData(wheel_pyside=pyside_wheel, wheel_shiboken=shiboken_wheel, + ndk_path=ndk_path, sdk_path=sdk_path) + + python = PythonExecutable(dry_run=dry_run, init=init, force=force) + + config_file_exists = config_file and Path(config_file).exists() + + if config_file_exists: + logging.info(f"[DEPLOY] Using existing config file {config_file}") + else: + config_file = create_config_file(dry_run=dry_run, config_file=config_file, + main_file=main_file) + + config = AndroidConfig(config_file=config_file, source_file=main_file, + python_exe=python.exe, dry_run=dry_run, android_data=android_data, + existing_config_file=config_file_exists, + extra_ignore_dirs=extra_ignore_dirs) + + if not config.wheel_pyside and not config.wheel_shiboken: + raise RuntimeError(f"[DEPLOY] No PySide{MAJOR_VERSION} and Shiboken{MAJOR_VERSION} wheels" + "found") + + cleanup(config=config, is_android=True) + + python.install_dependencies(config=config, packages="android_packages", is_android=True) + + # set application name + if name: + config.title = name + + try: + config.modules += list(set(extra_modules).difference(set(config.modules))) + + # this cannot be done when config file is initialized because cleanup() removes it + # so this can only be done after the cleanup() + config.find_and_set_jars_dir() + config.verify_and_set_recipe_dir() + + # TODO: include qml files from pysidedeploy.spec rather than from extensions + # buildozer currently includes all the files with .qml extension + + # init buildozer + Buildozer.dry_run = dry_run + logging.info("[DEPLOY] Creating buildozer.spec file") + Buildozer.initialize(pysidedeploy_config=config) + + # writing config file + if not dry_run: + config.update_config() + + if init: + # config file created above. Exiting. + logging.info(f"[DEPLOY]: Config file {config.config_file} created") + return + + # run buildozer + logging.info("[DEPLOY] Running buildozer deployment") + Buildozer.create_executable(config.mode) + + # move buildozer build files to {generated_files_path} + if not dry_run: + buildozer_build_dir = config.project_dir / ".buildozer" + if not buildozer_build_dir.exists(): + logging.info(f"[DEPLOY] Unable to copy {buildozer_build_dir} to " + f"{config.generated_files_path}. {buildozer_build_dir} does not exist") + logging.info(f"[DEPLOY] copy {buildozer_build_dir} to {config.generated_files_path}") + shutil.move(buildozer_build_dir, config.generated_files_path) + + logging.info(f"[DEPLOY] apk created in {config.exe_dir}") + except Exception: + print(f"Exception occurred: {traceback.format_exc()}") + finally: + if config.generated_files_path and config and not keep_deployment_files: + cleanup(config=config, is_android=True) + + logging.info("[DEPLOY] End") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=dedent(f""" + This tool deploys PySide{MAJOR_VERSION} to Android platforms. + + Note: The main python entrypoint should be named main.py + """), + formatter_class=argparse.RawTextHelpFormatter, + ) + + parser.add_argument("-c", "--config-file", type=lambda p: Path(p).absolute(), + default=(Path.cwd() / "pysidedeploy.spec"), + help="Path to the .spec config file") + + parser.add_argument( + "--init", action="store_true", + help="Create pysidedeploy.spec file, if it doesn't already exists") + + parser.add_argument( + "-v", "--verbose", help="run in verbose mode", action="store_const", + dest="loglevel", const=logging.INFO) + + parser.add_argument("--dry-run", action="store_true", help="show the commands to be run") + + parser.add_argument("--keep-deployment-files", action="store_true", + help="keep the generated deployment files generated") + + parser.add_argument("-f", "--force", action="store_true", help="force all input prompts") + + parser.add_argument("--name", type=str, help="Application name") + + parser.add_argument("--wheel-pyside", type=lambda p: Path(p).resolve(), + help=f"Path to PySide{MAJOR_VERSION} Android Wheel", + required=not config_option_exists()) + + parser.add_argument("--wheel-shiboken", type=lambda p: Path(p).resolve(), + help=f"Path to shiboken{MAJOR_VERSION} Android Wheel", + required=not config_option_exists()) + + parser.add_argument("--ndk-path", type=lambda p: Path(p).resolve(), + help=("Path to Android NDK. If omitted, the tool's cache at " + ".pyside6_android_deploy is checked to find the NDK") + ) + + parser.add_argument("--sdk-path", type=lambda p: Path(p).resolve(), + help=("Path to Android SDK. If omitted, the tool's cache at " + ".pyside6_android_deploy is checked to find the SDK. Otherwise " + "the default from buildozer is used.") + ) + + parser.add_argument("--extra-ignore-dirs", type=str, help=HELP_EXTRA_IGNORE_DIRS) + + parser.add_argument("--extra-modules", type=str, help=HELP_EXTRA_MODULES) + + args = parser.parse_args() + + main(args.name, args.wheel_pyside, args.wheel_shiboken, args.ndk_path, args.sdk_path, + args.config_file, args.init, args.loglevel, args.dry_run, args.keep_deployment_files, + args.force, args.extra_ignore_dirs, args.extra_modules) diff --git a/sources/pyside-tools/android_deploy.pyproject b/sources/pyside-tools/android_deploy.pyproject new file mode 100644 index 000000000..bc6347243 --- /dev/null +++ b/sources/pyside-tools/android_deploy.pyproject @@ -0,0 +1,9 @@ +{ + "files": ["deploy.py", "deploy_lib/__init__.py", "deploy_lib/commands.py", "deploy_lib/config.py", + "deploy_lib/default.spec", "deploy_lib/python_helper.py", "deploy_lib/deploy_util.py", + "deploy_lib/android/recipes/PySide6/__init__.tmpl.py", + "deploy_lib/android/recipes/shiboken6/__init__.tmpl.py", + "deploy_lib/android/__init__.py", "deploy_lib/android/android_helper.py", + "deploy_lib/android/buildozer.py", "deploy_lib/dependency_util.py" + ] +} diff --git a/sources/pyside-tools/cmake/PySideAndroid.cmake b/sources/pyside-tools/cmake/PySideAndroid.cmake new file mode 100644 index 000000000..4b6260cce --- /dev/null +++ b/sources/pyside-tools/cmake/PySideAndroid.cmake @@ -0,0 +1,52 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +set(QT_MAJOR_VERSION 6) + +# Locate Java +include(UseJava) +# Find JDK 8.0 +find_package(Java 1.8 COMPONENTS Development REQUIRED) +# Find QtJavaHelpers.java +include("${QT6_INSTALL_PREFIX}/${QT6_INSTALL_LIBS}/cmake/Qt6/QtJavaHelpers.cmake") + +macro(create_and_install_qt_javabindings) + + # create Qt6AndroidBindings.jar from the following {java_sources} + set(android_main_srcs "${QT6_INSTALL_PREFIX}/src/android/java/src/org/qtproject/qt/android/bindings") + set(java_sources + ${android_main_srcs}/QtActivity.java + ${android_main_srcs}/QtApplication.java + ${android_main_srcs}/QtService.java + ) + # set android.jar from the sdk, for compiling the java files into .jar + set(sdk_jar_location "${ANDROID_SDK_ROOT}/platforms/${ANDROID_PLATFORM}/android.jar") + if (NOT EXISTS "${sdk_jar_location}") + message(FATAL_ERROR "Could not locate Android SDK jar for api '${api}'") + endif() + + # this variable is accessed by qt_internal_add_jar + set(QT_ANDROID_JAR ${sdk_jar_location}) + + set(qt_jar_location "${QT6_INSTALL_PREFIX}/jar/Qt6Android.jar") + if (NOT EXISTS "${qt_jar_location}") + message(FATAL_ERROR "${qt_jar_location} does not exist. Qt6 installation maybe corrupted.") + endif() + + # to be done + list(APPEND included_jars ${sdk_jar_location} ${qt_jar_location}) + + qt_internal_add_jar(Qt${QT_MAJOR_VERSION}AndroidBindings + INCLUDE_JARS ${included_jars} + SOURCES ${java_sources} + ) + + install_jar(Qt${QT_MAJOR_VERSION}AndroidBindings + DESTINATION "${CMAKE_INSTALL_PREFIX}/lib/jar" + COMPONENT Devel + ) + + # install other relevant Android jars from the Qt installation. + # All the jars would be later packaged together with the Android wheels + install(DIRECTORY ${QT6_INSTALL_PREFIX}/jar/ DESTINATION lib/jar) +endmacro() diff --git a/sources/pyside-tools/cmake/PySideToolsHelpers.cmake b/sources/pyside-tools/cmake/PySideToolsHelpers.cmake new file mode 100644 index 000000000..9fb2ec3d0 --- /dev/null +++ b/sources/pyside-tools/cmake/PySideToolsHelpers.cmake @@ -0,0 +1,37 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +function(pyside_tools_internal_detect_if_cross_building) + if(CMAKE_CROSSCOMPILING OR QFP_SHIBOKEN_HOST_PATH) + set(is_cross_build TRUE) + else() + set(is_cross_build FALSE) + endif() + set(PYSIDE_TOOLS_IS_CROSS_BUILD "${is_cross_build}" PARENT_SCOPE) + message(STATUS "PYSIDE_TOOLS_IS_CROSS_BUILD: ${PYSIDE_TOOLS_IS_CROSS_BUILD}") +endfunction() + +function(pyside_tools_internal_set_up_extra_dependency_paths) + set(extra_root_path_vars + QFP_QT_TARGET_PATH + ) + foreach(root_path IN LISTS extra_root_path_vars) + set(new_root_path_value "${${root_path}}") + if(new_root_path_value) + set(new_prefix_path "${CMAKE_PREFIX_PATH}") + list(PREPEND new_prefix_path "${new_root_path_value}/lib/cmake") + set(CMAKE_PREFIX_PATH "${new_prefix_path}") + set(CMAKE_PREFIX_PATH "${new_prefix_path}" PARENT_SCOPE) + + # Need to adjust the prefix and root paths so that find_package(Qt) and other 3rd + # party packages are found successfully when they are located outside of the + # default sysroot (whatever that maybe for the target platform). + if(PYSIDE_TOOLS_IS_CROSS_BUILD) + set(new_root_path "${CMAKE_FIND_ROOT_PATH}") + list(PREPEND new_root_path "${new_root_path_value}") + set(CMAKE_FIND_ROOT_PATH "${new_root_path}") + set(CMAKE_FIND_ROOT_PATH "${new_root_path}" PARENT_SCOPE) + endif() + endif() + endforeach() +endfunction() diff --git a/sources/pyside-tools/cmake/PySideToolsSetup.cmake b/sources/pyside-tools/cmake/PySideToolsSetup.cmake new file mode 100644 index 000000000..93b39460d --- /dev/null +++ b/sources/pyside-tools/cmake/PySideToolsSetup.cmake @@ -0,0 +1,16 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") + +include(PySideToolsHelpers) + +pyside_tools_internal_detect_if_cross_building() +pyside_tools_internal_set_up_extra_dependency_paths() + +find_package(Qt6 REQUIRED COMPONENTS Core HostInfo) + +# Don't display "up-to-date / install" messages when installing, to reduce visual clutter. +if (QUIET_BUILD) + set(CMAKE_INSTALL_MESSAGE NEVER) +endif() diff --git a/sources/pyside-tools/deploy.py b/sources/pyside-tools/deploy.py new file mode 100644 index 000000000..bab5aa0de --- /dev/null +++ b/sources/pyside-tools/deploy.py @@ -0,0 +1,203 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +""" pyside6-deploy deployment tool + + Deployment tool that uses Nuitka to deploy PySide6 applications to various desktop (Windows, + Linux, macOS) platforms. + + How does it work? + + Command: pyside6-deploy path/to/main_file + pyside6-deploy (incase main file is called main.py) + pyside6-deploy -c /path/to/config_file + + Platforms supported: Linux, Windows, macOS + Module binary inclusion: + 1. for non-QML cases, only required modules are included + 2. for QML cases, all modules are included because of all QML plugins getting included + with nuitka + + Config file: + On the first run of the tool, it creates a config file called pysidedeploy.spec which + controls the various characteristic of the deployment. Users can simply change the value + in this config file to achieve different properties ie. change the application name, + deployment platform etc. + + Note: This file is used by both pyside6-deploy and pyside6-android-deploy + +""" + +import argparse +import logging +import traceback +from pathlib import Path +from textwrap import dedent + +from deploy_lib import (MAJOR_VERSION, DesktopConfig, cleanup, config_option_exists, + finalize, create_config_file, PythonExecutable, Nuitka, + HELP_EXTRA_MODULES, HELP_EXTRA_IGNORE_DIRS) + + +TOOL_DESCRIPTION = dedent(f""" + This tool deploys PySide{MAJOR_VERSION} to desktop (Windows, Linux, + macOS) platforms. The following types of executables are produced as per + the platform: + + Windows = .exe + macOS = .app + Linux = .bin + """) + +HELP_MODE = dedent(""" + The mode in which the application is deployed. The options are: onefile, + standalone. The default value is onefile. + + This options translates to the mode Nuitka uses to create the executable. + + macOS by default uses the --standalone option. + """) + + +def main(main_file: Path = None, name: str = None, config_file: Path = None, init: bool = False, + loglevel=logging.WARNING, dry_run: bool = False, keep_deployment_files: bool = False, + force: bool = False, extra_ignore_dirs: str = None, extra_modules_grouped: str = None, + mode: bool = False): + + logging.basicConfig(level=loglevel) + if config_file and not config_file.exists() and not main_file.exists(): + raise RuntimeError(dedent(""" + Directory does not contain main.py file. + Please specify the main python entrypoint file or the config file. + Run "pyside6-deploy desktop --help" to see info about cli options. + + pyside6-deploy exiting...""")) + + # Nuitka command to run + command_str = None + config = None + logging.info("[DEPLOY] Start") + + if extra_ignore_dirs: + extra_ignore_dirs = extra_ignore_dirs.split(",") + + extra_modules = [] + if extra_modules_grouped: + tmp_extra_modules = extra_modules_grouped.split(",") + for extra_module in tmp_extra_modules: + if extra_module.startswith("Qt"): + extra_modules.append(extra_module[2:]) + else: + extra_modules.append(extra_module) + + python = PythonExecutable(dry_run=dry_run, init=init, force=force) + config_file_exists = config_file and Path(config_file).exists() + + if config_file_exists: + logging.info(f"[DEPLOY] Using existing config file {config_file}") + else: + config_file = create_config_file(dry_run=dry_run, config_file=config_file, + main_file=main_file) + + config = DesktopConfig(config_file=config_file, source_file=main_file, python_exe=python.exe, + dry_run=dry_run, existing_config_file=config_file_exists, + extra_ignore_dirs=extra_ignore_dirs, mode=mode) + + # set application name + if name: + config.title = name + + cleanup(config=config) + + python.install_dependencies(config=config, packages="packages") + + # required by Nuitka for pyenv Python + add_arg = " --static-libpython=no" + if python.is_pyenv_python() and add_arg not in config.extra_args: + config.extra_args += add_arg + + config.modules += list(set(extra_modules).difference(set(config.modules))) + + # writing config file + # in the case of --dry-run, we use default.spec as reference. Do not save the changes + # for --dry-run + if not dry_run: + config.update_config() + + if config.qml_files: + logging.info(f"[DEPLOY] Included QML files: {config.qml_files}") + + if init: + # config file created above. Exiting. + logging.info(f"[DEPLOY]: Config file {config.config_file} created") + return + + try: + # create executable + if not dry_run: + logging.info("[DEPLOY] Deploying application") + + nuitka = Nuitka(nuitka=[python.exe, "-m", "nuitka"]) + command_str = nuitka.create_executable(source_file=config.source_file, + extra_args=config.extra_args, + qml_files=config.qml_files, + qt_plugins=config.qt_plugins, + excluded_qml_plugins=config.excluded_qml_plugins, + icon=config.icon, + dry_run=dry_run, + permissions=config.permissions, + mode=config.mode) + except Exception: + print(f"[DEPLOY] Exception occurred: {traceback.format_exc()}") + finally: + if config.generated_files_path and config: + finalize(config=config) + if not keep_deployment_files: + cleanup(config=config) + + logging.info("[DEPLOY] End") + return command_str + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=TOOL_DESCRIPTION) + + parser.add_argument("-c", "--config-file", type=lambda p: Path(p).absolute(), + default=(Path.cwd() / "pysidedeploy.spec"), + help="Path to the .spec config file") + + parser.add_argument( + type=lambda p: Path(p).absolute(), + help="Path to main python file", nargs="?", dest="main_file", + default=None if config_option_exists() else Path.cwd() / "main.py") + + parser.add_argument( + "--init", action="store_true", + help="Create pysidedeploy.spec file, if it doesn't already exists") + + parser.add_argument( + "-v", "--verbose", help="Run in verbose mode", action="store_const", + dest="loglevel", const=logging.INFO) + + parser.add_argument("--dry-run", action="store_true", help="Show the commands to be run") + + parser.add_argument( + "--keep-deployment-files", action="store_true", + help="Keep the generated deployment files generated") + + parser.add_argument("-f", "--force", action="store_true", help="Force all input prompts") + + parser.add_argument("--name", type=str, help="Application name") + + parser.add_argument("--extra-ignore-dirs", type=str, help=HELP_EXTRA_IGNORE_DIRS) + + parser.add_argument("--extra-modules", type=str, help=HELP_EXTRA_MODULES) + + parser.add_argument("--mode", choices=["onefile", "standalone"], default="desktop", + help=HELP_MODE) + + args = parser.parse_args() + + main(args.main_file, args.name, args.config_file, args.init, args.loglevel, args.dry_run, + args.keep_deployment_files, args.force, args.extra_ignore_dirs, args.extra_modules, + args.mode) diff --git a/sources/pyside-tools/deploy.pyproject b/sources/pyside-tools/deploy.pyproject new file mode 100644 index 000000000..0e6ca8251 --- /dev/null +++ b/sources/pyside-tools/deploy.pyproject @@ -0,0 +1,8 @@ +{ + "files": ["deploy.py", "deploy_lib/__init__.py", "deploy_lib/commands.py", "deploy_lib/config.py", + "deploy_lib/default.spec", "deploy_lib/nuitka_helper.py", "deploy_lib/pyside_icon.ico", + "deploy_lib/pyside_icon.icns","deploy_lib/pyside_icon.jpg", + "deploy_lib/python_helper.py", "deploy_lib/deploy_util.py", + "deploy_lib/dependency_util.py" + ] +} diff --git a/sources/pyside-tools/deploy_lib/__init__.py b/sources/pyside-tools/deploy_lib/__init__.py new file mode 100644 index 000000000..a40d0838b --- /dev/null +++ b/sources/pyside-tools/deploy_lib/__init__.py @@ -0,0 +1,59 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +import sys +from pathlib import Path +from textwrap import dedent + +MAJOR_VERSION = 6 + +if sys.platform == "win32": + IMAGE_FORMAT = ".ico" + EXE_FORMAT = ".exe" +elif sys.platform == "darwin": + IMAGE_FORMAT = ".icns" + EXE_FORMAT = ".app" +else: + IMAGE_FORMAT = ".jpg" + EXE_FORMAT = ".bin" + +DEFAULT_APP_ICON = str((Path(__file__).parent / f"pyside_icon{IMAGE_FORMAT}").resolve()) +IMPORT_WARNING_PYSIDE = (f"[DEPLOY] Found 'import PySide6' in file {0}" + ". Use 'from PySide6 import <module>' or pass the module" + " needed using --extra-modules command line argument") +HELP_EXTRA_IGNORE_DIRS = dedent(""" + Comma-separated directory names inside the project dir. These + directories will be skipped when searching for Python files + relevant to the project. + + Example usage: --extra-ignore-dirs=doc,translations + """) + +HELP_EXTRA_MODULES = dedent(""" + Comma-separated list of Qt modules to be added to the application, + in case they are not found automatically. + + This occurs when you have 'import PySide6' in your code instead + 'from PySide6 import <module>'. The module name is specified + by either omitting the prefix of Qt or including it. + + Example usage 1: --extra-modules=Network,Svg + Example usage 2: --extra-modules=QtNetwork,QtSvg + """) + + +def get_all_pyside_modules(): + """ + Returns all the modules installed with PySide6 + """ + import PySide6 + # They all start with `Qt` as the prefix. Removing this prefix and getting the actual + # module name + return [module[2:] for module in PySide6.__all__] + + +from .commands import run_command, run_qmlimportscanner +from .dependency_util import find_pyside_modules, find_permission_categories, QtDependencyReader +from .nuitka_helper import Nuitka +from .config import BaseConfig, Config, DesktopConfig +from .python_helper import PythonExecutable +from .deploy_util import cleanup, finalize, create_config_file, config_option_exists diff --git a/sources/pyside-tools/deploy_lib/android/__init__.py b/sources/pyside-tools/deploy_lib/android/__init__.py new file mode 100644 index 000000000..c3027762c --- /dev/null +++ b/sources/pyside-tools/deploy_lib/android/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +# maps instruction set to Android platform names +platform_map = {"aarch64": "arm64-v8a", + "armv7a": "armeabi-v7a", + "i686": "x86", + "x86_64": "x86_64", + "arm64-v8a": "arm64-v8a", + "armeabi-v7a": "armeabi-v7a", + "x86": "x86"} + +from .android_helper import (create_recipe, extract_and_copy_jar, get_wheel_android_arch, + AndroidData, get_llvm_readobj, find_lib_dependencies, + find_qtlibs_in_wheel) +from .android_config import AndroidConfig diff --git a/sources/pyside-tools/deploy_lib/android/android_config.py b/sources/pyside-tools/deploy_lib/android/android_config.py new file mode 100644 index 000000000..ad818c2ff --- /dev/null +++ b/sources/pyside-tools/deploy_lib/android/android_config.py @@ -0,0 +1,446 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +import re +import tempfile +import logging +import zipfile +import xml.etree.ElementTree as ET + +from typing import List +from pathlib import Path +from pkginfo import Wheel + +from . import (extract_and_copy_jar, get_wheel_android_arch, find_lib_dependencies, + get_llvm_readobj, find_qtlibs_in_wheel, platform_map, create_recipe) +from .. import (Config, find_pyside_modules, get_all_pyside_modules, MAJOR_VERSION) + +ANDROID_NDK_VERSION = "26b" +ANDROID_DEPLOY_CACHE = Path.home() / ".pyside6_android_deploy" + + +class AndroidConfig(Config): + """ + Wrapper class around pysidedeploy.spec file for pyside6-android-deploy + """ + def __init__(self, config_file: Path, source_file: Path, python_exe: Path, dry_run: bool, + android_data, existing_config_file: bool = False, + extra_ignore_dirs: List[str] = None): + super().__init__(config_file=config_file, source_file=source_file, python_exe=python_exe, + dry_run=dry_run, existing_config_file=existing_config_file) + + self.extra_ignore_dirs = extra_ignore_dirs + + if android_data.wheel_pyside: + self.wheel_pyside = android_data.wheel_pyside + else: + wheel_pyside_temp = self.get_value("android", "wheel_pyside") + if not wheel_pyside_temp: + raise RuntimeError("[DEPLOY] Unable to find PySide6 Android wheel") + self.wheel_pyside = Path(wheel_pyside_temp).resolve() + + if android_data.wheel_shiboken: + self.wheel_shiboken = android_data.wheel_shiboken + else: + wheel_shiboken_temp = self.get_value("android", "wheel_shiboken") + if not wheel_shiboken_temp: + raise RuntimeError("[DEPLOY] Unable to find shiboken6 Android wheel") + self.wheel_shiboken = Path(wheel_shiboken_temp).resolve() + + self.ndk_path = None + if android_data.ndk_path: + # from cli + self.ndk_path = android_data.ndk_path + else: + # from config + ndk_path_temp = self.get_value("buildozer", "ndk_path") + if ndk_path_temp: + self.ndk_path = Path(ndk_path_temp) + else: + ndk_path_temp = (ANDROID_DEPLOY_CACHE / "android-ndk" + / f"android-ndk-r{ANDROID_NDK_VERSION}") + if ndk_path_temp.exists(): + self.ndk_path = ndk_path_temp + + if self.ndk_path: + print(f"Using Android NDK: {str(self.ndk_path)}") + else: + raise FileNotFoundError("[DEPLOY] Unable to find Android NDK. Please pass the NDK " + "path either from the CLI or from pysidedeploy.spec") + + self.sdk_path = None + if android_data.sdk_path: + self.sdk_path = android_data.sdk_path + else: + sdk_path_temp = self.get_value("buildozer", "sdk_path") + if sdk_path_temp: + self.sdk_path = Path(sdk_path_temp) + else: + sdk_path_temp = ANDROID_DEPLOY_CACHE / "android-sdk" + if sdk_path_temp.exists(): + self.sdk_path = sdk_path_temp + else: + logging.info("[DEPLOY] Use default SDK from buildozer") + + if self.sdk_path: + print(f"Using Android SDK: {str(self.sdk_path)}") + + recipe_dir_temp = self.get_value("buildozer", "recipe_dir") + self.recipe_dir = Path(recipe_dir_temp) if recipe_dir_temp else None + + self._jars_dir = [] + jars_dir_temp = self.get_value("buildozer", "jars_dir") + if jars_dir_temp and Path(jars_dir_temp).resolve().exists(): + self.jars_dir = Path(jars_dir_temp).resolve() + + self._arch = None + if self.get_value("buildozer", "arch"): + self.arch = self.get_value("buildozer", "arch") + else: + self._find_and_set_arch() + + # maps to correct platform name incase the instruction set was specified + self._arch = platform_map[self.arch] + + self._mode = self.get_value("buildozer", "mode") + + self.qt_libs_path: zipfile.Path = find_qtlibs_in_wheel(wheel_pyside=self.wheel_pyside) + logging.info(f"[DEPLOY] Qt libs path inside wheel: {str(self.qt_libs_path)}") + + if self.get_value("qt", "modules"): + self.modules = self.get_value("qt", "modules").split(",") + else: + self._find_and_set_pysidemodules() + self._find_and_set_qtquick_modules() + self.modules += self._find_dependent_qt_modules() + # remove duplicates + self.modules = list(set(self.modules)) + + # gets the xml dependency files from Qt installation path + self._dependency_files = [] + self._find_and_set_dependency_files() + + dependent_plugins = [] + self._local_libs = [] + if self.get_value("buildozer", "local_libs"): + self._local_libs = self.get_value("buildozer", "local_libs").split(",") + else: + # the local_libs can also store dependent plugins + local_libs, dependent_plugins = self._find_local_libs() + self.local_libs = list(set(local_libs)) + + self._qt_plugins = [] + if self.get_value("android", "plugins"): + self._qt_plugins = self.get_value("android", "plugins").split(",") + elif dependent_plugins: + self._find_plugin_dependencies(dependent_plugins) + self.qt_plugins = list(set(dependent_plugins)) + + recipe_dir_temp = self.get_value("buildozer", "recipe_dir") + if recipe_dir_temp: + self.recipe_dir = Path(recipe_dir_temp) + + @property + def qt_plugins(self): + return self._qt_plugins + + @qt_plugins.setter + def qt_plugins(self, qt_plugins): + self._qt_plugins = qt_plugins + self.set_value("android", "plugins", ",".join(qt_plugins)) + + @property + def ndk_path(self): + return self._ndk_path + + @ndk_path.setter + def ndk_path(self, ndk_path: Path): + self._ndk_path = ndk_path.resolve() if ndk_path else None + if self._ndk_path: + self.set_value("buildozer", "ndk_path", str(self._ndk_path)) + + @property + def sdk_path(self) -> Path: + return self._sdk_path + + @sdk_path.setter + def sdk_path(self, sdk_path: Path): + self._sdk_path = sdk_path.resolve() if sdk_path else None + if self._sdk_path: + self.set_value("buildozer", "sdk_path", str(self._sdk_path)) + + @property + def arch(self): + return self._arch + + @arch.setter + def arch(self, arch): + self._arch = arch + self.set_value("buildozer", "arch", arch) + + @property + def mode(self): + return self._mode + + @property + def modules(self): + return self._modules + + @modules.setter + def modules(self, modules): + self._modules = modules + self.set_value("qt", "modules", ",".join(modules)) + + @property + def local_libs(self): + return self._local_libs + + @local_libs.setter + def local_libs(self, local_libs): + self._local_libs = local_libs + self.set_value("buildozer", "local_libs", ",".join(local_libs)) + + @property + def recipe_dir(self): + return self._recipe_dir + + @recipe_dir.setter + def recipe_dir(self, recipe_dir: Path): + self._recipe_dir = recipe_dir.resolve() if recipe_dir else None + if self._recipe_dir: + self.set_value("buildozer", "recipe_dir", str(self._recipe_dir)) + + def recipes_exist(self): + if not self._recipe_dir: + return False + + pyside_recipe_dir = Path(self.recipe_dir) / "PySide6" + shiboken_recipe_dir = Path(self.recipe_dir) / "shiboken6" + + return pyside_recipe_dir.is_dir() and shiboken_recipe_dir.is_dir() + + @property + def jars_dir(self) -> Path: + return self._jars_dir + + @jars_dir.setter + def jars_dir(self, jars_dir: Path): + self._jars_dir = jars_dir.resolve() if jars_dir else None + if self._jars_dir: + self.set_value("buildozer", "jars_dir", str(self._jars_dir)) + + @property + def wheel_pyside(self) -> Path: + return self._wheel_pyside + + @wheel_pyside.setter + def wheel_pyside(self, wheel_pyside: Path): + self._wheel_pyside = wheel_pyside.resolve() if wheel_pyside else None + if self._wheel_pyside: + self.set_value("android", "wheel_pyside", str(self._wheel_pyside)) + + @property + def wheel_shiboken(self) -> Path: + return self._wheel_shiboken + + @wheel_shiboken.setter + def wheel_shiboken(self, wheel_shiboken: Path): + self._wheel_shiboken = wheel_shiboken.resolve() if wheel_shiboken else None + if self._wheel_shiboken: + self.set_value("android", "wheel_shiboken", str(self._wheel_shiboken)) + + @property + def dependency_files(self): + return self._dependency_files + + @dependency_files.setter + def dependency_files(self, dependency_files): + self._dependency_files = dependency_files + + def _find_and_set_pysidemodules(self): + self.modules = find_pyside_modules(project_dir=self.project_dir, + extra_ignore_dirs=self.extra_ignore_dirs, + project_data=self.project_data) + logging.info("The following PySide modules were found from the python files of " + f"the project {self.modules}") + + def find_and_set_jars_dir(self): + """Extract out and copy .jar files to {generated_files_path} + """ + if not self.dry_run: + logging.info("[DEPLOY] Extract and copy jar files from PySide6 wheel to " + f"{self.generated_files_path}") + self.jars_dir = extract_and_copy_jar(wheel_path=self.wheel_pyside, + generated_files_path=self.generated_files_path) + + def _find_and_set_arch(self): + """Find architecture from wheel name + """ + self.arch = get_wheel_android_arch(wheel=self.wheel_pyside) + if not self.arch: + raise RuntimeError("[DEPLOY] PySide wheel corrupted. Wheel name should end with" + "platform name") + + def _find_dependent_qt_modules(self): + """ + Given pysidedeploy_config.modules, find all the other dependent Qt modules. This is + done by using llvm-readobj (readelf) to find the dependent libraries from the module + library. + """ + dependent_modules = set() + all_dependencies = set() + lib_pattern = re.compile(f"libQt6(?P<mod_name>.*)_{self.arch}") + + llvm_readobj = get_llvm_readobj(self.ndk_path) + if not llvm_readobj.exists(): + raise FileNotFoundError(f"[DEPLOY] {llvm_readobj} does not exist." + "Finding Qt dependencies failed") + + archive = zipfile.ZipFile(self.wheel_pyside) + lib_path_suffix = Path(str(self.qt_libs_path)).relative_to(self.wheel_pyside) + + with tempfile.TemporaryDirectory() as tmpdir: + archive.extractall(tmpdir) + qt_libs_tmpdir = Path(tmpdir) / lib_path_suffix + # find the lib folder where Qt libraries are stored + for module_name in sorted(self.modules): + qt_module_path = qt_libs_tmpdir / f"libQt6{module_name}_{self.arch}.so" + if not qt_module_path.exists(): + raise FileNotFoundError(f"[DEPLOY] libQt6{module_name}_{self.arch}.so not found" + " inside the wheel") + find_lib_dependencies(llvm_readobj=llvm_readobj, lib_path=qt_module_path, + dry_run=self.dry_run, + used_dependencies=all_dependencies) + + for dependency in all_dependencies: + match = lib_pattern.search(dependency) + if match: + module = match.group("mod_name") + if module not in self.modules: + dependent_modules.add(module) + + # check if the PySide6 binary for the Qt module actually exists + # eg: libQt6QmlModels.so exists and it includes QML types. Hence, it makes no + dependent_modules = [module for module in dependent_modules if module in + get_all_pyside_modules()] + dependent_modules_str = ",".join(dependent_modules) + logging.info("[DEPLOY] The following extra dependencies were found:" + f" {dependent_modules_str}") + + return dependent_modules + + def _find_and_set_dependency_files(self) -> List[zipfile.Path]: + """ + Based on `modules`, returns the Qt6{module}_{arch}-android-dependencies.xml file, which + contains the various dependencies of the module, like permissions, plugins etc + """ + needed_dependency_files = [(f"Qt{MAJOR_VERSION}{module}_{self.arch}" + "-android-dependencies.xml") for module in self.modules] + + for dependency_file_name in needed_dependency_files: + dependency_file = self.qt_libs_path / dependency_file_name + if dependency_file.exists(): + self._dependency_files.append(dependency_file) + + logging.info("[DEPLOY] The following dependency files were found: " + f"{*self._dependency_files,}") + + def _find_local_libs(self): + local_libs = set() + plugins = set() + lib_pattern = re.compile(f"lib(?P<lib_name>.*)_{self.arch}") + for dependency_file in self._dependency_files: + xml_content = dependency_file.read_text() + root = ET.fromstring(xml_content) + for local_lib in root.iter("lib"): + + if 'file' not in local_lib.attrib: + if 'name' not in local_lib.attrib: + logging.warning("[DEPLOY] Invalid android dependency file" + f" {str(dependency_file)}") + continue + + file = local_lib.attrib['file'] + if file.endswith(".so"): + # file_name starts with lib and ends with the platform name + # eg: lib<lib_name>_x86_64.so + file_name = Path(file).stem + + # we only need lib_name, because lib and arch gets re-added by + # python-for-android + match = lib_pattern.search(file_name) + if match: + lib_name = match.group("lib_name") + local_libs.add(lib_name) + if lib_name.startswith("plugins"): + plugin_name = lib_name.split('plugins_', 1)[1] + plugins.add(plugin_name) + + return list(local_libs), list(plugins) + + def _find_plugin_dependencies(self, dependent_plugins: List[str]): + # The `bundled` element in the dependency xml files points to the folder where + # additional dependencies for the application exists. Inspecting the depenency files + # in android, this always points to the specific Qt plugin dependency folder. + # eg: for application using Qt Multimedia, this looks like: + # <bundled file="./plugins/multimedia" /> + # The code recusively checks all these dependent folders and adds the necessary plugins + # as dependencies + lib_pattern = re.compile(f"libplugins_(?P<plugin_name>.*)_{self.arch}.so") + for dependency_file in self._dependency_files: + xml_content = dependency_file.read_text() + root = ET.fromstring(xml_content) + for bundled_element in root.iter("bundled"): + # the attribute 'file' can be misleading, but it always points to the plugin + # folder on inspecting the dependency files + if 'file' not in bundled_element.attrib: + logging.warning("[DEPLOY] Invalid Android dependency file" + f" {str(dependency_file)}") + continue + + # from "./plugins/multimedia" to absolute path in wheel + plugin_module_folder = bundled_element.attrib['file'] + # they all should start with `./plugins` + if plugin_module_folder.startswith("./plugins"): + plugin_module_folder = plugin_module_folder.partition("./plugins/")[2] + else: + continue + + absolute_plugin_module_folder = (self.qt_libs_path.parent / "plugins" + / plugin_module_folder) + + if not absolute_plugin_module_folder.is_dir(): + logging.warning(f"[DEPLOY] Qt plugin folder '{plugin_module_folder}' does not" + " exist or is not a directory for this Android platform") + continue + + for plugin in absolute_plugin_module_folder.iterdir(): + plugin_name = plugin.name + if plugin_name.endswith(".so") and plugin_name.startswith("libplugins"): + # we only need part of plugin_name, because `lib` prefix and `arch` suffix + # gets re-added by python-for-android + match = lib_pattern.search(plugin_name) + if match: + plugin_infix_name = match.group("plugin_name") + if plugin_infix_name not in dependent_plugins: + dependent_plugins.append(plugin_infix_name) + + def verify_and_set_recipe_dir(self): + # create recipes + # https://python-for-android.readthedocs.io/en/latest/recipes/ + # These recipes are manually added through buildozer.spec file to be used by + # python_for_android while building the distribution + + if not self.recipes_exist() and not self.dry_run: + logging.info("[DEPLOY] Creating p4a recipes for PySide6 and shiboken6") + version = Wheel(self.wheel_pyside).version + create_recipe(version=version, component=f"PySide{MAJOR_VERSION}", + wheel_path=self.wheel_pyside, + generated_files_path=self.generated_files_path, + qt_modules=self.modules, + local_libs=self.local_libs, + plugins=self.qt_plugins) + create_recipe(version=version, component=f"shiboken{MAJOR_VERSION}", + wheel_path=self.wheel_shiboken, + generated_files_path=self.generated_files_path) + self.recipe_dir = ((self.generated_files_path + / "recipes").resolve()) diff --git a/sources/pyside-tools/deploy_lib/android/android_helper.py b/sources/pyside-tools/deploy_lib/android/android_helper.py new file mode 100644 index 000000000..7d2f5d575 --- /dev/null +++ b/sources/pyside-tools/deploy_lib/android/android_helper.py @@ -0,0 +1,151 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import logging +import zipfile +from dataclasses import dataclass +from pathlib import Path +from typing import List, Set +from zipfile import ZipFile + +from jinja2 import Environment, FileSystemLoader + +from .. import run_command + + +@dataclass +class AndroidData: + """ + Dataclass to store all the Android data obtained through cli + """ + wheel_pyside: Path + wheel_shiboken: Path + ndk_path: Path + sdk_path: Path + + +def create_recipe(version: str, component: str, wheel_path: str, generated_files_path: Path, + qt_modules: List[str] = None, local_libs: List[str] = None, + plugins: List[str] = None): + ''' + Create python_for_android recipe for PySide6 and shiboken6 + ''' + qt_plugins = [] + if plugins: + # split plugins based on category + for plugin in plugins: + plugin_category, plugin_name = plugin.split('_', 1) + qt_plugins.append((plugin_category, plugin_name)) + + qt_local_libs = [] + if local_libs: + qt_local_libs = [local_lib for local_lib in local_libs if local_lib.startswith("Qt6")] + + rcp_tmpl_path = Path(__file__).parent / "recipes" / f"{component}" + environment = Environment(loader=FileSystemLoader(rcp_tmpl_path)) + template = environment.get_template("__init__.tmpl.py") + content = template.render( + version=version, + wheel_path=wheel_path, + qt_modules=qt_modules, + qt_local_libs=qt_local_libs, + qt_plugins=qt_plugins + ) + + recipe_path = generated_files_path / "recipes" / f"{component}" + recipe_path.mkdir(parents=True, exist_ok=True) + logging.info(f"[DEPLOY] Writing {component} recipe into {str(recipe_path)}") + with open(recipe_path / "__init__.py", mode="w", encoding="utf-8") as recipe: + recipe.write(content) + + +def extract_and_copy_jar(wheel_path: Path, generated_files_path: Path) -> str: + ''' + extracts the PySide6 wheel and copies the 'jar' folder to 'generated_files_path'. + These .jar files are added to the buildozer.spec file to be used later by buildozer + ''' + jar_path = generated_files_path / "jar" + jar_path.mkdir(parents=True, exist_ok=True) + archive = ZipFile(wheel_path) + jar_files = [file for file in archive.namelist() if file.startswith("PySide6/jar")] + for file in jar_files: + archive.extract(file, jar_path) + return (jar_path / "PySide6" / "jar").resolve() if jar_files else None + + +def get_wheel_android_arch(wheel: Path): + ''' + Get android architecture from wheel + ''' + supported_archs = ["aarch64", "armv7a", "i686", "x86_64"] + for arch in supported_archs: + if arch in wheel.stem: + return arch + + return None + + +def get_llvm_readobj(ndk_path: Path) -> Path: + ''' + Return the path to llvm_readobj from the Android Ndk + ''' + # TODO: Requires change if Windows platform supports Android Deployment or if we + # support host other than linux-x86_64 + return (ndk_path / "toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-readobj") + + +def find_lib_dependencies(llvm_readobj: Path, lib_path: Path, used_dependencies: Set[str] = None, + dry_run: bool = False): + """ + Find all the Qt dependencies of a library using llvm_readobj + """ + if lib_path.name in used_dependencies: + return + + used_dependencies.add(lib_path.name) + + command = [str(llvm_readobj), "--needed-libs", str(lib_path)] + + # even if dry_run is given, we need to run the actual command to see all the dependencies + # for which llvm-readelf is run. + if dry_run: + _, output = run_command(command=command, dry_run=dry_run, fetch_output=True) + _, output = run_command(command=command, dry_run=False, fetch_output=True) + + dependencies = set() + neededlibraries_found = False + for line in output.splitlines(): + line = line.decode("utf-8").lstrip() + if line.startswith("NeededLibraries") and not neededlibraries_found: + neededlibraries_found = True + if neededlibraries_found and line.startswith("libQt"): + dependencies.add(line) + used_dependencies.add(line) + dependent_lib_path = lib_path.parent / line + find_lib_dependencies(llvm_readobj, dependent_lib_path, used_dependencies, dry_run) + + if dependencies: + logging.info(f"[DEPLOY] Following dependencies found for {lib_path.stem}: {dependencies}") + else: + logging.info(f"[DEPLOY] No Qt dependencies found for {lib_path.stem}") + + +def find_qtlibs_in_wheel(wheel_pyside: Path): + """ + Find the path to Qt/lib folder inside the wheel. + """ + archive = ZipFile(wheel_pyside) + qt_libs_path = wheel_pyside / "PySide6/Qt/lib" + qt_libs_path = zipfile.Path(archive, at=qt_libs_path) + if not qt_libs_path.exists(): + for file in archive.namelist(): + # the dependency files are inside the libs folder + if file.endswith("android-dependencies.xml"): + qt_libs_path = zipfile.Path(archive, at=file).parent + # all dependency files are in the same path + break + + if not qt_libs_path: + raise FileNotFoundError("[DEPLOY] Unable to find Qt libs folder inside the wheel") + + return qt_libs_path diff --git a/sources/pyside-tools/deploy_lib/android/buildozer.py b/sources/pyside-tools/deploy_lib/android/buildozer.py new file mode 100644 index 000000000..0c314c356 --- /dev/null +++ b/sources/pyside-tools/deploy_lib/android/buildozer.py @@ -0,0 +1,147 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import sys +import logging +import xml.etree.ElementTree as ET +import zipfile +from pathlib import Path +from typing import List + +from . import AndroidConfig +from .. import BaseConfig, run_command + + +class BuildozerConfig(BaseConfig): + def __init__(self, buildozer_spec_file: Path, pysidedeploy_config: AndroidConfig): + super().__init__(buildozer_spec_file, comment_prefixes="#") + self.set_value("app", "title", pysidedeploy_config.title) + self.set_value("app", "package.name", pysidedeploy_config.title) + self.set_value("app", "package.domain", + f"org.{pysidedeploy_config.title}") + + include_exts = self.get_value("app", "source.include_exts") + include_exts = f"{include_exts},qml,js" + self.set_value("app", "source.include_exts", include_exts, raise_warning=False) + + self.set_value("app", "requirements", "python3,shiboken6,PySide6") + + # android platform specific + if pysidedeploy_config.ndk_path: + self.set_value("app", "android.ndk_path", str(pysidedeploy_config.ndk_path)) + + if pysidedeploy_config.sdk_path: + self.set_value("app", "android.sdk_path", str(pysidedeploy_config.sdk_path)) + + self.set_value("app", "android.archs", pysidedeploy_config.arch) + + # p4a changes + self.set_value("app", "p4a.bootstrap", "qt") + self.set_value('app', "p4a.local_recipes", str(pysidedeploy_config.recipe_dir)) + + # add p4a branch + # by default the master branch is used + # https://github.com/kivy/python-for-android/commit/b92522fab879dbfc0028966ca3c59ef46ab7767d + # has not been merged to master yet. So, we use the develop branch for now + # TODO: remove this once the above commit is merged to master + self.set_value("app", "p4a.branch", "develop") + + # add permissions + permissions = self.__find_permissions(pysidedeploy_config.dependency_files) + permissions = ", ".join(permissions) + self.set_value("app", "android.permissions", permissions) + + # add jars and initClasses for the jars + jars, init_classes = self.__find_jars(pysidedeploy_config.dependency_files, + pysidedeploy_config.jars_dir) + self.set_value("app", "android.add_jars", ",".join(jars)) + + # extra arguments specific to Qt + modules = ",".join(pysidedeploy_config.modules) + local_libs = ",".join(pysidedeploy_config.local_libs) + init_classes = ",".join(init_classes) + extra_args = (f"--qt-libs={modules} --load-local-libs={local_libs}" + f" --init-classes={init_classes}") + self.set_value("app", "p4a.extra_args", extra_args) + + # TODO: does not work atm. Seems like a bug with buildozer + # change buildozer build_dir + # self.set_value("buildozer", "build_dir", str(build_dir.relative_to(Path.cwd()))) + + # change final apk/aab path + self.set_value("buildozer", "bin_dir", str(pysidedeploy_config.exe_dir.resolve())) + + # set application icon + self.set_value("app", "icon.filename", pysidedeploy_config.icon) + + self.update_config() + + def __find_permissions(self, dependency_files: List[zipfile.Path]): + permissions = set() + for dependency_file in dependency_files: + xml_content = dependency_file.read_text() + root = ET.fromstring(xml_content) + for permission in root.iter("permission"): + permissions.add(permission.attrib['name']) + return permissions + + def __find_jars(self, dependency_files: List[zipfile.Path], jars_dir: Path): + jars, init_classes = set(), set() + for dependency_file in dependency_files: + xml_content = dependency_file.read_text() + root = ET.fromstring(xml_content) + for jar in root.iter("jar"): + jar_file = jar.attrib['file'] + if jar_file.startswith("jar/"): + jar_file_name = jar_file[4:] + if (jars_dir / jar_file_name).exists(): + jars.add(str(jars_dir / jar_file_name)) + else: + logging.warning(f"[DEPLOY] Unable to include {jar_file}. " + f"{jar_file} does not exist in {jars_dir}") + continue + else: + logging.warning(f"[DEPLOY] Unable to include {jar_file}. " + "All jar file paths should begin with 'jar/'") + continue + + jar_init_class = jar.attrib.get('initClass') + if jar_init_class: + init_classes.add(jar_init_class) + + # add the jar with all the activity and service java files + # this is created from Qt for Python instead of Qt + # The initClasses for this are already taken care of by python-for-android + android_bindings_jar = jars_dir / "Qt6AndroidBindings.jar" + if android_bindings_jar.exists(): + jars.add(str(android_bindings_jar)) + else: + raise FileNotFoundError(f"{android_bindings_jar} not found in wheel") + + return jars, init_classes + + +class Buildozer: + dry_run = False + + @staticmethod + def initialize(pysidedeploy_config: AndroidConfig): + project_dir = Path(pysidedeploy_config.project_dir) + buildozer_spec = project_dir / "buildozer.spec" + if buildozer_spec.exists(): + logging.warning(f"[DEPLOY] buildozer.spec already present in {str(project_dir)}." + "Using it") + return + + # creates buildozer.spec config file + command = [sys.executable, "-m", "buildozer", "init"] + run_command(command=command, dry_run=Buildozer.dry_run) + if not Buildozer.dry_run: + if not buildozer_spec.exists(): + raise RuntimeError(f"buildozer.spec not found in {Path.cwd()}") + BuildozerConfig(buildozer_spec, pysidedeploy_config) + + @staticmethod + def create_executable(mode: str): + command = [sys.executable, "-m", "buildozer", "android", mode] + run_command(command=command, dry_run=Buildozer.dry_run) diff --git a/sources/pyside-tools/deploy_lib/android/recipes/PySide6/__init__.tmpl.py b/sources/pyside-tools/deploy_lib/android/recipes/PySide6/__init__.tmpl.py new file mode 100644 index 000000000..8a8615798 --- /dev/null +++ b/sources/pyside-tools/deploy_lib/android/recipes/PySide6/__init__.tmpl.py @@ -0,0 +1,64 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import shutil +import zipfile +from pathlib import Path + +from pythonforandroid.logger import info +from pythonforandroid.recipe import PythonRecipe + + +class PySideRecipe(PythonRecipe): + version = '{{ version }}' + wheel_path = '{{ wheel_path }}' + depends = ["shiboken6"] + call_hostpython_via_targetpython = False + install_in_hostpython = False + + def build_arch(self, arch): + """Unzip the wheel and copy into site-packages of target""" + + info("Copying libc++_shared.so from SDK to be loaded on startup") + libcpp_path = f"{self.ctx.ndk.sysroot_lib_dir}/{arch.command_prefix}/libc++_shared.so" + shutil.copyfile(libcpp_path, Path(self.ctx.get_libs_dir(arch.arch)) / "libc++_shared.so") + + info(f"Installing {self.name} into site-packages") + with zipfile.ZipFile(self.wheel_path, "r") as zip_ref: + info("Unzip wheels and copy into {}".format(self.ctx.get_python_install_dir(arch.arch))) + zip_ref.extractall(self.ctx.get_python_install_dir(arch.arch)) + + lib_dir = Path(f"{self.ctx.get_python_install_dir(arch.arch)}/PySide6/Qt/lib") + + info("Copying Qt libraries to be loaded on startup") + shutil.copytree(lib_dir, self.ctx.get_libs_dir(arch.arch), dirs_exist_ok=True) + shutil.copyfile(lib_dir.parent.parent / "libpyside6.abi3.so", + Path(self.ctx.get_libs_dir(arch.arch)) / "libpyside6.abi3.so") + + {% for module in qt_modules %} # noqa: E999 + shutil.copyfile(lib_dir.parent.parent / f"Qt{{ module }}.abi3.so", + Path(self.ctx.get_libs_dir(arch.arch)) / "Qt{{ module }}.abi3.so") + {% if module == "Qml" -%} # noqa: E999 + shutil.copyfile(lib_dir.parent.parent / "libpyside6qml.abi3.so", + Path(self.ctx.get_libs_dir(arch.arch)) / "libpyside6qml.abi3.so") + {% endif %} # noqa: E999 + {% endfor %} # noqa: E999 + + {% for lib in qt_local_libs %} # noqa: E999 + lib_path = lib_dir / f"lib{{ lib }}_{arch.arch}.so" + if lib_path.exists(): + shutil.copyfile(lib_path, + Path(self.ctx.get_libs_dir(arch.arch)) / f"lib{{ lib }}_{arch.arch}.so") + {% endfor %} # noqa: E999 + + {% for plugin_category,plugin_name in qt_plugins %} # noqa: E999 + plugin_path = (lib_dir.parent / "plugins" / "{{ plugin_category }}" / + f"libplugins_{{ plugin_category }}_{{ plugin_name }}_{arch.arch}.so") + if plugin_path.exists(): + shutil.copyfile(plugin_path, + (Path(self.ctx.get_libs_dir(arch.arch)) / + f"libplugins_{{ plugin_category }}_{{ plugin_name }}_{arch.arch}.so")) + {% endfor %} # noqa: E999 + + +recipe = PySideRecipe() diff --git a/sources/pyside-tools/deploy_lib/android/recipes/shiboken6/__init__.tmpl.py b/sources/pyside-tools/deploy_lib/android/recipes/shiboken6/__init__.tmpl.py new file mode 100644 index 000000000..d6ab037bf --- /dev/null +++ b/sources/pyside-tools/deploy_lib/android/recipes/shiboken6/__init__.tmpl.py @@ -0,0 +1,31 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import shutil +import zipfile +from pathlib import Path + +from pythonforandroid.logger import info +from pythonforandroid.recipe import PythonRecipe + + +class ShibokenRecipe(PythonRecipe): + version = '{{ version }}' + wheel_path = '{{ wheel_path }}' + + call_hostpython_via_targetpython = False + install_in_hostpython = False + + def build_arch(self, arch): + ''' Unzip the wheel and copy into site-packages of target''' + info('Installing {} into site-packages'.format(self.name)) + with zipfile.ZipFile(self.wheel_path, 'r') as zip_ref: + info('Unzip wheels and copy into {}'.format(self.ctx.get_python_install_dir(arch.arch))) + zip_ref.extractall(self.ctx.get_python_install_dir(arch.arch)) + + lib_dir = Path(f"{self.ctx.get_python_install_dir(arch.arch)}/shiboken6") + shutil.copyfile(lib_dir / "libshiboken6.abi3.so", + Path(self.ctx.get_libs_dir(arch.arch)) / "libshiboken6.abi3.so") + + +recipe = ShibokenRecipe() diff --git a/sources/pyside-tools/deploy_lib/commands.py b/sources/pyside-tools/deploy_lib/commands.py new file mode 100644 index 000000000..3a7e2a2e2 --- /dev/null +++ b/sources/pyside-tools/deploy_lib/commands.py @@ -0,0 +1,60 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import json +import subprocess +import sys +from pathlib import Path +from typing import List + +""" +All utility functions for deployment +""" + + +def run_command(command, dry_run: bool, fetch_output: bool = False): + command_str = " ".join([str(cmd) for cmd in command]) + output = None + is_windows = (sys.platform == "win32") + try: + if not dry_run: + if fetch_output: + output = subprocess.check_output(command, shell=is_windows) + else: + subprocess.check_call(command, shell=is_windows) + else: + print(command_str + "\n") + except FileNotFoundError as error: + raise FileNotFoundError(f"[DEPLOY] {error.filename} not found") + except subprocess.CalledProcessError as error: + raise RuntimeError( + f"[DEPLOY] Command {command_str} failed with error {error} and return_code" + f"{error.returncode}" + ) + except Exception as error: + raise RuntimeError(f"[DEPLOY] Command {command_str} failed with error {error}") + + return command_str, output + + +def run_qmlimportscanner(qml_files: List[Path], dry_run: bool): + """ + Runs pyside6-qmlimportscanner to find all the imported qml modules + """ + if not qml_files: + return [] + + qml_modules = [] + cmd = ["pyside6-qmlimportscanner", "-qmlFiles"] + cmd.extend([str(qml_file) for qml_file in qml_files]) + + if dry_run: + run_command(command=cmd, dry_run=True) + + # we need to run qmlimportscanner during dry_run as well to complete the + # command being run by nuitka + _, json_string = run_command(command=cmd, dry_run=False, fetch_output=True) + json_string = json_string.decode("utf-8") + json_array = json.loads(json_string) + qml_modules = [item['name'] for item in json_array if item['type'] == "module"] + return qml_modules diff --git a/sources/pyside-tools/deploy_lib/config.py b/sources/pyside-tools/deploy_lib/config.py new file mode 100644 index 000000000..5d5070bef --- /dev/null +++ b/sources/pyside-tools/deploy_lib/config.py @@ -0,0 +1,480 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import sys +import configparser +import logging +import warnings +from configparser import ConfigParser +from typing import List +from pathlib import Path +from enum import Enum + +from project import ProjectData +from . import (DEFAULT_APP_ICON, find_pyside_modules, find_permission_categories, + QtDependencyReader, run_qmlimportscanner) + +# Some QML plugins like QtCore are excluded from this list as they don't contribute much to +# executable size. Excluding them saves the extra processing of checking for them in files +EXCLUDED_QML_PLUGINS = {"QtQuick", "QtQuick3D", "QtCharts", "QtWebEngine", "QtTest", "QtSensors"} + +PERMISSION_MAP = {"Bluetooth": "NSBluetoothAlwaysUsageDescription:BluetoothAccess", + "Camera": "NSCameraUsageDescription:CameraAccess", + "Microphone": "NSMicrophoneUsageDescription:MicrophoneAccess", + "Contacts": "NSContactsUsageDescription:ContactsAccess", + "Calendar": "NSCalendarsUsageDescription:CalendarAccess", + # for iOS NSLocationWhenInUseUsageDescription and + # NSLocationAlwaysAndWhenInUseUsageDescription are also required. + "Location": "NSLocationUsageDescription:LocationAccess", + } + + +class BaseConfig: + """Wrapper class around any .spec file with function to read and set values for the .spec file + """ + def __init__(self, config_file: Path, comment_prefixes: str = "/", + existing_config_file: bool = False) -> None: + self.config_file = config_file + self.existing_config_file = existing_config_file + self.parser = ConfigParser(comment_prefixes=comment_prefixes, strict=False, + allow_no_value=True) + self.parser.read(self.config_file) + + def update_config(self): + logging.info(f"[DEPLOY] Creating {self.config_file}") + with open(self.config_file, "w+") as config_file: + self.parser.write(config_file, space_around_delimiters=True) + + def set_value(self, section: str, key: str, new_value: str, raise_warning: bool = True): + try: + current_value = self.get_value(section, key, ignore_fail=True) + if current_value != new_value: + self.parser.set(section, key, new_value) + except configparser.NoOptionError: + if raise_warning: + logging.warning(f"[DEPLOY] Key {key} does not exist") + except configparser.NoSectionError: + if raise_warning: + logging.warning(f"[DEPLOY] Section {section} does not exist") + + def get_value(self, section: str, key: str, ignore_fail: bool = False): + try: + return self.parser.get(section, key) + except configparser.NoOptionError: + if not ignore_fail: + logging.warning(f"[DEPLOY] Key {key} does not exist") + except configparser.NoSectionError: + if not ignore_fail: + logging.warning(f"[DEPLOY] Section {section} does not exist") + + +class Config(BaseConfig): + """ + Wrapper class around pysidedeploy.spec file, whose options are used to control the executable + creation + """ + + def __init__(self, config_file: Path, source_file: Path, python_exe: Path, dry_run: bool, + existing_config_file: bool = False, extra_ignore_dirs: List[str] = None): + super().__init__(config_file=config_file, existing_config_file=existing_config_file) + + self.extra_ignore_dirs = extra_ignore_dirs + self._dry_run = dry_run + self.qml_modules = set() + # set source_file + self.source_file = Path( + self.set_or_fetch(config_property_val=source_file, config_property_key="input_file") + ).resolve() + + # set python path + self.python_path = Path( + self.set_or_fetch( + config_property_val=python_exe, + config_property_key="python_path", + config_property_group="python", + ) + ) + + self.title = self.get_value("app", "title") + + # set application icon + config_icon = self.get_value("app", "icon") + if config_icon: + self.icon = str(Path(config_icon).resolve()) + else: + self.icon = DEFAULT_APP_ICON + + self.project_dir = None + if self.get_value("app", "project_dir"): + self.project_dir = Path(self.get_value("app", "project_dir")).absolute() + else: + self._find_and_set_project_dir() + + self.exe_dir = None + if self.get_value("app", "exec_directory"): + self.exe_dir = Path(self.get_value("app", "exec_directory")).absolute() + else: + self._find_and_set_exe_dir() + + self.project_data: ProjectData = None + if self.get_value("app", "project_file"): + project_file = Path(self.get_value("app", "project_file")).absolute() + self.project_data = ProjectData(project_file=project_file) + else: + self._find_and_set_project_file() + + self.qml_files = [] + config_qml_files = self.get_value("qt", "qml_files") + if config_qml_files and self.project_dir and self.existing_config_file: + self.qml_files = [Path(self.project_dir) / file for file in config_qml_files.split(",")] + else: + self._find_and_set_qml_files() + + self.excluded_qml_plugins = [] + if self.get_value("qt", "excluded_qml_plugins") and self.existing_config_file: + self.excluded_qml_plugins = self.get_value("qt", "excluded_qml_plugins").split(",") + else: + self._find_and_set_excluded_qml_plugins() + + self._generated_files_path = self.project_dir / "deployment" + + self.modules = [] + + def set_or_fetch(self, config_property_val, config_property_key, config_property_group="app"): + """ + Write to config_file if 'config_property_key' is known without config_file + Fetch and return from config_file if 'config_property_key' is unknown, but + config_file exists + Otherwise, raise an exception + """ + if config_property_val: + self.set_value(config_property_group, config_property_key, str(config_property_val)) + return config_property_val + elif self.get_value(config_property_group, config_property_key): + return self.get_value(config_property_group, config_property_key) + else: + raise RuntimeError( + f"[DEPLOY] No {config_property_key} specified in config file or as cli option" + ) + + @property + def dry_run(self): + return self._dry_run + + @property + def generated_files_path(self): + return self._generated_files_path + + @property + def qml_files(self): + return self._qml_files + + @qml_files.setter + def qml_files(self, qml_files): + self._qml_files = qml_files + + @property + def project_dir(self): + return self._project_dir + + @project_dir.setter + def project_dir(self, project_dir): + self._project_dir = project_dir + + @property + def title(self): + return self._title + + @title.setter + def title(self, title): + self._title = title + self.set_value("app", "title", title) + + @property + def icon(self): + return self._icon + + @icon.setter + def icon(self, icon): + self._icon = icon + self.set_value("app", "icon", icon) + + @property + def source_file(self): + return self._source_file + + @source_file.setter + def source_file(self, source_file: Path): + self._source_file = source_file + + @property + def python_path(self): + return self._python_path + + @python_path.setter + def python_path(self, python_path: Path): + self._python_path = python_path + + @property + def extra_args(self): + return self.get_value("nuitka", "extra_args") + + @extra_args.setter + def extra_args(self, extra_args): + self.set_value("nuitka", "extra_args", extra_args) + + @property + def excluded_qml_plugins(self): + return self._excluded_qml_plugins + + @excluded_qml_plugins.setter + def excluded_qml_plugins(self, excluded_qml_plugins): + self._excluded_qml_plugins = excluded_qml_plugins + + @property + def exe_dir(self): + return self._exe_dir + + @exe_dir.setter + def exe_dir(self, exe_dir: Path): + self._exe_dir = exe_dir + + @property + def modules(self): + return self._modules + + @modules.setter + def modules(self, modules): + self._modules = modules + self.set_value("qt", "modules", ",".join(modules)) + + def _find_and_set_qml_files(self): + """Fetches all the qml_files in the folder and sets them if the + field qml_files is empty in the config_dir""" + + if self.project_data: + qml_files = self.project_data.qml_files + for sub_project_file in self.project_data.sub_projects_files: + qml_files.extend(ProjectData(project_file=sub_project_file).qml_files) + self.qml_files = qml_files + else: + qml_files_temp = None + if self.source_file and self.python_path: + if not self.qml_files: + qml_files_temp = list(self.source_file.parent.glob("**/*.qml")) + + # add all QML files, excluding the ones shipped with installed PySide6 + # The QML files shipped with PySide6 gets added if venv is used, + # because of recursive glob + if self.python_path.parent.parent == self.source_file.parent: + # python venv path is inside the main source dir + qml_files_temp = list( + set(qml_files_temp) - set(self.python_path.parent.parent.rglob("*.qml")) + ) + + if len(qml_files_temp) > 500: + if "site-packages" in str(qml_files_temp[-1]): + raise RuntimeError( + "You are including a lot of QML files from a local virtual env." + " This can lead to errors in deployment." + ) + else: + warnings.warn( + "You seem to include a lot of QML files. This can lead to errors in " + "deployment." + ) + + if qml_files_temp: + extra_qml_files = [Path(file) for file in qml_files_temp] + self.qml_files.extend(extra_qml_files) + if self.qml_files: + self.set_value( + "qt", + "qml_files", + ",".join([str(file.absolute().relative_to(self.project_dir)) + for file in self.qml_files]), + ) + logging.info("[DEPLOY] QML files identified and set in config_file") + + def _find_and_set_project_dir(self): + # there is no other way to find the project_dir than assume it is the parent directory + # of source_file + self.project_dir = self.source_file.parent + + # update input_file path + self.set_value("app", "input_file", str(self.source_file.relative_to(self.project_dir))) + + if self.project_dir != Path.cwd(): + self.set_value("app", "project_dir", str(self.project_dir)) + else: + self.set_value("app", "project_dir", str(self.project_dir.relative_to(Path.cwd()))) + + def _find_and_set_project_file(self): + if self.project_dir: + files = list(self.project_dir.glob("*.pyproject")) + else: + logging.exception("[DEPLOY] Project directory not set in config file") + raise + + if not files: + logging.info("[DEPLOY] No .pyproject file found. Project file not set") + elif len(files) > 1: + logging.warning("DEPLOY: More that one .pyproject files found. Project file not set") + raise + else: + self.project_data = ProjectData(files[0]) + self.set_value("app", "project_file", str(files[0].relative_to(self.project_dir))) + logging.info(f"[DEPLOY] Project file {files[0]} found and set in config file") + + def _find_and_set_excluded_qml_plugins(self): + if self.qml_files: + self.qml_modules = set(run_qmlimportscanner(qml_files=self.qml_files, + dry_run=self.dry_run)) + self.excluded_qml_plugins = EXCLUDED_QML_PLUGINS.difference(self.qml_modules) + + # needed for dry_run testing + self.excluded_qml_plugins = sorted(self.excluded_qml_plugins) + + if self.excluded_qml_plugins: + self.set_value("qt", "excluded_qml_plugins", ",".join(self.excluded_qml_plugins)) + + def _find_and_set_exe_dir(self): + if self.project_dir == Path.cwd(): + self.exe_dir = self.project_dir.relative_to(Path.cwd()) + else: + self.exe_dir = self.project_dir + self.exe_dir = Path( + self.set_or_fetch( + config_property_val=self.exe_dir, config_property_key="exec_directory" + ) + ).absolute() + + def _find_and_set_pysidemodules(self): + self.modules = find_pyside_modules(project_dir=self.project_dir, + extra_ignore_dirs=self.extra_ignore_dirs, + project_data=self.project_data) + logging.info("The following PySide modules were found from the Python files of " + f"the project {self.modules}") + + def _find_and_set_qtquick_modules(self): + """Identify if QtQuick is used in QML files and add them as dependency + """ + extra_modules = [] + if not self.qml_modules: + self.qml_modules = set(run_qmlimportscanner(qml_files=self.qml_files, + dry_run=self.dry_run)) + + if "QtQuick" in self.qml_modules: + extra_modules.append("Quick") + + if "QtQuick.Controls" in self.qml_modules: + extra_modules.append("QuickControls2") + + self.modules += extra_modules + + +class DesktopConfig(Config): + """Wrapper class around pysidedeploy.spec, but specific to Desktop deployment + """ + class NuitkaMode(Enum): + ONEFILE = "onefile" + STANDALONE = "standalone" + + def __init__(self, config_file: Path, source_file: Path, python_exe: Path, dry_run: bool, + existing_config_file: bool = False, extra_ignore_dirs: List[str] = None, + mode: str = "onefile"): + super().__init__(config_file, source_file, python_exe, dry_run, existing_config_file, + extra_ignore_dirs) + self.dependency_reader = QtDependencyReader(dry_run=self.dry_run) + if self.get_value("qt", "modules"): + self.modules = self.get_value("qt", "modules").split(",") + else: + self._find_and_set_pysidemodules() + self._find_and_set_qtquick_modules() + self._find_dependent_qt_modules() + + self._qt_plugins = [] + if self.get_value("qt", "plugins"): + self._qt_plugins = self.get_value("qt", "plugins").split(",") + else: + self.qt_plugins = self.dependency_reader.find_plugin_dependencies(self.modules, + python_exe) + + self._permissions = [] + if sys.platform == "darwin": + nuitka_macos_permissions = self.get_value("nuitka", "macos.permissions") + if nuitka_macos_permissions: + self._permissions = nuitka_macos_permissions.split(",") + else: + self._find_and_set_permissions() + + self._mode = self.NuitkaMode.ONEFILE + if self.get_value("nuitka", "mode") == self.NuitkaMode.STANDALONE.value: + self._mode = self.NuitkaMode.STANDALONE + elif mode == self.NuitkaMode.STANDALONE.value: + self.mode = self.NuitkaMode.STANDALONE + + @property + def qt_plugins(self): + return self._qt_plugins + + @qt_plugins.setter + def qt_plugins(self, qt_plugins): + self._qt_plugins = qt_plugins + self.set_value("qt", "plugins", ",".join(qt_plugins)) + + @property + def permissions(self): + return self._permissions + + @permissions.setter + def permissions(self, permissions): + self._permissions = permissions + self.set_value("nuitka", "macos.permissions", ",".join(permissions)) + + @property + def mode(self): + return self._mode + + @mode.setter + def mode(self, mode: NuitkaMode): + self._mode = mode + self.set_value("nuitka", "mode", mode.value) + + def _find_dependent_qt_modules(self): + """ + Given pysidedeploy_config.modules, find all the other dependent Qt modules. + """ + all_modules = set(self.modules) + + if not self.dependency_reader.lib_reader: + warnings.warn(f"[DEPLOY] Unable to find {self.dependency_reader.lib_reader_name}. This " + "tool helps to find the Qt module dependencies of the application. " + "Skipping checking for dependencies.", category=RuntimeWarning) + return + + for module_name in self.modules: + self.dependency_reader.find_dependencies(module=module_name, used_modules=all_modules) + + self.modules = list(all_modules) + + def _find_and_set_permissions(self): + """ + Finds and sets the usage description string required for each permission requested by the + macOS application. + """ + permissions = [] + perm_categories = find_permission_categories(project_dir=self.project_dir, + extra_ignore_dirs=self.extra_ignore_dirs, + project_data=self.project_data) + + perm_categories_str = ",".join(perm_categories) + logging.info(f"[DEPLOY] Usage descriptions for the {perm_categories_str} will be added to " + "the Info.plist file of the macOS application bundle") + + # handling permissions + for perm_category in perm_categories: + if perm_category in PERMISSION_MAP: + permissions.append(PERMISSION_MAP[perm_category]) + + self.permissions = permissions diff --git a/sources/pyside-tools/deploy_lib/default.spec b/sources/pyside-tools/deploy_lib/default.spec new file mode 100644 index 000000000..2e28b2f7c --- /dev/null +++ b/sources/pyside-tools/deploy_lib/default.spec @@ -0,0 +1,100 @@ +[app] + +# Title of your application +title = pyside_app_demo + +# Project Directory. The general assumption is that project_dir is the parent directory +# of input_file +project_dir = + +# Source file path +input_file = + +# Directory where exec is stored +exec_directory = + +# Path to .pyproject project file +project_file = + +# Application icon +icon = + +[python] + +# Python path +python_path = + +# python packages to install +# ordered-set: increase compile time performance of nuitka packaging +# zstandard: provides final executable size optimization +packages = Nuitka==2.1 + +# buildozer: for deploying Android application +android_packages = buildozer==1.5.0,cython==0.29.33 + +[qt] + +# Comma separated path to QML files required +# normally all the QML files required by the project are added automatically +qml_files = + +# excluded qml plugin binaries +excluded_qml_plugins = + +# Qt modules used. Comma separated +modules = + +# Qt plugins used by the application +plugins = + +[android] + +# path to PySide wheel +wheel_pyside = + +# path to Shiboken wheel +wheel_shiboken = + +# plugins to be copied to libs folder of the packaged application. Comma separated +plugins = + +[nuitka] + +# usage description for permissions requested by the app as found in the Info.plist file +# of the app bundle +# eg: NSCameraUsageDescription:CameraAccess +macos.permissions = + +# mode of using Nuitka. Accepts standalone or onefile. Default is onefile. +mode = onefile + +# (str) specify any extra nuitka arguments +# eg: extra_args = --show-modules --follow-stdlib +extra_args = --quiet --noinclude-qt-translations + +[buildozer] + +# build mode +# possible options: [release, debug] +# release creates an aab, while debug creates an apk +mode = debug + +# contrains path to PySide6 and shiboken6 recipe dir +recipe_dir = + +# path to extra Qt Android jars to be loaded by the application +jars_dir = + +# if empty uses default ndk path downloaded by buildozer +ndk_path = + +# if empty uses default sdk path downloaded by buildozer +sdk_path = + +# other libraries to be loaded. Comma separated. +# loaded at app startup +local_libs = + +# architecture of deployed platform +# possible values: ["aarch64", "armv7a", "i686", "x86_64"] +arch = diff --git a/sources/pyside-tools/deploy_lib/dependency_util.py b/sources/pyside-tools/deploy_lib/dependency_util.py new file mode 100644 index 000000000..2d5b188d3 --- /dev/null +++ b/sources/pyside-tools/deploy_lib/dependency_util.py @@ -0,0 +1,319 @@ +# Copyright (C) 2024 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import ast +import re +import os +import site +import json +import warnings +import logging +import shutil +import sys +from pathlib import Path +from typing import List, Set +from functools import lru_cache + +from . import IMPORT_WARNING_PYSIDE, run_command + + +@lru_cache(maxsize=None) +def get_py_files(project_dir: Path, extra_ignore_dirs: List[Path] = None, project_data=None): + """Finds and returns all the Python files in the project + """ + py_candidates = [] + ignore_dirs = ["__pycache__", "env", "venv", "deployment"] + + if project_data: + py_candidates = project_data.python_files + ui_candidates = project_data.ui_files + qrc_candidates = project_data.qrc_files + + def add_uic_qrc_candidates(candidates, candidate_type): + possible_py_candidates = [(file.parent / f"{candidate_type}_{file.stem}.py") + for file in candidates + if (file.parent / f"{candidate_type}_{file.stem}.py").exists() + ] + + if len(possible_py_candidates) != len(candidates): + warnings.warn(f"[DEPLOY] The number of {candidate_type} files and their " + "corresponding Python files don't match.", + category=RuntimeWarning) + + py_candidates.extend(possible_py_candidates) + + if ui_candidates: + add_uic_qrc_candidates(ui_candidates, "ui") + + if qrc_candidates: + add_uic_qrc_candidates(qrc_candidates, "qrc") + + return py_candidates + + # incase there is not .pyproject file, search all python files in project_dir, except + # ignore_dirs + if extra_ignore_dirs: + ignore_dirs.extend(extra_ignore_dirs) + + # find relevant .py files + _walk = os.walk(project_dir) + for root, dirs, files in _walk: + dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")] + for py_file in files: + if py_file.endswith(".py"): + py_candidates.append(Path(root) / py_file) + + return py_candidates + + +@lru_cache(maxsize=None) +def get_ast(py_file: Path): + """Given a Python file returns the abstract syntax tree + """ + contents = py_file.read_text(encoding="utf-8") + try: + tree = ast.parse(contents) + except SyntaxError: + print(f"[DEPLOY] Unable to parse {py_file}") + return tree + + +def find_permission_categories(project_dir: Path, extra_ignore_dirs: List[Path] = None, + project_data=None): + """Given the project directory, finds all the permission categories required by the + project. eg: Camera, Bluetooth, Contacts etc. + + Note: This function is only relevant for mac0S deployment. + """ + all_perm_categories = set() + mod_pattern = re.compile("Q(?P<mod_name>.*)Permission") + + def pyside_permission_imports(py_file: Path): + perm_categories = [] + try: + tree = get_ast(py_file) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + main_mod_name = node.module + if main_mod_name == "PySide6.QtCore": + # considers 'from PySide6.QtCore import QtMicrophonePermission' + for imported_module in node.names: + full_mod_name = imported_module.name + match = mod_pattern.search(full_mod_name) + if match: + mod_name = match.group("mod_name") + perm_categories.append(mod_name) + continue + + if isinstance(node, ast.Import): + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name == "PySide6": + logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file))) + except Exception as e: + raise RuntimeError(f"[DEPLOY] Finding permission categories failed on file " + f"{str(py_file)} with error {e}") + + return set(perm_categories) + + py_candidates = get_py_files(project_dir, extra_ignore_dirs, project_data) + for py_candidate in py_candidates: + all_perm_categories = all_perm_categories.union(pyside_permission_imports(py_candidate)) + + if not all_perm_categories: + ValueError("[DEPLOY] No permission categories were found for macOS app bundle creation.") + + return all_perm_categories + + +def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None, + project_data=None): + """ + Searches all the python files in the project to find all the PySide modules used by + the application. + """ + all_modules = set() + mod_pattern = re.compile("PySide6.Qt(?P<mod_name>.*)") + + def pyside_module_imports(py_file: Path): + modules = [] + try: + tree = get_ast(py_file) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + main_mod_name = node.module + if main_mod_name.startswith("PySide6"): + if main_mod_name == "PySide6": + # considers 'from PySide6 import QtCore' + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name.startswith("Qt"): + modules.append(full_mod_name[2:]) + continue + + # considers 'from PySide6.QtCore import Qt' + match = mod_pattern.search(main_mod_name) + if match: + mod_name = match.group("mod_name") + modules.append(mod_name) + else: + logging.warning(( + f"[DEPLOY] Unable to find module name from {ast.dump(node)}")) + + if isinstance(node, ast.Import): + for imported_module in node.names: + full_mod_name = imported_module.name + if full_mod_name == "PySide6": + logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file))) + except Exception as e: + raise RuntimeError(f"[DEPLOY] Finding module import failed on file {str(py_file)} with " + f"error {e}") + + return set(modules) + + py_candidates = get_py_files(project_dir, extra_ignore_dirs, project_data) + for py_candidate in py_candidates: + all_modules = all_modules.union(pyside_module_imports(py_candidate)) + + if not all_modules: + ValueError("[DEPLOY] No PySide6 modules were found") + + return list(all_modules) + + +class QtDependencyReader: + def __init__(self, dry_run: bool = False) -> None: + self.dry_run = dry_run + self.lib_reader_name = None + self.qt_module_path_pattern = None + self.lib_pattern = None + self.command = None + self.qt_libs_dir = None + + if sys.platform == "linux": + self.lib_reader_name = "readelf" + self.qt_module_path_pattern = "libQt6{module}.so.6" + self.lib_pattern = re.compile("libQt6(?P<mod_name>.*).so.6") + self.command_args = "-d" + elif sys.platform == "darwin": + self.lib_reader_name = "dyld_info" + self.qt_module_path_pattern = "Qt{module}.framework/Versions/A/Qt{module}" + self.lib_pattern = re.compile("@rpath/Qt(?P<mod_name>.*).framework/Versions/A/") + self.command_args = "-dependents" + elif sys.platform == "win32": + self.lib_reader_name = "dumpbin" + self.qt_module_path_pattern = "Qt6{module}.dll" + self.lib_pattern = re.compile("Qt6(?P<mod_name>.*).dll") + self.command_args = "/dependents" + else: + print(f"[DEPLOY] Deployment on unsupported platfrom {sys.platform}") + sys.exit(1) + + self.pyside_install_dir = None + self.qt_libs_dir = self.get_qt_libs_dir() + self._lib_reader = shutil.which(self.lib_reader_name) + + def get_qt_libs_dir(self): + """ + Finds the path to the Qt libs directory inside PySide6 package installation + """ + for possible_site_package in site.getsitepackages(): + if possible_site_package.endswith("site-packages"): + self.pyside_install_dir = Path(possible_site_package) / "PySide6" + + if not self.pyside_install_dir: + print("Unable to find site-packages. Exiting ...") + sys.exit(-1) + + if sys.platform == "win32": + return self.pyside_install_dir + + return self.pyside_install_dir / "Qt" / "lib" # for linux and macOS + + @property + def lib_reader(self): + return self._lib_reader + + def find_dependencies(self, module: str, used_modules: Set[str] = None): + """ + Given a Qt module, find all the other Qt modules it is dependent on and add it to the + 'used_modules' set + """ + qt_module_path = self.qt_libs_dir / self.qt_module_path_pattern.format(module=module) + if not qt_module_path.exists(): + warnings.warn(f"[DEPLOY] {qt_module_path.name} not found in {str(qt_module_path)}." + "Skipping finding its dependencies.", category=RuntimeWarning) + return + + lib_pattern = re.compile(self.lib_pattern) + command = [self.lib_reader, self.command_args, str(qt_module_path)] + # print the command if dry_run is True. + # Normally run_command is going to print the command in dry_run mode. But, this is a + # special case where we need to print the command as well as to run it. + if self.dry_run: + command_str = " ".join(command) + print(command_str + "\n") + + # We need to run this even for dry run, to see the full Nuitka command being executed + _, output = run_command(command=command, dry_run=False, fetch_output=True) + + dependent_modules = set() + for line in output.splitlines(): + line = line.decode("utf-8").lstrip() + if sys.platform == "darwin": + if line.endswith(f"Qt{module} [arm64]:"): + # macOS Qt frameworks bundles have both x86_64 and arm64 architectures + # We only need to consider one as the dependencies are redundant + break + elif line.endswith(f"Qt{module} [X86_64]:"): + # this line needs to be skipped because it matches with the pattern + # and is related to the module itself, not the dependencies of the module + continue + elif sys.platform == "win32" and line.startswith("Summary"): + # the dependencies would be found before the `Summary` line + break + match = lib_pattern.search(line) + if match: + dep_module = match.group("mod_name") + dependent_modules.add(dep_module) + if dep_module not in used_modules: + used_modules.add(dep_module) + self.find_dependencies(module=dep_module, used_modules=used_modules) + + if dependent_modules: + logging.info(f"[DEPLOY] Following dependencies found for {module}: {dependent_modules}") + else: + logging.info(f"[DEPLOY] No Qt dependencies found for {module}") + + def find_plugin_dependencies(self, used_modules: List[str], python_exe: Path) -> List[str]: + """ + Given the modules used by the application, returns all the required plugins + """ + plugins = set() + pyside_wheels = ["PySide6_Essentials", "PySide6_Addons"] + # TODO from 3.12 use list(dist.name for dist in importlib.metadata.distributions()) + _, installed_packages = run_command(command=[str(python_exe), "-m", "pip", "freeze"], + dry_run=False, fetch_output=True) + installed_packages = [p.decode().split('==')[0] for p in installed_packages.split()] + for pyside_wheel in pyside_wheels: + if pyside_wheel not in installed_packages: + # the wheel is not installed and hence no plugins are checked for its modules + logging.warning((f"[DEPLOY] The package {pyside_wheel} is not installed. ")) + continue + pyside_mod_plugin_json_name = f"{pyside_wheel}.json" + pyside_mod_plugin_json_file = self.pyside_install_dir / pyside_mod_plugin_json_name + if not pyside_mod_plugin_json_file.exists(): + warnings.warn(f"[DEPLOY] Unable to find {pyside_mod_plugin_json_file}.", + category=RuntimeWarning) + continue + + # convert the json to dict + pyside_mod_dict = {} + with open(pyside_mod_plugin_json_file) as pyside_json: + pyside_mod_dict = json.load(pyside_json) + + # find all the plugins in the modules + for module in used_modules: + plugins.update(pyside_mod_dict.get(module, [])) + + return list(plugins) diff --git a/sources/pyside-tools/deploy_lib/deploy_util.py b/sources/pyside-tools/deploy_lib/deploy_util.py new file mode 100644 index 000000000..1e0e2712a --- /dev/null +++ b/sources/pyside-tools/deploy_lib/deploy_util.py @@ -0,0 +1,81 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import logging +import shutil +import sys +from pathlib import Path + +from . import EXE_FORMAT +from .config import Config, DesktopConfig + + +def config_option_exists(): + for argument in sys.argv: + if any(item in argument for item in ["--config-file", "-c"]): + return True + + return False + + +def cleanup(config: Config, is_android: bool = False): + """ + Cleanup the generated build folders/files + """ + if config.generated_files_path.exists(): + shutil.rmtree(config.generated_files_path) + logging.info("[DEPLOY] Deployment directory purged") + + if is_android: + buildozer_spec: Path = config.project_dir / "buildozer.spec" + if buildozer_spec.exists(): + buildozer_spec.unlink() + logging.info(f"[DEPLOY] {str(buildozer_spec)} removed") + + buildozer_build: Path = config.project_dir / ".buildozer" + if buildozer_build.exists(): + shutil.rmtree(buildozer_build) + logging.info(f"[DEPLOY] {str(buildozer_build)} removed") + + +def create_config_file(dry_run: bool = False, config_file: Path = None, main_file: Path = None): + """ + Sets up a new pysidedeploy.spec or use an existing config file + """ + + if main_file: + if main_file.parent != Path.cwd(): + config_file = main_file.parent / "pysidedeploy.spec" + else: + config_file = Path.cwd() / "pysidedeploy.spec" + + logging.info(f"[DEPLOY] Creating config file {config_file}") + if not dry_run: + shutil.copy(Path(__file__).parent / "default.spec", config_file) + + # the config parser needs a reference to parse. So, in the case of --dry-run + # use the default.spec file. + if dry_run: + config_file = Path(__file__).parent / "default.spec" + + return config_file + + +def finalize(config: DesktopConfig): + """ + Copy the executable into the final location + For Android deployment, this is done through buildozer + """ + dist_format = EXE_FORMAT + if config.mode == DesktopConfig.NuitkaMode.STANDALONE and sys.platform != "darwin": + dist_format = ".dist" + + generated_exec_path = config.generated_files_path / (config.source_file.stem + dist_format) + if generated_exec_path.exists() and config.exe_dir: + if sys.platform == "darwin" or config.mode == DesktopConfig.NuitkaMode.STANDALONE: + shutil.copytree(generated_exec_path, config.exe_dir / (config.title + dist_format), + dirs_exist_ok=True) + else: + shutil.copy(generated_exec_path, config.exe_dir) + print("[DEPLOY] Executed file created in " + f"{str(config.exe_dir / (config.source_file.stem + dist_format))}") diff --git a/sources/pyside-tools/deploy_lib/nuitka_helper.py b/sources/pyside-tools/deploy_lib/nuitka_helper.py new file mode 100644 index 000000000..5d0e9032f --- /dev/null +++ b/sources/pyside-tools/deploy_lib/nuitka_helper.py @@ -0,0 +1,124 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +# enables to use typehints for classes that has not been defined yet or imported +# used for resolving circular imports +from __future__ import annotations +import logging +import os +import sys +from pathlib import Path +from typing import List + +from . import MAJOR_VERSION, run_command +from .config import DesktopConfig + + +class Nuitka: + """ + Wrapper class around the nuitka executable, enabling its usage through python code + """ + + def __init__(self, nuitka): + self.nuitka = nuitka + # plugins to ignore. The sensible plugins are include by default by Nuitka for PySide6 + # application deployment + self.qt_plugins_to_ignore = ["imageformats", # being Nuitka `sensible`` plugins + "iconengines", + "mediaservice", + "printsupport", + "platforms", + "platformthemes", + "styles", + "wayland-shell-integration", + "wayland-decoration-client", + "wayland-graphics-integration-client", + "egldeviceintegrations", + "xcbglintegrations", + "tls", # end Nuitka `sensible` plugins + "generic" # plugins that error with Nuitka + ] + + # .webp are considered to be dlls by Nuitka instead of data files causing + # the packaging to fail + # https://github.com/Nuitka/Nuitka/issues/2854 + # TODO: Remove .webp when the issue is fixed + self.files_to_ignore = [".cpp.o", ".qsb", ".webp"] + + @staticmethod + def icon_option(): + if sys.platform == "linux": + return "--linux-icon" + elif sys.platform == "win32": + return "--windows-icon-from-ico" + else: + return "--macos-app-icon" + + def create_executable(self, source_file: Path, extra_args: str, qml_files: List[Path], + qt_plugins: List[str], excluded_qml_plugins: List[str], icon: str, + dry_run: bool, permissions: List[str], + mode: DesktopConfig.NuitkaMode): + qt_plugins = [plugin for plugin in qt_plugins if plugin not in self.qt_plugins_to_ignore] + extra_args = extra_args.split() + + # macOS uses the --standalone option by default to create an app bundle + if sys.platform == "darwin": + # create an app bundle + extra_args.extend(["--standalone", "--macos-create-app-bundle"]) + permission_pattern = "--macos-app-protected-resource={permission}" + for permission in permissions: + extra_args.append(permission_pattern.format(permission=permission)) + else: + extra_args.append(f"--{mode.value}") + + qml_args = [] + if qml_files: + # This will generate options for each file using: + # --include-data-files=ABSOLUTE_PATH_TO_FILE=RELATIVE_PATH_TO ROOT + # for each file. This will preserve the directory structure of QML resources. + qml_args.extend( + [f"--include-data-files={qml_file.resolve()}=" + f"./{qml_file.resolve().relative_to(source_file.parent)}" + for qml_file in qml_files] + ) + # add qml plugin. The `qml`` plugin name is not present in the module json files shipped + # with Qt and hence not in `qt_plugins``. However, Nuitka uses the 'qml' plugin name to + # include the necessary qml plugins. There we have to add it explicitly for a qml + # application + qt_plugins.append("qml") + + if excluded_qml_plugins: + prefix = "lib" if sys.platform != "win32" else "" + for plugin in excluded_qml_plugins: + dll_name = plugin.replace("Qt", f"Qt{MAJOR_VERSION}") + qml_args.append(f"--noinclude-dlls={prefix}{dll_name}*") + + # Exclude .qen json files from QtQuickEffectMaker + # These files are not relevant for PySide6 applications + qml_args.append("--noinclude-dlls=*/qml/QtQuickEffectMaker/*") + + # Exclude files that cannot be processed by Nuitka + for file in self.files_to_ignore: + extra_args.append(f"--noinclude-dlls=*{file}") + + output_dir = source_file.parent / "deployment" + if not dry_run: + output_dir.mkdir(parents=True, exist_ok=True) + logging.info("[DEPLOY] Running Nuitka") + command = self.nuitka + [ + os.fspath(source_file), + "--follow-imports", + "--enable-plugin=pyside6", + f"--output-dir={output_dir}", + ] + + command.extend(extra_args + qml_args) + command.append(f"{self.__class__.icon_option()}={icon}") + if qt_plugins: + # sort qt_plugins so that the result is definitive when testing + qt_plugins.sort() + qt_plugins_str = ",".join(qt_plugins) + command.append(f"--include-qt-plugins={qt_plugins_str}") + + command_str, _ = run_command(command=command, dry_run=dry_run) + return command_str diff --git a/sources/pyside-tools/deploy_lib/pyside_icon.icns b/sources/pyside-tools/deploy_lib/pyside_icon.icns Binary files differnew file mode 100644 index 000000000..a6eb02bb0 --- /dev/null +++ b/sources/pyside-tools/deploy_lib/pyside_icon.icns diff --git a/sources/pyside-tools/deploy_lib/pyside_icon.ico b/sources/pyside-tools/deploy_lib/pyside_icon.ico Binary files differnew file mode 100644 index 000000000..332a3a568 --- /dev/null +++ b/sources/pyside-tools/deploy_lib/pyside_icon.ico diff --git a/sources/pyside-tools/deploy_lib/pyside_icon.jpg b/sources/pyside-tools/deploy_lib/pyside_icon.jpg Binary files differnew file mode 100644 index 000000000..647c42c71 --- /dev/null +++ b/sources/pyside-tools/deploy_lib/pyside_icon.jpg diff --git a/sources/pyside-tools/deploy_lib/python_helper.py b/sources/pyside-tools/deploy_lib/python_helper.py new file mode 100644 index 000000000..7cbf323ed --- /dev/null +++ b/sources/pyside-tools/deploy_lib/python_helper.py @@ -0,0 +1,122 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import logging +import os +import sys + +from importlib import util +from importlib.metadata import version +from pathlib import Path + +from . import Config, run_command + + +class PythonExecutable: + """ + Wrapper class around Python executable + """ + + def __init__(self, python_path: Path = None, dry_run: bool = False, init: bool = False, + force: bool = False): + + self.dry_run = dry_run + self.init = init + if not python_path: + response = "yes" + # checking if inside virtual environment + if not self.is_venv() and not force and not self.dry_run and not self.init: + response = input(("You are not using a virtual environment. pyside6-deploy needs " + "to install a few Python packages for deployment to work " + "seamlessly. \n Proceed? [Y/n]")) + + if response.lower() in ["no", "n"]: + print("[DEPLOY] Exiting ...") + sys.exit(0) + + self.exe = Path(sys.executable) + else: + self.exe = python_path + + logging.info(f"[DEPLOY] Using Python at {str(self.exe)}") + + @property + def exe(self): + return Path(self._exe) + + @exe.setter + def exe(self, exe): + self._exe = exe + + @staticmethod + def is_venv(): + venv = os.environ.get("VIRTUAL_ENV") + return True if venv else False + + def is_pyenv_python(self): + pyenv_root = os.environ.get("PYENV_ROOT") + + if pyenv_root: + resolved_exe = self.exe.resolve() + if str(resolved_exe).startswith(pyenv_root): + return True + + return False + + def install(self, packages: list = None): + _, installed_packages = run_command(command=[str(self.exe), "-m", "pip", "freeze"], + dry_run=False, fetch_output=True) + installed_packages = [p.decode().split('==')[0] for p in installed_packages.split()] + for package in packages: + package_info = package.split('==') + package_components_len = len(package_info) + package_name, package_version = None, None + if package_components_len == 1: + package_name = package_info[0] + elif package_components_len == 2: + package_name = package_info[0] + package_version = package_info[1] + else: + raise ValueError(f"{package} should be of the format 'package_name'=='version'") + if (package_name not in installed_packages) and (not self.is_installed(package_name)): + logging.info(f"[DEPLOY] Installing package: {package}") + run_command( + command=[self.exe, "-m", "pip", "install", package], + dry_run=self.dry_run, + ) + elif package_version: + installed_version = version(package_name) + if package_version != installed_version: + logging.info(f"[DEPLOY] Installing package: {package_name}" + f"version: {package_version}") + run_command( + command=[self.exe, "-m", "pip", "install", "--force", package], + dry_run=self.dry_run, + ) + else: + logging.info(f"[DEPLOY] package: {package_name}=={package_version}" + " already installed") + else: + logging.info(f"[DEPLOY] package: {package_name} already installed") + + def is_installed(self, package): + return bool(util.find_spec(package)) + + def install_dependencies(self, config: Config, packages: str, is_android: bool = False): + """ + Installs the python package dependencies for the target deployment platform + """ + packages = config.get_value("python", packages).split(",") + if not self.init: + # install packages needed for deployment + logging.info("[DEPLOY] Installing dependencies") + self.install(packages=packages) + # nuitka requires patchelf to make patchelf rpath changes for some Qt files + if sys.platform.startswith("linux") and not is_android: + self.install(packages=["patchelf"]) + elif is_android: + # install only buildozer + logging.info("[DEPLOY] Installing buildozer") + buildozer_package_with_version = ([package for package in packages + if package.startswith("buildozer")]) + self.install(packages=list(buildozer_package_with_version)) diff --git a/sources/pyside-tools/metaobjectdump.py b/sources/pyside-tools/metaobjectdump.py new file mode 100644 index 000000000..0970f9974 --- /dev/null +++ b/sources/pyside-tools/metaobjectdump.py @@ -0,0 +1,452 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import ast +import json +import os +import sys +import tokenize +from argparse import ArgumentParser, RawTextHelpFormatter +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple, Union + + +DESCRIPTION = """Parses Python source code to create QObject metatype +information in JSON format for qmltyperegistrar.""" + + +REVISION = 68 + + +CPP_TYPE_MAPPING = {"str": "QString"} + + +QML_IMPORT_NAME = "QML_IMPORT_NAME" +QML_IMPORT_MAJOR_VERSION = "QML_IMPORT_MAJOR_VERSION" +QML_IMPORT_MINOR_VERSION = "QML_IMPORT_MINOR_VERSION" +QT_MODULES = "QT_MODULES" + + +ITEM_MODELS = ["QAbstractListModel", "QAbstractProxyModel", + "QAbstractTableModel", "QConcatenateTablesProxyModel", + "QFileSystemModel", "QIdentityProxyModel", "QPdfBookmarkModel", + "QPdfSearchModel", "QSortFilterProxyModel", "QSqlQueryModel", + "QStandardItemModel", "QStringListModel", "QTransposeProxyModel", + "QWebEngineHistoryModel"] + + +QOBJECT_DERIVED = ["QObject", "QQuickItem", "QQuickPaintedItem"] + ITEM_MODELS + + +AstDecorator = Union[ast.Name, ast.Call] +AstPySideTypeSpec = Union[ast.Name, ast.Constant] + + +ClassList = List[dict] + + +PropertyEntry = Dict[str, Union[str, int, bool]] + +Argument = Dict[str, str] +Arguments = List[Argument] +Signal = Dict[str, Union[str, Arguments]] +Slot = Dict[str, Union[str, Arguments]] + + +def _decorator(name: str, value: str) -> Dict[str, str]: + """Create a QML decorator JSON entry""" + return {"name": name, "value": value} + + +def _attribute(node: ast.Attribute) -> Tuple[str, str]: + """Split an attribute.""" + return node.value.id, node.attr + + +def _name(node: Union[ast.Name, ast.Attribute]) -> str: + """Return the name of something that is either an attribute or a name, + such as base classes or call.func""" + if isinstance(node, ast.Attribute): + qualifier, name = _attribute(node) + return f"{qualifier}.{node.attr}" + return node.id + + +def _func_name(node: ast.Call) -> str: + return _name(node.func) + + +def _python_to_cpp_type(type: str) -> str: + """Python to C++ type""" + c = CPP_TYPE_MAPPING.get(type) + return c if c else type + + +def _parse_property_kwargs(keywords: List[ast.keyword], prop: PropertyEntry): + """Parse keyword arguments of @Property""" + for k in keywords: + if k.arg == "notify": + prop["notify"] = _name(k.value) + + +def _parse_assignment(node: ast.Assign) -> Tuple[Optional[str], Optional[ast.AST]]: + """Parse an assignment and return a tuple of name, value.""" + if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + var_name = node.targets[0].id + return (var_name, node.value) + return (None, None) + + +def _parse_pyside_type(type_spec: AstPySideTypeSpec) -> str: + """Parse type specification of a Slot/Property decorator. Usually a type, + but can also be a string constant with a C++ type name.""" + if isinstance(type_spec, ast.Constant): + return type_spec.value + return _python_to_cpp_type(_name(type_spec)) + + +def _parse_call_args(call: ast.Call): + """Parse arguments of a Signal call/Slot decorator (type list).""" + result: Arguments = [] + for n, arg in enumerate(call.args): + par_name = f"a{n+1}" + par_type = _parse_pyside_type(arg) + result.append({"name": par_name, "type": par_type}) + return result + + +def _parse_slot(func_name: str, call: ast.Call) -> Slot: + """Parse a 'Slot' decorator.""" + return_type = "void" + for kwarg in call.keywords: + if kwarg.arg == "result": + return_type = _python_to_cpp_type(_name(kwarg.value)) + break + return {"access": "public", "name": func_name, + "arguments": _parse_call_args(call), + "returnType": return_type} + + +class VisitorContext: + """Stores a list of QObject-derived classes encountered in order to find + out which classes inherit QObject.""" + + def __init__(self): + self.qobject_derived = QOBJECT_DERIVED + + +class MetaObjectDumpVisitor(ast.NodeVisitor): + """AST visitor for parsing sources and creating the data structure for + JSON.""" + + def __init__(self, context: VisitorContext): + super().__init__() + self._context = context + self._json_class_list: ClassList = [] + # Property by name, which will be turned into the JSON List later + self._properties: List[PropertyEntry] = [] + self._signals: List[Signal] = [] + self._within_class: bool = False + self._qt_modules: Set[str] = set() + self._qml_import_name = "" + self._qml_import_major_version = 0 + self._qml_import_minor_version = 0 + + def json_class_list(self) -> ClassList: + return self._json_class_list + + def qml_import_name(self) -> str: + return self._qml_import_name + + def qml_import_version(self) -> Tuple[int, int]: + return (self._qml_import_major_version, self._qml_import_minor_version) + + def qt_modules(self): + return sorted(self._qt_modules) + + @staticmethod + def create_ast(filename: Path) -> ast.Module: + """Create an Abstract Syntax Tree on which a visitor can be run""" + node = None + with tokenize.open(filename) as file: + node = ast.parse(file.read(), mode="exec") + return node + + def visit_Assign(self, node: ast.Assign): + """Parse the global constants for QML-relevant values""" + var_name, value_node = _parse_assignment(node) + if not var_name or not isinstance(value_node, ast.Constant): + return + value = value_node.value + if var_name == QML_IMPORT_NAME: + self._qml_import_name = value + elif var_name == QML_IMPORT_MAJOR_VERSION: + self._qml_import_major_version = value + elif var_name == QML_IMPORT_MINOR_VERSION: + self._qml_import_minor_version = value + + def visit_ClassDef(self, node: ast.Module): + """Visit a class definition""" + self._properties = [] + self._signals = [] + self._slots = [] + self._within_class = True + qualified_name = node.name + last_dot = qualified_name.rfind('.') + name = (qualified_name[last_dot + 1:] if last_dot != -1 + else qualified_name) + + data = {"className": name, + "qualifiedClassName": qualified_name} + + q_object = False + bases = [] + for b in node.bases: + # PYSIDE-2202: catch weird constructs like "class C(type(Base)):" + if isinstance(b, ast.Name): + base_name = _name(b) + if base_name in self._context.qobject_derived: + q_object = True + self._context.qobject_derived.append(name) + base_dict = {"access": "public", "name": base_name} + bases.append(base_dict) + + data["object"] = q_object + if bases: + data["superClasses"] = bases + + class_decorators: List[dict] = [] + for d in node.decorator_list: + self._parse_class_decorator(d, class_decorators) + + if class_decorators: + data["classInfos"] = class_decorators + + for b in node.body: + if isinstance(b, ast.Assign): + self._parse_class_variable(b) + else: + self.visit(b) + + if self._properties: + data["properties"] = self._properties + + if self._signals: + data["signals"] = self._signals + + if self._slots: + data["slots"] = self._slots + + self._json_class_list.append(data) + + self._within_class = False + + def visit_FunctionDef(self, node): + if self._within_class: + for d in node.decorator_list: + self._parse_function_decorator(node.name, d) + + def _parse_class_decorator(self, node: AstDecorator, + class_decorators: List[dict]): + """Parse ClassInfo decorators.""" + if isinstance(node, ast.Call): + name = _func_name(node) + if name == "QmlUncreatable": + class_decorators.append(_decorator("QML.Creatable", "false")) + if node.args: + reason = node.args[0].value + if isinstance(reason, str): + d = _decorator("QML.UncreatableReason", reason) + class_decorators.append(d) + elif name == "QmlAttached" and len(node.args) == 1: + d = _decorator("QML.Attached", node.args[0].id) + class_decorators.append(d) + elif name == "QmlExtended" and len(node.args) == 1: + d = _decorator("QML.Extended", node.args[0].id) + class_decorators.append(d) + elif name == "ClassInfo" and node.keywords: + kw = node.keywords[0] + class_decorators.append(_decorator(kw.arg, kw.value.value)) + elif name == "QmlForeign" and len(node.args) == 1: + d = _decorator("QML.Foreign", node.args[0].id) + class_decorators.append(d) + elif name == "QmlNamedElement" and node.args: + name = node.args[0].value + class_decorators.append(_decorator("QML.Element", name)) + elif name.startswith('Q'): + print('Unknown decorator with parameters:', name, + file=sys.stderr) + return + + if isinstance(node, ast.Name): + name = node.id + if name == "QmlElement": + class_decorators.append(_decorator("QML.Element", "auto")) + elif name == "QmlSingleton": + class_decorators.append(_decorator("QML.Singleton", "true")) + elif name == "QmlAnonymous": + class_decorators.append(_decorator("QML.Element", "anonymous")) + elif name.startswith('Q'): + print('Unknown decorator:', name, file=sys.stderr) + return + + def _index_of_property(self, name: str) -> int: + """Search a property by name""" + for i in range(len(self._properties)): + if self._properties[i]["name"] == name: + return i + return -1 + + def _create_property_entry(self, name: str, type: str, + getter: Optional[str] = None) -> PropertyEntry: + """Create a property JSON entry.""" + result: PropertyEntry = {"name": name, "type": type, + "index": len(self._properties)} + if getter: + result["read"] = getter + return result + + def _parse_function_decorator(self, func_name: str, node: AstDecorator): + """Parse function decorators.""" + if isinstance(node, ast.Attribute): + name = node.value.id + value = node.attr + if value == "setter": # Property setter + idx = self._index_of_property(name) + if idx != -1: + self._properties[idx]["write"] = func_name + return + + if isinstance(node, ast.Call): + name = _name(node.func) + if name == "Property": # Property getter + if node.args: # 1st is type/type string + type = _parse_pyside_type(node.args[0]) + prop = self._create_property_entry(func_name, type, + func_name) + _parse_property_kwargs(node.keywords, prop) + self._properties.append(prop) + elif name == "Slot": + self._slots.append(_parse_slot(func_name, node)) + else: + print('Unknown decorator with parameters:', name, + file=sys.stderr) + + def _parse_class_variable(self, node: ast.Assign): + """Parse a class variable assignment (Property, Signal, etc.)""" + (var_name, call) = _parse_assignment(node) + if not var_name or not isinstance(node.value, ast.Call): + return + func_name = _func_name(call) + if func_name == "Signal" or func_name == "QtCore.Signal": + signal: Signal = {"access": "public", "name": var_name, + "arguments": _parse_call_args(call), + "returnType": "void"} + self._signals.append(signal) + elif func_name == "Property" or func_name == "QtCore.Property": + type = _python_to_cpp_type(call.args[0].id) + prop = self._create_property_entry(var_name, type, call.args[1].id) + if len(call.args) > 2: + prop["write"] = call.args[2].id + _parse_property_kwargs(call.keywords, prop) + self._properties.append(prop) + elif func_name == "ListProperty" or func_name == "QtCore.ListProperty": + type = _python_to_cpp_type(call.args[0].id) + type = f"QQmlListProperty<{type}>" + prop = self._create_property_entry(var_name, type) + self._properties.append(prop) + + def visit_Import(self, node): + for n in node.names: # "import PySide6.QtWidgets" + self._handle_import(n.name) + + def visit_ImportFrom(self, node): + if "." in node.module: # "from PySide6.QtWidgets import QWidget" + self._handle_import(node.module) + elif node.module == "PySide6": # "from PySide6 import QtWidgets" + for n in node.names: + if n.name.startswith("Qt"): + self._qt_modules.add(n.name) + + def _handle_import(self, mod: str): + if mod.startswith("PySide6."): + self._qt_modules.add(mod[8:]) + + +def create_arg_parser(desc: str) -> ArgumentParser: + parser = ArgumentParser(description=desc, + formatter_class=RawTextHelpFormatter) + parser.add_argument('--compact', '-c', action='store_true', + help='Use compact format') + parser.add_argument('--suppress-file', '-s', action='store_true', + help='Suppress inputFile entry (for testing)') + parser.add_argument('--quiet', '-q', action='store_true', + help='Suppress warnings') + parser.add_argument('files', type=str, nargs="+", + help='Python source file') + parser.add_argument('--out-file', '-o', type=str, + help='Write output to file rather than stdout') + return parser + + +def parse_file(file: Path, context: VisitorContext, + suppress_file: bool = False) -> Optional[Dict]: + """Parse a file and return its json data""" + ast_tree = MetaObjectDumpVisitor.create_ast(file) + visitor = MetaObjectDumpVisitor(context) + visitor.visit(ast_tree) + + class_list = visitor.json_class_list() + if not class_list: + return None + result = {"classes": class_list, + "outputRevision": REVISION} + + # Non-standard QML-related values for pyside6-build usage + if visitor.qml_import_name(): + result[QML_IMPORT_NAME] = visitor.qml_import_name() + qml_import_version = visitor.qml_import_version() + if qml_import_version[0]: + result[QML_IMPORT_MAJOR_VERSION] = qml_import_version[0] + result[QML_IMPORT_MINOR_VERSION] = qml_import_version[1] + + qt_modules = visitor.qt_modules() + if qt_modules: + result[QT_MODULES] = qt_modules + + if not suppress_file: + result["inputFile"] = os.fspath(file).replace("\\", "/") + return result + + +if __name__ == '__main__': + arg_parser = create_arg_parser(DESCRIPTION) + args = arg_parser.parse_args() + + context = VisitorContext() + json_list = [] + + for file_name in args.files: + file = Path(file_name).resolve() + if not file.is_file(): + print(f'{file_name} does not exist or is not a file.', + file=sys.stderr) + sys.exit(-1) + + try: + json_data = parse_file(file, context, args.suppress_file) + if json_data: + json_list.append(json_data) + elif not args.quiet: + print(f"No classes found in {file_name}", file=sys.stderr) + except (AttributeError, SyntaxError) as e: + reason = str(e) + print(f"Error parsing {file_name}: {reason}", file=sys.stderr) + raise + + indent = None if args.compact else 4 + if args.out_file: + with open(args.out_file, 'w') as f: + json.dump(json_list, f, indent=indent) + else: + json.dump(json_list, sys.stdout, indent=indent) diff --git a/sources/pyside-tools/project.py b/sources/pyside-tools/project.py new file mode 100644 index 000000000..3706a2985 --- /dev/null +++ b/sources/pyside-tools/project.py @@ -0,0 +1,300 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + + +""" +Builds a '.pyproject' file + +Builds Qt Designer forms, resource files and QML type files + +Deploys the application by creating an executable for the corresponding platform + +For each entry in a '.pyproject' file: +- <name>.pyproject: Recurse to handle subproject +- <name>.qrc : Runs the resource compiler to create a file rc_<name>.py +- <name>.ui : Runs the user interface compiler to create a file ui_<name>.py + +For a Python file declaring a QML module, a directory matching the URI is +created and populated with .qmltypes and qmldir files for use by code analysis +tools. Currently, only one QML module consisting of several classes can be +handled per project file. +""" +import sys +import os +from typing import List, Tuple, Optional +from pathlib import Path +from argparse import ArgumentParser, RawTextHelpFormatter + +from project import (QmlProjectData, check_qml_decorators, is_python_file, + QMLDIR_FILE, MOD_CMD, METATYPES_JSON_SUFFIX, + SHADER_SUFFIXES, TRANSLATION_SUFFIX, + requires_rebuild, run_command, remove_path, + ProjectData, resolve_project_file, new_project, + ProjectType, ClOptions) + +MODE_HELP = """build Builds the project +run Builds the project and runs the first file") +clean Cleans the build artifacts") +qmllint Runs the qmllint tool +deploy Deploys the application +lupdate Updates translation (.ts) files +new-ui Creates a new QtWidgets project with a Qt Designer-based main window +new-widget Creates a new QtWidgets project with a main window +new-quick Creates a new QtQuick project +""" + +UIC_CMD = "pyside6-uic" +RCC_CMD = "pyside6-rcc" +LRELEASE_CMD = "pyside6-lrelease" +LUPDATE_CMD = "pyside6-lupdate" +QMLTYPEREGISTRAR_CMD = "pyside6-qmltyperegistrar" +QMLLINT_CMD = "pyside6-qmllint" +QSB_CMD = "pyside6-qsb" +DEPLOY_CMD = "pyside6-deploy" + +NEW_PROJECT_TYPES = {"new-quick": ProjectType.QUICK, + "new-ui": ProjectType.WIDGET_FORM, + "new-widget": ProjectType.WIDGET} + + +def _sort_sources(files: List[Path]) -> List[Path]: + """Sort the sources for building, ensure .qrc is last since it might depend + on generated files.""" + + def key_func(p: Path): + return p.suffix if p.suffix != ".qrc" else ".zzzz" + + return sorted(files, key=key_func) + + +class Project: + """ + Class to wrap the various operations on Project + """ + def __init__(self, project_file: Path): + self.project = ProjectData(project_file=project_file) + self.cl_options = ClOptions() + + # Files for QML modules using the QmlElement decorators + self._qml_module_sources: List[Path] = [] + self._qml_module_dir: Optional[Path] = None + self._qml_dir_file: Optional[Path] = None + self._qml_project_data = QmlProjectData() + self._qml_module_check() + + def _qml_module_check(self): + """Run a pre-check on Python source files and find the ones with QML + decorators (representing a QML module).""" + # Quick check for any QML files (to avoid running moc for no reason). + if not self.cl_options.qml_module and not self.project.qml_files: + return + for file in self.project.files: + if is_python_file(file): + has_class, data = check_qml_decorators(file) + if has_class: + self._qml_module_sources.append(file) + if data: + self._qml_project_data = data + + if not self._qml_module_sources: + return + if not self._qml_project_data: + print("Detected QML-decorated files, " "but was unable to detect QML_IMPORT_NAME") + sys.exit(1) + + self._qml_module_dir = self.project.project_file.parent + for uri_dir in self._qml_project_data.import_name.split("."): + self._qml_module_dir /= uri_dir + print(self._qml_module_dir) + self._qml_dir_file = self._qml_module_dir / QMLDIR_FILE + + if not self.cl_options.quiet: + count = len(self._qml_module_sources) + print(f"{self.project.project_file.name}, {count} QML file(s)," + f" {self._qml_project_data}") + + def _get_artifacts(self, file: Path) -> Tuple[List[Path], Optional[List[str]]]: + """Return path and command for a file's artifact""" + if file.suffix == ".ui": # Qt form files + py_file = f"{file.parent}/ui_{file.stem}.py" + return ([Path(py_file)], [UIC_CMD, os.fspath(file), "--rc-prefix", "-o", py_file]) + if file.suffix == ".qrc": # Qt resources + py_file = f"{file.parent}/rc_{file.stem}.py" + return ([Path(py_file)], [RCC_CMD, os.fspath(file), "-o", py_file]) + # generate .qmltypes from sources with Qml decorators + if file.suffix == ".py" and file in self._qml_module_sources: + assert self._qml_module_dir + qml_module_dir = os.fspath(self._qml_module_dir) + json_file = f"{qml_module_dir}/{file.stem}{METATYPES_JSON_SUFFIX}" + return ([Path(json_file)], [MOD_CMD, "-o", json_file, os.fspath(file)]) + # Run qmltyperegistrar + if file.name.endswith(METATYPES_JSON_SUFFIX): + assert self._qml_module_dir + stem = file.name[: len(file.name) - len(METATYPES_JSON_SUFFIX)] + qmltypes_file = self._qml_module_dir / f"{stem}.qmltypes" + cpp_file = self._qml_module_dir / f"{stem}_qmltyperegistrations.cpp" + cmd = [QMLTYPEREGISTRAR_CMD, "--generate-qmltypes", + os.fspath(qmltypes_file), "-o", os.fspath(cpp_file), + os.fspath(file)] + cmd.extend(self._qml_project_data.registrar_options()) + return ([qmltypes_file, cpp_file], cmd) + + if file.name.endswith(TRANSLATION_SUFFIX): + qm_file = f"{file.parent}/{file.stem}.qm" + cmd = [LRELEASE_CMD, os.fspath(file), "-qm", qm_file] + return ([Path(qm_file)], cmd) + + if file.suffix in SHADER_SUFFIXES: + qsb_file = f"{file.parent}/{file.stem}.qsb" + cmd = [QSB_CMD, "-o", qsb_file, os.fspath(file)] + return ([Path(qsb_file)], cmd) + + return ([], None) + + def _regenerate_qmldir(self): + """Regenerate the 'qmldir' file.""" + if self.cl_options.dry_run or not self._qml_dir_file: + return + if self.cl_options.force or requires_rebuild(self._qml_module_sources, self._qml_dir_file): + with self._qml_dir_file.open("w") as qf: + qf.write(f"module {self._qml_project_data.import_name}\n") + for f in self._qml_module_dir.glob("*.qmltypes"): + qf.write(f"typeinfo {f.name}\n") + + def _build_file(self, source: Path): + """Build an artifact.""" + artifacts, command = self._get_artifacts(source) + for artifact in artifacts: + if self.cl_options.force or requires_rebuild([source], artifact): + run_command(command, cwd=self.project.project_file.parent) + self._build_file(artifact) # Recurse for QML (json->qmltypes) + + def build(self): + """Build.""" + for sub_project_file in self.project.sub_projects_files: + Project(project_file=sub_project_file).build() + if self._qml_module_dir: + self._qml_module_dir.mkdir(exist_ok=True, parents=True) + for file in _sort_sources(self.project.files): + self._build_file(file) + self._regenerate_qmldir() + + def run(self): + """Runs the project""" + self.build() + cmd = [sys.executable, str(self.project.main_file)] + run_command(cmd, cwd=self.project.project_file.parent) + + def _clean_file(self, source: Path): + """Clean an artifact.""" + artifacts, command = self._get_artifacts(source) + for artifact in artifacts: + remove_path(artifact) + self._clean_file(artifact) # Recurse for QML (json->qmltypes) + + def clean(self): + """Clean build artifacts.""" + for sub_project_file in self.project.sub_projects_files: + Project(project_file=sub_project_file).clean() + for file in self.project.files: + self._clean_file(file) + if self._qml_module_dir and self._qml_module_dir.is_dir(): + remove_path(self._qml_module_dir) + # In case of a dir hierarchy ("a.b" -> a/b), determine and delete + # the root directory + if self._qml_module_dir.parent != self.project.project_file.parent: + project_dir_parts = len(self.project.project_file.parent.parts) + first_module_dir = self._qml_module_dir.parts[project_dir_parts] + remove_path(self.project.project_file.parent / first_module_dir) + + def _qmllint(self): + """Helper for running qmllint on .qml files (non-recursive).""" + if not self.project.qml_files: + print(f"{self.project.project_file.name}: No QML files found", file=sys.stderr) + return + + cmd = [QMLLINT_CMD] + if self._qml_dir_file: + cmd.extend(["-i", os.fspath(self._qml_dir_file)]) + for f in self.project.qml_files: + cmd.append(os.fspath(f)) + run_command(cmd, cwd=self.project.project_file.parent, ignore_fail=True) + + def qmllint(self): + """Run qmllint on .qml files.""" + self.build() + for sub_project_file in self.project.sub_projects_files: + Project(project_file=sub_project_file)._qmllint() + self._qmllint() + + def deploy(self): + """Deploys the application""" + cmd = [DEPLOY_CMD] + cmd.extend([str(self.project.main_file), "-f"]) + run_command(cmd, cwd=self.project.project_file.parent) + + def lupdate(self): + for sub_project_file in self.project.sub_projects_files: + Project(project_file=sub_project_file).lupdate() + + if not self.project.ts_files: + print(f"{self.project.project_file.name}: No .ts file found.", + file=sys.stderr) + return + + source_files = self.project.python_files + self.project.ui_files + cmd_prefix = [LUPDATE_CMD] + [p.name for p in source_files] + cmd_prefix.append("-ts") + for ts_file in self.project.ts_files: + if requires_rebuild(source_files, ts_file): + cmd = cmd_prefix + cmd.append(ts_file.name) + run_command(cmd, cwd=self.project.project_file.parent) + + +if __name__ == "__main__": + parser = ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter) + parser.add_argument("--quiet", "-q", action="store_true", help="Quiet") + parser.add_argument("--dry-run", "-n", action="store_true", help="Only print commands") + parser.add_argument("--force", "-f", action="store_true", help="Force rebuild") + parser.add_argument("--qml-module", "-Q", action="store_true", + help="Perform check for QML module") + mode_choices = ["build", "run", "clean", "qmllint", "deploy", "lupdate"] + mode_choices.extend(NEW_PROJECT_TYPES.keys()) + parser.add_argument("mode", choices=mode_choices, default="build", + type=str, help=MODE_HELP) + parser.add_argument("file", help="Project file", nargs="?", type=str) + + options = parser.parse_args() + cl_options = ClOptions(dry_run=options.dry_run, quiet=options.quiet, force=options.force, + qml_module=options.qml_module) + + mode = options.mode + + new_project_type = NEW_PROJECT_TYPES.get(mode) + if new_project_type: + if not options.file: + print(f"{mode} requires a directory name.", file=sys.stderr) + sys.exit(1) + sys.exit(new_project(options.file, new_project_type)) + + project_file = resolve_project_file(options.file) + if not project_file: + print(f"Cannot determine project_file {options.file}", file=sys.stderr) + sys.exit(1) + project = Project(project_file) + if mode == "build": + project.build() + elif mode == "run": + project.run() + elif mode == "clean": + project.clean() + elif mode == "qmllint": + project.qmllint() + elif mode == "deploy": + project.deploy() + elif mode == "lupdate": + project.lupdate() + else: + print(f"Invalid mode {mode}", file=sys.stderr) + sys.exit(1) diff --git a/sources/pyside-tools/project.pyproject b/sources/pyside-tools/project.pyproject new file mode 100644 index 000000000..346ef0465 --- /dev/null +++ b/sources/pyside-tools/project.pyproject @@ -0,0 +1,4 @@ +{ + "files": ["project.py", "project/__init__.py", "project/newproject.py", + "project/project_data.py", "project/utils.py"] +} diff --git a/sources/pyside-tools/project/__init__.py b/sources/pyside-tools/project/__init__.py new file mode 100644 index 000000000..e57a9ff88 --- /dev/null +++ b/sources/pyside-tools/project/__init__.py @@ -0,0 +1,46 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +from dataclasses import dataclass + +QTPATHS_CMD = "qtpaths6" +MOD_CMD = "pyside6-metaobjectdump" + +PROJECT_FILE_SUFFIX = ".pyproject" +QMLDIR_FILE = "qmldir" + +QML_IMPORT_NAME = "QML_IMPORT_NAME" +QML_IMPORT_MAJOR_VERSION = "QML_IMPORT_MAJOR_VERSION" +QML_IMPORT_MINOR_VERSION = "QML_IMPORT_MINOR_VERSION" +QT_MODULES = "QT_MODULES" + +METATYPES_JSON_SUFFIX = "metatypes.json" +TRANSLATION_SUFFIX = ".ts" +SHADER_SUFFIXES = ".vert", ".frag" + + +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + return cls._instances[cls] + + +@dataclass(frozen=True) +class ClOptions(metaclass=Singleton): + """ + Dataclass to store the cl options that needs to be passed as arguments. + """ + dry_run: bool + quiet: bool + force: bool + qml_module: bool + + +from .utils import (run_command, requires_rebuild, remove_path, package_dir, qtpaths, + qt_metatype_json_dir, resolve_project_file) +from .project_data import (is_python_file, ProjectData, QmlProjectData, + check_qml_decorators) +from .newproject import new_project, ProjectType diff --git a/sources/pyside-tools/project/newproject.py b/sources/pyside-tools/project/newproject.py new file mode 100644 index 000000000..c363a9fc0 --- /dev/null +++ b/sources/pyside-tools/project/newproject.py @@ -0,0 +1,165 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import json +import os +import sys +from enum import Enum +from pathlib import Path +from typing import List, Tuple + +"""New project generation code.""" + + +Project = List[Tuple[str, str]] # tuple of (filename, contents). + + +class ProjectType(Enum): + WIDGET_FORM = 1 + WIDGET = 2 + QUICK = 3 + + +_WIDGET_MAIN = """if __name__ == '__main__': + app = QApplication(sys.argv) + window = MainWindow() + window.show() + sys.exit(app.exec()) +""" + + +_WIDGET_IMPORTS = """import sys +from PySide6.QtWidgets import QApplication, QMainWindow +""" + + +_WIDGET_CLASS_DEFINITION = """class MainWindow(QMainWindow): + def __init__(self): + super().__init__() +""" + + +_WIDGET_SETUP_UI_CODE = """ self._ui = Ui_MainWindow() + self._ui.setupUi(self) +""" + + +_MAINWINDOW_FORM = """<?xml version="1.0" encoding="UTF-8"?> +<ui version="4.0"> + <class>MainWindow</class> + <widget class="QMainWindow" name="MainWindow"> + <property name="geometry"> + <rect> + <x>0</x> + <y>0</y> + <width>800</width> + <height>600</height> + </rect> + </property> + <property name="windowTitle"> + <string>MainWindow</string> + </property> + <widget class="QWidget" name="centralwidget"/> + <widget class="QMenuBar" name="menubar"> + <property name="geometry"> + <rect> + <x>0</x> + <y>0</y> + <width>800</width> + <height>22</height> + </rect> + </property> + </widget> + <widget class="QStatusBar" name="statusbar"/> + </widget> +</ui> +""" + + +_QUICK_FORM = """import QtQuick +import QtQuick.Controls + +ApplicationWindow { + id: window + width: 1024 + height: 600 + visible: true +} +""" + +_QUICK_MAIN = """import sys +from pathlib import Path + +from PySide6.QtGui import QGuiApplication +from PySide6.QtCore import QUrl +from PySide6.QtQml import QQmlApplicationEngine + + +if __name__ == "__main__": + app = QGuiApplication() + engine = QQmlApplicationEngine() + qml_file = Path(__file__).parent / 'main.qml' + engine.load(QUrl.fromLocalFile(qml_file)) + if not engine.rootObjects(): + sys.exit(-1) + exit_code = app.exec() + del engine + sys.exit(exit_code) +""" + + +def _write_project(directory: Path, files: Project): + """Write out the project.""" + file_list = [] + for file, contents in files: + (directory / file).write_text(contents) + print(f"Wrote {directory.name}{os.sep}{file}.") + file_list.append(file) + pyproject = {"files": file_list} + pyproject_file = f"{directory}.pyproject" + (directory / pyproject_file).write_text(json.dumps(pyproject)) + print(f"Wrote {directory.name}{os.sep}{pyproject_file}.") + + +def _widget_project() -> Project: + """Create a (form-less) widgets project.""" + main_py = (_WIDGET_IMPORTS + "\n\n" + _WIDGET_CLASS_DEFINITION + "\n\n" + + _WIDGET_MAIN) + return [("main.py", main_py)] + + +def _ui_form_project() -> Project: + """Create a Qt Designer .ui form based widgets project.""" + main_py = (_WIDGET_IMPORTS + + "\nfrom ui_mainwindow import Ui_MainWindow\n\n\n" + + _WIDGET_CLASS_DEFINITION + _WIDGET_SETUP_UI_CODE + + "\n\n" + _WIDGET_MAIN) + return [("main.py", main_py), + ("mainwindow.ui", _MAINWINDOW_FORM)] + + +def _qml_project() -> Project: + """Create a QML project.""" + return [("main.py", _QUICK_MAIN), + ("main.qml", _QUICK_FORM)] + + +def new_project(directory_s: str, + project_type: ProjectType = ProjectType.WIDGET_FORM) -> int: + directory = Path(directory_s) + if directory.exists(): + print(f"{directory_s} already exists.", file=sys.stderr) + return -1 + directory.mkdir(parents=True) + + if project_type == ProjectType.WIDGET_FORM: + project = _ui_form_project() + elif project_type == ProjectType.QUICK: + project = _qml_project() + else: + project = _widget_project() + _write_project(directory, project) + if project_type == ProjectType.WIDGET_FORM: + print(f'Run "pyside6-project build {directory_s}" to build the project') + print(f'Run "python {directory.name}{os.sep}main.py" to run the project') + return 0 diff --git a/sources/pyside-tools/project/project_data.py b/sources/pyside-tools/project/project_data.py new file mode 100644 index 000000000..52e20be3f --- /dev/null +++ b/sources/pyside-tools/project/project_data.py @@ -0,0 +1,244 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import json +import os +import subprocess +import sys +from typing import List, Tuple +from pathlib import Path +from . import (METATYPES_JSON_SUFFIX, PROJECT_FILE_SUFFIX, TRANSLATION_SUFFIX, + qt_metatype_json_dir, MOD_CMD, QML_IMPORT_MAJOR_VERSION, + QML_IMPORT_MINOR_VERSION, QML_IMPORT_NAME, QT_MODULES) + + +def is_python_file(file: Path) -> bool: + return (file.suffix == ".py" + or sys.platform == "win32" and file.suffix == ".pyw") + + +class ProjectData: + def __init__(self, project_file: Path) -> None: + """Parse the project.""" + self._project_file = project_file + self._sub_projects_files: List[Path] = [] + + # All sources except subprojects + self._files: List[Path] = [] + # QML files + self._qml_files: List[Path] = [] + # Python files + self.main_file: Path = None + self._python_files: List[Path] = [] + # ui files + self._ui_files: List[Path] = [] + # qrc files + self._qrc_files: List[Path] = [] + # ts files + self._ts_files: List[Path] = [] + + with project_file.open("r") as pyf: + pyproject = json.load(pyf) + for f in pyproject["files"]: + file = Path(project_file.parent / f) + if file.suffix == PROJECT_FILE_SUFFIX: + self._sub_projects_files.append(file) + else: + self._files.append(file) + if file.suffix == ".qml": + self._qml_files.append(file) + elif is_python_file(file): + if file.stem == "main": + self.main_file = file + self._python_files.append(file) + elif file.suffix == ".ui": + self._ui_files.append(file) + elif file.suffix == ".qrc": + self._qrc_files.append(file) + elif file.suffix == TRANSLATION_SUFFIX: + self._ts_files.append(file) + + if not self.main_file: + self._find_main_file() + + @property + def project_file(self): + return self._project_file + + @property + def files(self): + return self._files + + @property + def main_file(self): + return self._main_file + + @main_file.setter + def main_file(self, main_file): + self._main_file = main_file + + @property + def python_files(self): + return self._python_files + + @property + def ui_files(self): + return self._ui_files + + @property + def qrc_files(self): + return self._qrc_files + + @property + def qml_files(self): + return self._qml_files + + @property + def ts_files(self): + return self._ts_files + + @property + def sub_projects_files(self): + return self._sub_projects_files + + def _find_main_file(self) -> str: + """Find the entry point file containing the main function""" + + def is_main(file): + return "__main__" in file.read_text(encoding="utf-8") + + if not self.main_file: + for python_file in self.python_files: + if is_main(python_file): + self.main_file = python_file + return str(python_file) + + # __main__ not found + print( + "Python file with main function not found. Add the file to" f" {self.project_file}", + file=sys.stderr, + ) + sys.exit(1) + + +class QmlProjectData: + """QML relevant project data.""" + + def __init__(self): + self._import_name: str = "" + self._import_major_version: int = 0 + self._import_minor_version: int = 0 + self._qt_modules: List[str] = [] + + def registrar_options(self): + result = [ + "--import-name", + self._import_name, + "--major-version", + str(self._import_major_version), + "--minor-version", + str(self._import_minor_version), + ] + if self._qt_modules: + # Add Qt modules as foreign types + foreign_files: List[str] = [] + meta_dir = qt_metatype_json_dir() + for mod in self._qt_modules: + mod_id = mod[2:].lower() + pattern = f"qt6{mod_id}_*" + if sys.platform != "win32": + pattern += "_" # qt6core_debug_metatypes.json (Linux) + pattern += METATYPES_JSON_SUFFIX + for f in meta_dir.glob(pattern): + foreign_files.append(os.fspath(f)) + break + if foreign_files: + foreign_files_str = ",".join(foreign_files) + result.append(f"--foreign-types={foreign_files_str}") + return result + + @property + def import_name(self): + return self._import_name + + @import_name.setter + def import_name(self, n): + self._import_name = n + + @property + def import_major_version(self): + return self._import_major_version + + @import_major_version.setter + def import_major_version(self, v): + self._import_major_version = v + + @property + def import_minor_version(self): + return self._import_minor_version + + @import_minor_version.setter + def import_minor_version(self, v): + self._import_minor_version = v + + @property + def qt_modules(self): + return self._qt_modules + + @qt_modules.setter + def qt_modules(self, v): + self._qt_modules = v + + def __str__(self) -> str: + vmaj = self._import_major_version + vmin = self._import_minor_version + return f'"{self._import_name}" v{vmaj}.{vmin}' + + def __bool__(self) -> bool: + return len(self._import_name) > 0 and self._import_major_version > 0 + + +def _has_qml_decorated_class(class_list: List) -> bool: + """Check for QML-decorated classes in the moc json output.""" + for d in class_list: + class_infos = d.get("classInfos") + if class_infos: + for e in class_infos: + if "QML" in e["name"]: + return True + return False + + +def check_qml_decorators(py_file: Path) -> Tuple[bool, QmlProjectData]: + """Check if a Python file has QML-decorated classes by running a moc check + and return whether a class was found and the QML data.""" + data = None + try: + cmd = [MOD_CMD, "--quiet", os.fspath(py_file)] + with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: + data = json.load(proc.stdout) + proc.wait() + except Exception as e: + t = type(e).__name__ + print(f"{t}: running {MOD_CMD} on {py_file}: {e}", file=sys.stderr) + sys.exit(1) + + qml_project_data = QmlProjectData() + if not data: + return (False, qml_project_data) # No classes in file + + first = data[0] + class_list = first["classes"] + has_class = _has_qml_decorated_class(class_list) + if has_class: + v = first.get(QML_IMPORT_NAME) + if v: + qml_project_data.import_name = v + v = first.get(QML_IMPORT_MAJOR_VERSION) + if v: + qml_project_data.import_major_version = v + qml_project_data.import_minor_version = first.get(QML_IMPORT_MINOR_VERSION) + v = first.get(QT_MODULES) + if v: + qml_project_data.qt_modules = v + return (has_class, qml_project_data) diff --git a/sources/pyside-tools/project/utils.py b/sources/pyside-tools/project/utils.py new file mode 100644 index 000000000..d2bff65af --- /dev/null +++ b/sources/pyside-tools/project/utils.py @@ -0,0 +1,107 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import sys +import subprocess +from pathlib import Path +from typing import List, Dict, Optional + +from . import QTPATHS_CMD, PROJECT_FILE_SUFFIX, ClOptions + + +def run_command(command: List[str], cwd: str = None, ignore_fail: bool = False): + """Run a command observing quiet/dry run""" + cloptions = ClOptions() + if not cloptions.quiet or cloptions.dry_run: + print(" ".join(command)) + if not cloptions.dry_run: + ex = subprocess.call(command, cwd=cwd) + if ex != 0 and not ignore_fail: + sys.exit(ex) + + +def requires_rebuild(sources: List[Path], artifact: Path) -> bool: + """Returns whether artifact needs to be rebuilt depending on sources""" + if not artifact.is_file(): + return True + artifact_mod_time = artifact.stat().st_mtime + for source in sources: + if source.stat().st_mtime > artifact_mod_time: + return True + return False + + +def _remove_path_recursion(path: Path): + """Recursion to remove a file or directory.""" + if path.is_file(): + path.unlink() + elif path.is_dir(): + for item in path.iterdir(): + _remove_path_recursion(item) + path.rmdir() + + +def remove_path(path: Path): + """Remove path (file or directory) observing opt_dry_run.""" + cloptions = ClOptions() + if not path.exists(): + return + if not cloptions.quiet: + print(f"Removing {path.name}...") + if cloptions.dry_run: + return + _remove_path_recursion(path) + + +def package_dir() -> Path: + """Return the PySide6 root.""" + return Path(__file__).resolve().parents[2] + + +_qtpaths_info: Dict[str, str] = {} + + +def qtpaths() -> Dict[str, str]: + """Run qtpaths and return a dict of values.""" + global _qtpaths_info + if not _qtpaths_info: + output = subprocess.check_output([QTPATHS_CMD, "--query"]) + for line in output.decode("utf-8").split("\n"): + tokens = line.strip().split(":", maxsplit=1) # "Path=C:\..." + if len(tokens) == 2: + _qtpaths_info[tokens[0]] = tokens[1] + return _qtpaths_info + + +_qt_metatype_json_dir: Optional[Path] = None + + +def qt_metatype_json_dir() -> Path: + """Return the location of the Qt QML metatype files.""" + global _qt_metatype_json_dir + if not _qt_metatype_json_dir: + qt_dir = package_dir() + if sys.platform != "win32": + qt_dir /= "Qt" + metatypes_dir = qt_dir / "metatypes" + if metatypes_dir.is_dir(): # Fully installed case + _qt_metatype_json_dir = metatypes_dir + else: + # Fallback for distro builds/development. + print( + f"Falling back to {QTPATHS_CMD} to determine metatypes directory.", file=sys.stderr + ) + _qt_metatype_json_dir = Path(qtpaths()["QT_INSTALL_ARCHDATA"]) / "metatypes" + return _qt_metatype_json_dir + + +def resolve_project_file(cmdline: str) -> Optional[Path]: + """Return the project file from the command line value, either + from the file argument or directory""" + project_file = Path(cmdline).resolve() if cmdline else Path.cwd() + if project_file.is_file(): + return project_file + if project_file.is_dir(): + for m in project_file.glob(f"*{PROJECT_FILE_SUFFIX}"): + return m + return None diff --git a/sources/pyside-tools/pyside_tool.py b/sources/pyside-tools/pyside_tool.py new file mode 100644 index 000000000..b369be8a2 --- /dev/null +++ b/sources/pyside-tools/pyside_tool.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import importlib +import os +import subprocess +import sys +import sysconfig +from pathlib import Path + +import PySide6 as ref_mod + +VIRTUAL_ENV = "VIRTUAL_ENV" + + +def is_pyenv_python(): + pyenv_root = os.environ.get("PYENV_ROOT") + + if pyenv_root: + resolved_exe = Path(sys.executable).resolve() + if str(resolved_exe).startswith(pyenv_root): + return True + + return False + + +def is_virtual_env(): + return sys.prefix != sys.base_prefix + + +def init_virtual_env(): + """PYSIDE-2251: Enable running from a non-activated virtual environment + as is the case for Visual Studio Code by setting the VIRTUAL_ENV + variable which is used by the Qt Designer plugin.""" + if is_virtual_env() and not os.environ.get(VIRTUAL_ENV): + os.environ[VIRTUAL_ENV] = sys.prefix + + +def main(): + # This will take care of "pyside6-lupdate" listed as an entrypoint + # in setup.py are copied to 'scripts/..' + cmd = os.path.join("..", os.path.basename(sys.argv[0])) + command = [os.path.join(os.path.dirname(os.path.realpath(__file__)), cmd)] + command.extend(sys.argv[1:]) + sys.exit(subprocess.call(command)) + + +def qt_tool_wrapper(qt_tool, args, libexec=False): + # Taking care of pyside6-uic, pyside6-rcc, and pyside6-designer + # listed as an entrypoint in setup.py + pyside_dir = Path(ref_mod.__file__).resolve().parent + if libexec and sys.platform != "win32": + exe = pyside_dir / 'Qt' / 'libexec' / qt_tool + else: + exe = pyside_dir / qt_tool + + cmd = [os.fspath(exe)] + args + returncode = subprocess.call(cmd) + if returncode != 0: + command = ' '.join(cmd) + print(f"'{command}' returned {returncode}", file=sys.stderr) + sys.exit(returncode) + + +def pyside_script_wrapper(script_name): + """Launch a script shipped with PySide.""" + script = Path(__file__).resolve().parent / script_name + command = [sys.executable, os.fspath(script)] + sys.argv[1:] + sys.exit(subprocess.call(command)) + + +def ui_tool_binary(binary): + """Return the binary of a UI tool (App bundle on macOS).""" + if sys.platform != "darwin": + return binary + name = binary[0:1].upper() + binary[1:] + return f"{name}.app/Contents/MacOS/{name}" + + +def lrelease(): + qt_tool_wrapper("lrelease", sys.argv[1:]) + + +def lupdate(): + qt_tool_wrapper("lupdate", sys.argv[1:]) + + +def uic(): + qt_tool_wrapper("uic", ['-g', 'python'] + sys.argv[1:], True) + + +def rcc(): + args = [] + user_args = sys.argv[1:] + if "--binary" not in user_args: + args.extend(['-g', 'python']) + args.extend(user_args) + qt_tool_wrapper("rcc", args, True) + + +def qmltyperegistrar(): + qt_tool_wrapper("qmltyperegistrar", sys.argv[1:], True) + + +def qmlimportscanner(): + qt_tool_wrapper("qmlimportscanner", sys.argv[1:], True) + + +def qmlcachegen(): + qt_tool_wrapper("qmlcachegen", sys.argv[1:], True) + + +def qmllint(): + qt_tool_wrapper("qmllint", sys.argv[1:]) + + +def qmlformat(): + qt_tool_wrapper("qmlformat", sys.argv[1:]) + + +def qmlls(): + qt_tool_wrapper("qmlls", sys.argv[1:]) + + +def assistant(): + qt_tool_wrapper(ui_tool_binary("assistant"), sys.argv[1:]) + + +def _extend_path_var(var, value, prepend=False): + env_value = os.environ.get(var) + if env_value: + env_value = (f'{value}{os.pathsep}{env_value}' + if prepend else f'{env_value}{os.pathsep}{value}') + else: + env_value = value + os.environ[var] = env_value + + +def designer(): + init_virtual_env() + + # https://www.python.org/dev/peps/pep-0384/#linkage : + # "On Unix systems, the ABI is typically provided by the python executable + # itself", that is, libshiboken does not link against any Python library + # and expects to get these symbols from a python executable. Since no + # python executable is involved when loading this plugin, pre-load python.so + # This should also help to work around a numpy issue, see + # https://stackoverflow.com/questions/49784583/numpy-import-fails-on-multiarray-extension-library-when-called-from-embedded-pyt + major_version = sys.version_info[0] + minor_version = sys.version_info[1] + os.environ['PY_MAJOR_VERSION'] = str(major_version) + os.environ['PY_MINOR_VERSION'] = str(minor_version) + if sys.platform == 'linux': + # Determine library name (examples/utils/pyside_config.py) + version = f'{major_version}.{minor_version}' + library_name = f'libpython{version}{sys.abiflags}.so' + if is_pyenv_python(): + library_name = str(Path(sysconfig.get_config_var('LIBDIR')) / library_name) + os.environ['LD_PRELOAD'] = library_name + elif sys.platform == 'darwin': + library_name = sysconfig.get_config_var("LDLIBRARY") + framework_prefix = sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX") + lib_path = None + if framework_prefix: + lib_path = os.fspath(Path(framework_prefix) / library_name) + elif is_pyenv_python(): + lib_path = str(Path(sysconfig.get_config_var('LIBDIR')) / library_name) + else: + # ideally this should never be reached because the system Python and Python installed + # from python.org are all framework builds + print("Unable to find Python library directory. Use a framework build of Python.", + file=sys.stderr) + sys.exit(0) + os.environ['DYLD_INSERT_LIBRARIES'] = lib_path + elif sys.platform == 'win32': + # Find Python DLLs from the base installation + if is_virtual_env(): + _extend_path_var("PATH", os.fspath(Path(sys._base_executable).parent), True) + + qt_tool_wrapper(ui_tool_binary("designer"), sys.argv[1:]) + + +def linguist(): + qt_tool_wrapper(ui_tool_binary("linguist"), sys.argv[1:]) + + +def genpyi(): + pyside_dir = Path(__file__).resolve().parents[1] + support = pyside_dir / "support" + cmd = support / "generate_pyi.py" + command = [sys.executable, os.fspath(cmd)] + sys.argv[1:] + sys.exit(subprocess.call(command)) + + +def metaobjectdump(): + pyside_script_wrapper("metaobjectdump.py") + + +def project(): + pyside_script_wrapper("project.py") + + +def qml(): + pyside_script_wrapper("qml.py") + + +def qtpy2cpp(): + pyside_script_wrapper("qtpy2cpp.py") + + +def deploy(): + pyside_script_wrapper("deploy.py") + + +def android_deploy(): + if not sys.platform == "linux": + print("pyside6-android-deploy only works from a Linux host") + else: + android_requirements_file = Path(__file__).parent / "requirements-android.txt" + with open(android_requirements_file, 'r', encoding='UTF-8') as file: + while line := file.readline(): + dependent_package = line.rstrip() + if not bool(importlib.util.find_spec(dependent_package)): + command = [sys.executable, "-m", "pip", "install", dependent_package] + subprocess.run(command) + pyside_script_wrapper("android_deploy.py") + + +def qsb(): + qt_tool_wrapper("qsb", sys.argv[1:]) + + +def balsam(): + qt_tool_wrapper("balsam", sys.argv[1:]) + + +def balsamui(): + qt_tool_wrapper("balsamui", sys.argv[1:]) + + +if __name__ == "__main__": + main() diff --git a/sources/pyside-tools/qml.py b/sources/pyside-tools/qml.py new file mode 100644 index 000000000..5d029f93d --- /dev/null +++ b/sources/pyside-tools/qml.py @@ -0,0 +1,246 @@ +# Copyright (C) 2018 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +"""pyside6-qml tool implementation. This tool mimics the capabilities of qml runtime utility +for python and enables quick protyping with python modules""" + +import argparse +import importlib.util +import logging +import sys +import os +from pathlib import Path +from pprint import pprint +from typing import List, Set + +from PySide6.QtCore import QCoreApplication, Qt, QLibraryInfo, QUrl, SignalInstance +from PySide6.QtGui import QGuiApplication, QSurfaceFormat +from PySide6.QtQml import QQmlApplicationEngine, QQmlComponent +from PySide6.QtQuick import QQuickView, QQuickItem +from PySide6.QtWidgets import QApplication + + +def import_qml_modules(qml_parent_path: Path, module_paths: List[Path] = []): + ''' + Import all the python modules in the qml_parent_path. This way all the classes + containing the @QmlElement/@QmlNamedElement are also imported + + Parameters: + qml_parent_path (Path): Parent directory of the qml file + module_paths (int): user give import paths obtained through cli + ''' + + search_dir_paths = [] + search_file_paths = [] + + if not module_paths: + search_dir_paths.append(qml_parent_path) + else: + for module_path in module_paths: + if module_path.is_dir(): + search_dir_paths.append(module_path) + elif module_path.exists() and module_path.suffix == ".py": + search_file_paths.append(module_path) + + def import_module(import_module_paths: Set[Path]): + """Import the modules in 'import_module_paths'""" + for module_path in import_module_paths: + module_name = module_path.name[:-3] + _spec = importlib.util.spec_from_file_location(f"{module_name}", module_path) + _module = importlib.util.module_from_spec(_spec) + _spec.loader.exec_module(module=_module) + + modules_to_import = set() + for search_path in search_dir_paths: + possible_modules = list(search_path.glob("**/*.py")) + for possible_module in possible_modules: + if possible_module.is_file() and possible_module.name != "__init__.py": + module_parent = str(possible_module.parent) + if module_parent not in sys.path: + sys.path.append(module_parent) + modules_to_import.add(possible_module) + + for search_path in search_file_paths: + sys.path.append(str(search_path.parent)) + modules_to_import.add(search_path) + + import_module(import_module_paths=modules_to_import) + + +def print_configurations(): + return "Built-in configurations \n\t default \n\t resizeToItem" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="This tools mimics the capabilities of qml runtime utility by directly" + " invoking QQmlEngine/QQuickView. It enables quick prototyping with qml files.", + formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument( + "file", + type=lambda p: Path(p).absolute(), + help="Path to qml file to display", + ) + parser.add_argument( + "--module-paths", "-I", + type=lambda p: Path(p).absolute(), + nargs="+", + help="Specify space separated folder/file paths where the Qml classes are defined. By" + " default,the parent directory of the qml_path is searched recursively for all .py" + " files and they are imported. Otherwise only the paths give in module paths are" + " searched", + ) + parser.add_argument( + "--list-conf", + action="version", + help="List the built-in configurations.", + version=print_configurations() + ) + parser.add_argument( + "--apptype", "-a", + choices=["core", "gui", "widget"], + default="gui", + help="Select which application class to use. Default is gui", + ) + parser.add_argument( + "--config", "-c", + choices=["default", "resizeToItem"], + default="default", + help="Select the built-in configurations.", + ) + parser.add_argument( + "--rhi", "-r", + choices=["vulkan", "metal", "d3dll", "gl"], + help="Set the backend for the Qt graphics abstraction (RHI).", + ) + parser.add_argument( + "--core-profile", + action="store_true", + help="Force use of OpenGL Core Profile.", + ) + parser.add_argument( + '-v', '--verbose', + help="Print information about what qml is doing, like specific file URLs being loaded.", + action="store_const", dest="loglevel", const=logging.INFO, + ) + + gl_group = parser.add_mutually_exclusive_group(required=False) + gl_group.add_argument( + "--gles", + action="store_true", + help="Force use of GLES (AA_UseOpenGLES)", + ) + gl_group.add_argument( + "--desktop", + action="store_true", + help="Force use of desktop OpenGL (AA_UseDesktopOpenGL)", + ) + gl_group.add_argument( + "--software", + action="store_true", + help="Force use of software rendering(AA_UseSoftwareOpenGL)", + ) + gl_group.add_argument( + "--disable-context-sharing", + action="store_true", + help=" Disable the use of a shared GL context for QtQuick Windows", + ) + + args = parser.parse_args() + apptype = args.apptype + + qquick_present = False + + with open(args.file) as myfile: + if 'import QtQuick' in myfile.read(): + qquick_present = True + + # no import QtQuick => QQCoreApplication + if not qquick_present: + apptype = "core" + + import_qml_modules(args.file.parent, args.module_paths) + + logging.basicConfig(level=args.loglevel) + logging.info(f"qml: {QLibraryInfo.build()}") + logging.info(f"qml: Using built-in configuration: {args.config}") + + if args.rhi: + os.environ['QSG_RHI_BACKEND'] = args.rhi + + logging.info(f"qml: loading {args.file}") + qml_file = QUrl.fromLocalFile(str(args.file)) + + if apptype == "gui": + if args.gles: + logging.info("qml: Using attribute AA_UseOpenGLES") + QCoreApplication.setAttribute(Qt.AA_UseOpenGLES) + elif args.desktop: + logging.info("qml: Using attribute AA_UseDesktopOpenGL") + QCoreApplication.setAttribute(Qt.AA_UseDesktopOpenGL) + elif args.software: + logging.info("qml: Using attribute AA_UseSoftwareOpenGL") + QCoreApplication.setAttribute(Qt.AA_UseSoftwareOpenGL) + + # context-sharing is enabled by default + if not args.disable_context_sharing: + logging.info("qml: Using attribute AA_ShareOpenGLContexts") + QCoreApplication.setAttribute(Qt.AA_ShareOpenGLContexts) + + if apptype == "core": + logging.info("qml: Core application") + app = QCoreApplication(sys.argv) + elif apptype == "widgets": + logging.info("qml: Widget application") + app = QApplication(sys.argv) + else: + logging.info("qml: Gui application") + app = QGuiApplication(sys.argv) + + engine = QQmlApplicationEngine() + + # set OpenGLContextProfile + if apptype == "gui" and args.core_profile: + logging.info("qml: Set profile for QSurfaceFormat as CoreProfile") + surfaceFormat = QSurfaceFormat() + surfaceFormat.setStencilBufferSize(8) + surfaceFormat.setDepthBufferSize(24) + surfaceFormat.setVersion(4, 1) + surfaceFormat.setProfile(QSurfaceFormat.CoreProfile) + QSurfaceFormat.setDefaultFormat(surfaceFormat) + + # in the case of QCoreApplication we print the attributes of the object created via + # QQmlComponent and exit + if apptype == "core": + component = QQmlComponent(engine, qml_file) + obj = component.create() + filtered_attributes = {k: v for k, v in vars(obj).items() if type(v) is not SignalInstance} + logging.info("qml: component object attributes are") + pprint(filtered_attributes) + del engine + sys.exit(0) + + engine.load(qml_file) + rootObjects = engine.rootObjects() + if not rootObjects: + sys.exit(-1) + + qquick_view = False + if isinstance(rootObjects[0], QQuickItem) and qquick_present: + logging.info("qml: loading with QQuickView") + viewer = QQuickView() + viewer.setSource(qml_file) + if args.config != "resizeToItem": + viewer.setResizeMode(QQuickView.SizeRootObjectToView) + else: + viewer.setResizeMode(QQuickView.SizeViewToRootObject) + viewer.show() + qquick_view = True + + if not qquick_view: + logging.info("qml: loading with QQmlApplicationEngine") + if args.config == "resizeToItem": + logging.info("qml: Not a QQuickview item. resizeToItem is done by default") + + sys.exit(app.exec()) diff --git a/sources/pyside-tools/qtpy2cpp.py b/sources/pyside-tools/qtpy2cpp.py new file mode 100644 index 000000000..857b12b67 --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp.py @@ -0,0 +1,62 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import logging +import os +import sys +from argparse import ArgumentParser, RawTextHelpFormatter +from pathlib import Path + +from qtpy2cpp_lib.visitor import ConvertVisitor + +DESCRIPTION = "Tool to convert Python to C++" + + +def create_arg_parser(desc): + parser = ArgumentParser(description=desc, + formatter_class=RawTextHelpFormatter) + parser.add_argument("--debug", "-d", action="store_true", + help="Debug") + parser.add_argument("--stdout", "-s", action="store_true", + help="Write to stdout") + parser.add_argument("--force", "-f", action="store_true", + help="Force overwrite of existing files") + parser.add_argument("files", type=str, nargs="+", help="Python source file(s)") + return parser + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(__name__) + arg_parser = create_arg_parser(DESCRIPTION) + args = arg_parser.parse_args() + ConvertVisitor.debug = args.debug + + for input_file_str in args.files: + input_file = Path(input_file_str) + if not input_file.is_file(): + logger.error(f"{input_file_str} does not exist or is not a file.") + sys.exit(-1) + file_root, ext = os.path.splitext(input_file) + if input_file.suffix != ".py": + logger.error(f"{input_file_str} does not appear to be a Python file.") + sys.exit(-1) + + ast_tree = ConvertVisitor.create_ast(input_file_str) + if args.stdout: + sys.stdout.write(f"// Converted from {input_file.name}\n") + ConvertVisitor(input_file, sys.stdout).visit(ast_tree) + else: + target_file = input_file.parent / (input_file.stem + ".cpp") + if target_file.exists(): + if not target_file.is_file(): + logger.error(f"{target_file} exists and is not a file.") + sys.exit(-1) + if not args.force: + logger.error(f"{target_file} exists. Use -f to overwrite.") + sys.exit(-1) + + with target_file.open("w") as file: + file.write(f"// Converted from {input_file.name}\n") + ConvertVisitor(input_file, file).visit(ast_tree) + logger.info(f"Wrote {target_file}.") diff --git a/sources/pyside-tools/qtpy2cpp.pyproject b/sources/pyside-tools/qtpy2cpp.pyproject new file mode 100644 index 000000000..a059aebca --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp.pyproject @@ -0,0 +1,7 @@ +{ + "files": ["qtpy2cpp.py", + "qtpy2cpp_lib/formatter.py", "qtpy2cpp_lib/visitor.py", "qtpy2cpp_lib/nodedump.py", + "qtpy2cpp_lib/astdump.py", "qtpy2cpp_lib/tokenizer.py", "qtpy2cpp_lib/qt.py", + "qtpy2cpp_lib/tests/test_qtpy2cpp.py", + "qtpy2cpp_lib/tests/baseline/basic_test.py", "qtpy2cpp_lib/tests/baseline/uic.py"] +} diff --git a/sources/pyside-tools/qtpy2cpp_lib/astdump.py b/sources/pyside-tools/qtpy2cpp_lib/astdump.py new file mode 100644 index 000000000..d92fb7589 --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp_lib/astdump.py @@ -0,0 +1,111 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +"""Tool to dump a Python AST""" + + +import ast +import tokenize +from argparse import ArgumentParser, RawTextHelpFormatter +from enum import Enum + +from nodedump import debug_format_node + +DESCRIPTION = "Tool to dump a Python AST" + + +_source_lines = [] +_opt_verbose = False + + +def first_non_space(s): + for i, c in enumerate(s): + if c != ' ': + return i + return 0 + + +class NodeType(Enum): + IGNORE = 1 + PRINT_ONE_LINE = 2 # Print as a one liner, do not visit children + PRINT = 3 # Print with opening closing tag, visit children + PRINT_WITH_SOURCE = 4 # Like PRINT, but print source line above + + +def get_node_type(node): + if isinstance(node, (ast.Load, ast.Store, ast.Delete)): + return NodeType.IGNORE + if isinstance(node, (ast.Add, ast.alias, ast.arg, ast.Eq, ast.Gt, ast.Lt, + ast.Mult, ast.Name, ast.NotEq, ast.NameConstant, ast.Not, + ast.Num, ast.Str)): + return NodeType.PRINT_ONE_LINE + if not hasattr(node, 'lineno'): + return NodeType.PRINT + if isinstance(node, (ast.Attribute)): + return NodeType.PRINT_ONE_LINE if isinstance(node.value, ast.Name) else NodeType.PRINT + return NodeType.PRINT_WITH_SOURCE + + +class DumpVisitor(ast.NodeVisitor): + def __init__(self): + ast.NodeVisitor.__init__(self) + self._indent = 0 + self._printed_source_lines = {-1} + + def generic_visit(self, node): + node_type = get_node_type(node) + if _opt_verbose and node_type in (NodeType.IGNORE, NodeType.PRINT_ONE_LINE): + node_type = NodeType.PRINT + if node_type == NodeType.IGNORE: + return + self._indent = self._indent + 1 + indent = ' ' * self._indent + + if node_type == NodeType.PRINT_WITH_SOURCE: + line_number = node.lineno - 1 + if line_number not in self._printed_source_lines: + self._printed_source_lines.add(line_number) + line = _source_lines[line_number] + non_space = first_non_space(line) + print('{:04d} {}{}'.format(line_number, '_' * non_space, + line[non_space:])) + + if node_type == NodeType.PRINT_ONE_LINE: + print(indent, debug_format_node(node)) + else: + print(indent, '>', debug_format_node(node)) + ast.NodeVisitor.generic_visit(self, node) + print(indent, '<', type(node).__name__) + + self._indent = self._indent - 1 + + +def parse_ast(filename): + node = None + with tokenize.open(filename) as f: + global _source_lines + source = f.read() + _source_lines = source.split('\n') + node = ast.parse(source, mode="exec") + return node + + +def create_arg_parser(desc): + parser = ArgumentParser(description=desc, + formatter_class=RawTextHelpFormatter) + parser.add_argument('--verbose', '-v', action='store_true', + help='Verbose') + parser.add_argument('source', type=str, help='Python source') + return parser + + +if __name__ == '__main__': + arg_parser = create_arg_parser(DESCRIPTION) + options = arg_parser.parse_args() + _opt_verbose = options.verbose + title = f'AST tree for {options.source}' + print('=' * len(title)) + print(title) + print('=' * len(title)) + tree = parse_ast(options.source) + DumpVisitor().visit(tree) diff --git a/sources/pyside-tools/qtpy2cpp_lib/formatter.py b/sources/pyside-tools/qtpy2cpp_lib/formatter.py new file mode 100644 index 000000000..9a38e803d --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp_lib/formatter.py @@ -0,0 +1,265 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +"""C++ formatting helper functions and formatter class""" + + +import ast + +from .qt import ClassFlag, qt_class_flags + +CLOSING = {"{": "}", "(": ")", "[": "]"} # Closing parenthesis for C++ + + +def _fix_function_argument_type(type, for_return): + """Fix function argument/return qualifiers using some heuristics for Qt.""" + if type == "float": + return "double" + if type == "str": + type = "QString" + if not type.startswith("Q"): + return type + flags = qt_class_flags(type) + if flags & ClassFlag.PASS_BY_VALUE: + return type + if flags & ClassFlag.PASS_BY_CONSTREF: + return type if for_return else f"const {type} &" + if flags & ClassFlag.PASS_BY_REF: + return type if for_return else f"{type} &" + return type + " *" # Assume pointer by default + + +def to_string(node): + """Helper to retrieve a string from the (Lists of)Name/Attribute + aggregated into some nodes""" + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return '' + + +def format_inheritance(class_def_node): + """Returns inheritance specification of a class""" + result = '' + for base in class_def_node.bases: + name = to_string(base) + if name != 'object': + result += ', public ' if result else ' : public ' + result += name + return result + + +def format_for_target(target_node): + if isinstance(target_node, ast.Tuple): # for i,e in enumerate() + result = '' + for i, el in enumerate(target_node.elts): + if i > 0: + result += ', ' + result += format_reference(el) + return result + return format_reference(target_node) + + +def format_for_loop(f_node): + """Format a for loop + This applies some heuristics to detect: + 1) "for a in [1,2])" -> "for (f: {1, 2}) {" + 2) "for i in range(5)" -> "for (i = 0; i < 5; ++i) {" + 3) "for i in range(2,5)" -> "for (i = 2; i < 5; ++i) {" + + TODO: Detect other cases, maybe including enumerate(). + """ + loop_vars = format_for_target(f_node.target) + result = 'for (' + loop_vars + if isinstance(f_node.iter, ast.Call): + f = format_reference(f_node.iter.func) + if f == 'range': + start = 0 + end = -1 + if len(f_node.iter.args) == 2: + start = format_literal(f_node.iter.args[0]) + end = format_literal(f_node.iter.args[1]) + elif len(f_node.iter.args) == 1: + end = format_literal(f_node.iter.args[0]) + result += f' = {start}; {loop_vars} < {end}; ++{loop_vars}' + elif isinstance(f_node.iter, ast.List): + # Range based for over list + result += ': ' + format_literal_list(f_node.iter) + elif isinstance(f_node.iter, ast.Name): + # Range based for over variable + result += ': ' + f_node.iter.id + result += ') {' + return result + + +def format_name_constant(node): + """Format a ast.NameConstant.""" + if node.value is None: + return "nullptr" + return "true" if node.value else "false" + + +def format_literal(node): + """Returns the value of number/string literals""" + if isinstance(node, ast.NameConstant): + return format_name_constant(node) + if isinstance(node, ast.Num): + return str(node.n) + if isinstance(node, ast.Str): + # Fixme: escaping + return f'"{node.s}"' + return '' + + +def format_literal_list(l_node, enclosing='{'): + """Formats a list/tuple of number/string literals as C++ initializer list""" + result = enclosing + for i, el in enumerate(l_node.elts): + if i > 0: + result += ', ' + result += format_literal(el) + result += CLOSING[enclosing] + return result + + +def format_member(attrib_node, qualifier_in='auto'): + """Member access foo->member() is expressed as an attribute with + further nested Attributes/Names as value""" + n = attrib_node + result = '' + # Black magic: Guess '::' if name appears to be a class name + qualifier = qualifier_in + if qualifier_in == 'auto': + qualifier = '::' if n.attr[0:1].isupper() else '->' + while isinstance(n, ast.Attribute): + result = n.attr if not result else n.attr + qualifier + result + n = n.value + if isinstance(n, ast.Name) and n.id != 'self': + if qualifier_in == 'auto' and n.id == "Qt": # Qt namespace + qualifier = "::" + result = n.id + qualifier + result + return result + + +def format_reference(node, qualifier='auto'): + """Format member reference or free item""" + return node.id if isinstance(node, ast.Name) else format_member(node, qualifier) + + +def format_function_def_arguments(function_def_node): + """Formats arguments of a function definition""" + # Default values is a list of the last default values, expand + # so that indexes match + argument_count = len(function_def_node.args.args) + default_values = function_def_node.args.defaults + while len(default_values) < argument_count: + default_values.insert(0, None) + result = '' + for i, a in enumerate(function_def_node.args.args): + if result: + result += ', ' + if a.arg != 'self': + if a.annotation and isinstance(a.annotation, ast.Name): + result += _fix_function_argument_type(a.annotation.id, False) + ' ' + result += a.arg + if default_values[i]: + result += ' = ' + default_value = default_values[i] + if isinstance(default_value, ast.Attribute): + result += format_reference(default_value) + else: + result += format_literal(default_value) + return result + + +def format_start_function_call(call_node): + """Format a call of a free or member function""" + return format_reference(call_node.func) + '(' + + +def write_import(file, i_node): + """Print an import of a Qt class as #include""" + for alias in i_node.names: + if alias.name.startswith('Q'): + file.write(f'#include <{alias.name}>\n') + + +def write_import_from(file, i_node): + """Print an import from Qt classes as #include sequence""" + # "from PySide6.QtGui import QGuiApplication" or + # "from PySide6 import QtGui" + mod = i_node.module + if not mod.startswith('PySide') and not mod.startswith('PyQt'): + return + dot = mod.find('.') + qt_module = mod[dot + 1:] + '/' if dot >= 0 else '' + for i in i_node.names: + if i.name.startswith('Q'): + file.write(f'#include <{qt_module}{i.name}>\n') + + +class Indenter: + """Helper for Indentation""" + + def __init__(self, output_file): + self._indent_level = 0 + self._indentation = '' + self._output_file = output_file + + def indent_string(self, string): + """Start a new line by a string""" + self._output_file.write(self._indentation) + self._output_file.write(string) + + def indent_line(self, line): + """Write an indented line""" + self._output_file.write(self._indentation) + self._output_file.write(line) + self._output_file.write('\n') + + def INDENT(self): + """Write indentation""" + self._output_file.write(self._indentation) + + def indent(self): + """Increase indentation level""" + self._indent_level = self._indent_level + 1 + self._indentation = ' ' * self._indent_level + + def dedent(self): + """Decrease indentation level""" + self._indent_level = self._indent_level - 1 + self._indentation = ' ' * self._indent_level + + +class CppFormatter(Indenter): + """Provides helpers for formatting multi-line C++ constructs""" + + def __init__(self, output_file): + Indenter.__init__(self, output_file) + + def write_class_def(self, class_node): + """Print a class definition with inheritance""" + self._output_file.write('\n') + inherits = format_inheritance(class_node) + self.indent_line(f'class {class_node.name}{inherits}') + self.indent_line('{') + self.indent_line('public:') + + def write_function_def(self, f_node, class_context): + """Print a function definition with arguments""" + self._output_file.write('\n') + arguments = format_function_def_arguments(f_node) + if f_node.name == '__init__' and class_context: # Constructor + name = class_context + elif f_node.name == '__del__' and class_context: # Destructor + name = '~' + class_context + else: + return_type = "void" + if f_node.returns and isinstance(f_node.returns, ast.Name): + return_type = _fix_function_argument_type(f_node.returns.id, True) + name = return_type + " " + f_node.name + self.indent_string(f'{name}({arguments})') + self._output_file.write('\n') + self.indent_line('{') diff --git a/sources/pyside-tools/qtpy2cpp_lib/nodedump.py b/sources/pyside-tools/qtpy2cpp_lib/nodedump.py new file mode 100644 index 000000000..de62e9700 --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp_lib/nodedump.py @@ -0,0 +1,50 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +"""Helper to dump AST nodes for debugging""" + + +import ast + + +def to_string(node): + """Helper to retrieve a string from the (Lists of )Name/Attribute + aggregated into some nodes""" + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return node.attr + return '' + + +def debug_format_node(node): + """Format AST node for debugging""" + if isinstance(node, ast.alias): + return f'alias("{node.name}")' + if isinstance(node, ast.arg): + return f'arg({node.arg})' + if isinstance(node, ast.Attribute): + if isinstance(node.value, ast.Name): + nested_name = debug_format_node(node.value) + return f'Attribute("{node.attr}", {nested_name})' + return f'Attribute("{node.attr}")' + if isinstance(node, ast.Call): + return 'Call({}({}))'.format(to_string(node.func), len(node.args)) + if isinstance(node, ast.ClassDef): + base_names = [to_string(base) for base in node.bases] + bases = ': ' + ','.join(base_names) if base_names else '' + return f'ClassDef({node.name}{bases})' + if isinstance(node, ast.ImportFrom): + return f'ImportFrom("{node.module}")' + if isinstance(node, ast.FunctionDef): + arg_names = [a.arg for a in node.args.args] + return 'FunctionDef({}({}))'.format(node.name, ', '.join(arg_names)) + if isinstance(node, ast.Name): + return 'Name("{}", Ctx={})'.format(node.id, type(node.ctx).__name__) + if isinstance(node, ast.NameConstant): + return f'NameConstant({node.value})' + if isinstance(node, ast.Num): + return f'Num({node.n})' + if isinstance(node, ast.Str): + return f'Str("{node.s}")' + return type(node).__name__ diff --git a/sources/pyside-tools/qtpy2cpp_lib/qt.py b/sources/pyside-tools/qtpy2cpp_lib/qt.py new file mode 100644 index 000000000..69bd54aeb --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp_lib/qt.py @@ -0,0 +1,56 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +"""Provides some type information on Qt classes""" + + +from enum import Flag + + +class ClassFlag(Flag): + PASS_BY_CONSTREF = 1 + PASS_BY_REF = 2 + PASS_BY_VALUE = 4 + PASS_ON_STACK_MASK = PASS_BY_CONSTREF | PASS_BY_REF | PASS_BY_VALUE + INSTANTIATE_ON_STACK = 8 + + +_QT_CLASS_FLAGS = { + "QBrush": ClassFlag.PASS_BY_CONSTREF | ClassFlag.INSTANTIATE_ON_STACK, + "QGradient": ClassFlag.PASS_BY_CONSTREF | ClassFlag.INSTANTIATE_ON_STACK, + "QIcon": ClassFlag.PASS_BY_CONSTREF | ClassFlag.INSTANTIATE_ON_STACK, + "QLine": ClassFlag.PASS_BY_CONSTREF | ClassFlag.INSTANTIATE_ON_STACK, + "QLineF": ClassFlag.PASS_BY_CONSTREF | ClassFlag.INSTANTIATE_ON_STACK, + "QPixmap": ClassFlag.PASS_BY_CONSTREF | ClassFlag.INSTANTIATE_ON_STACK, + "QPointF": ClassFlag.PASS_BY_CONSTREF | ClassFlag.INSTANTIATE_ON_STACK, + "QRect": ClassFlag.PASS_BY_CONSTREF | ClassFlag.INSTANTIATE_ON_STACK, + "QRectF": ClassFlag.PASS_BY_CONSTREF | ClassFlag.INSTANTIATE_ON_STACK, + "QSizeF": ClassFlag.PASS_BY_CONSTREF | ClassFlag.INSTANTIATE_ON_STACK, + "QString": ClassFlag.PASS_BY_CONSTREF | ClassFlag.INSTANTIATE_ON_STACK, + "QFile": ClassFlag.PASS_BY_REF | ClassFlag.INSTANTIATE_ON_STACK, + "QSettings": ClassFlag.PASS_BY_REF | ClassFlag.INSTANTIATE_ON_STACK, + "QTextStream": ClassFlag.PASS_BY_REF | ClassFlag.INSTANTIATE_ON_STACK, + "QColor": ClassFlag.PASS_BY_VALUE | ClassFlag.INSTANTIATE_ON_STACK, + "QPoint": ClassFlag.PASS_BY_VALUE | ClassFlag.INSTANTIATE_ON_STACK, + "QSize": ClassFlag.PASS_BY_VALUE | ClassFlag.INSTANTIATE_ON_STACK, + "QApplication": ClassFlag.INSTANTIATE_ON_STACK, + "QColorDialog": ClassFlag.INSTANTIATE_ON_STACK, + "QCoreApplication": ClassFlag.INSTANTIATE_ON_STACK, + "QFileDialog": ClassFlag.INSTANTIATE_ON_STACK, + "QFileInfo": ClassFlag.INSTANTIATE_ON_STACK, + "QFontDialog": ClassFlag.INSTANTIATE_ON_STACK, + "QGuiApplication": ClassFlag.INSTANTIATE_ON_STACK, + "QMessageBox": ClassFlag.INSTANTIATE_ON_STACK, + "QPainter": ClassFlag.INSTANTIATE_ON_STACK, + "QPen": ClassFlag.INSTANTIATE_ON_STACK, + "QQmlApplicationEngine": ClassFlag.INSTANTIATE_ON_STACK, + "QQmlComponent": ClassFlag.INSTANTIATE_ON_STACK, + "QQmlEngine": ClassFlag.INSTANTIATE_ON_STACK, + "QQuickView": ClassFlag.INSTANTIATE_ON_STACK, + "QSaveFile": ClassFlag.INSTANTIATE_ON_STACK +} + + +def qt_class_flags(type): + f = _QT_CLASS_FLAGS.get(type) + return f if f else ClassFlag(0) diff --git a/sources/pyside-tools/qtpy2cpp_lib/tests/baseline/basic_test.cpp b/sources/pyside-tools/qtpy2cpp_lib/tests/baseline/basic_test.cpp new file mode 100644 index 000000000..8ee7be31e --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp_lib/tests/baseline/basic_test.cpp @@ -0,0 +1,62 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +// Converted from basic_test.py +#include <QtCore/Qt> +#include <QtGui/QColor> +#include <QtGui/QPainter> +#include <QtGui/QPaintEvent> +#include <QtGui/QShortcut> +#include <QtWidgets/QApplication> +#include <QtWidgets/QWidget> + +class Window : public QWidget +{ +public: + + Window(QWidget * parent = nullptr) + { + super()->__init__(parent); + } + + void paintEvent(QPaintEvent * e) + { + paint("bla"); + } + + void paint(const QString & what, color = Qt::blue) + { + { // Converted from context manager + p = QPainter(); + p->setPen(QColor(color)); + rect = rect(); + w = rect->width(); + h = rect->height(); + p->drawLine(0, 0, w, h); + p->drawLine(0, h, w, 0); + p->drawText(rect->center(), what); + } + } + + void sum() + { + values = {1, 2, 3}; + result = 0; + for (v: values) { + result += v + } + return result; + } +}; + +int main(int argc, char *argv[]) +{ + QApplication app(sys->argv); + window = Window(); + auto *sc = new QShortcut((Qt::CTRL | Qt::Key_Q), window); + sc->activated->connect(window->close); + window->setWindowTitle("Test"); + window->show(); + sys->exit(app.exec()); + return 0; +} diff --git a/sources/pyside-tools/qtpy2cpp_lib/tests/baseline/basic_test.py b/sources/pyside-tools/qtpy2cpp_lib/tests/baseline/basic_test.py new file mode 100644 index 000000000..1466ac6b1 --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp_lib/tests/baseline/basic_test.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +import sys + +from PySide6.QtCore import Qt +from PySide6.QtGui import QColor, QPainter, QPaintEvent, QShortcut +from PySide6.QtWidgets import QApplication, QWidget + + +class Window(QWidget): + def __init__(self, parent: QWidget = None): + super().__init__(parent) + + def paintEvent(self, e: QPaintEvent): + self.paint("bla") + + def paint(self, what: str, color: Qt.GlobalColor = Qt.blue): + with QPainter(self) as p: + p.setPen(QColor(color)) + rect = self.rect() + w = rect.width() + h = rect.height() + p.drawLine(0, 0, w, h) + p.drawLine(0, h, w, 0) + p.drawText(rect.center(), what) + + def sum(self): + values = [1, 2, 3] + result = 0 + for v in values: + result += v + return result + + +if __name__ == '__main__': + app = QApplication(sys.argv) + window = Window() + sc = QShortcut(Qt.CTRL | Qt.Key_Q, window) + sc.activated.connect(window.close) + window.setWindowTitle("Test") + window.show() + sys.exit(app.exec()) diff --git a/sources/pyside-tools/qtpy2cpp_lib/tests/test_qtpy2cpp.py b/sources/pyside-tools/qtpy2cpp_lib/tests/test_qtpy2cpp.py new file mode 100644 index 000000000..894b2a958 --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp_lib/tests/test_qtpy2cpp.py @@ -0,0 +1,54 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import subprocess +import tempfile +import sys +from pathlib import Path + +# run pytest-3 + + +def diff_code(actual_code, expected_file): + """Helper to run diff if something fails (Linux only).""" + with tempfile.NamedTemporaryFile(suffix=".cpp") as tf: + tf.write(actual_code.encode('utf-8')) + tf.flush() + diff_cmd = ["diff", "-u", expected_file, tf.name] + subprocess.run(diff_cmd) + + +def run_converter(tool, file): + """Run the converter and return C++ code generated from file.""" + cmd = [sys.executable, tool, "--stdout", file] + output = "" + with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: + output_b, errors_b = proc.communicate() + output = output_b.decode('utf-8') + if errors_b: + print(errors_b.decode('utf-8'), file=sys.stderr) + return output + + +def test_examples(): + dir = Path(__file__).resolve().parent + tool = dir.parents[1] / "qtpy2cpp.py" + assert tool.is_file + for test_file in (dir / "baseline").glob("*.py"): + assert test_file.is_file + expected_file = test_file.parent / (test_file.stem + ".cpp") + if expected_file.is_file(): + actual_code = run_converter(tool, test_file) + assert actual_code + expected_code = expected_file.read_text() + # Strip the license + code_start = expected_code.find("// Converted from") + assert code_start != -1 + expected_code = expected_code[code_start:] + + if actual_code != expected_code: + diff_code(actual_code, expected_file) + assert actual_code == expected_code + else: + print(f"Warning, {test_file} is missing a .cpp file.", + file=sys.stderr) diff --git a/sources/pyside-tools/qtpy2cpp_lib/tokenizer.py b/sources/pyside-tools/qtpy2cpp_lib/tokenizer.py new file mode 100644 index 000000000..d5e26c2a8 --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp_lib/tokenizer.py @@ -0,0 +1,55 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +"""Tool to dump Python Tokens""" + + +import sys +import tokenize + + +def format_token(t): + r = repr(t) + if r.startswith('TokenInfo('): + r = r[10:] + pos = r.find("), line='") + if pos < 0: + pos = r.find('), line="') + if pos > 0: + r = r[:pos + 1] + return r + + +def first_non_space(s): + for i, c in enumerate(s): + if c != ' ': + return i + return 0 + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Specify file Name") + sys.exit(1) + filename = sys.argv[1] + indent_level = 0 + indent = '' + last_line_number = -1 + with tokenize.open(filename) as f: + generator = tokenize.generate_tokens(f.readline) + for t in generator: + line_number = t.start[0] + if line_number != last_line_number: + code_line = t.line.rstrip() + non_space = first_non_space(code_line) + print('{:04d} {}{}'.format(line_number, '_' * non_space, + code_line[non_space:])) + last_line_number = line_number + if t.type == tokenize.INDENT: + indent_level = indent_level + 1 + indent = ' ' * indent_level + elif t.type == tokenize.DEDENT: + indent_level = indent_level - 1 + indent = ' ' * indent_level + else: + print(' ', indent, format_token(t)) diff --git a/sources/pyside-tools/qtpy2cpp_lib/visitor.py b/sources/pyside-tools/qtpy2cpp_lib/visitor.py new file mode 100644 index 000000000..2056951ae --- /dev/null +++ b/sources/pyside-tools/qtpy2cpp_lib/visitor.py @@ -0,0 +1,442 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +"""AST visitor printing out C++""" + +import ast +import sys +import tokenize +import warnings + +from .formatter import (CppFormatter, format_for_loop, format_literal, + format_name_constant, + format_reference, write_import, write_import_from) +from .nodedump import debug_format_node +from .qt import ClassFlag, qt_class_flags + + +def _is_qt_constructor(assign_node): + """Is this assignment node a plain construction of a Qt class? + 'f = QFile(name)'. Returns the class_name.""" + call = assign_node.value + if (isinstance(call, ast.Call) and isinstance(call.func, ast.Name)): + func = call.func.id + if func.startswith("Q"): + return func + return None + + +def _is_if_main(if_node): + """Return whether an if statement is: if __name__ == '__main__' """ + test = if_node.test + return (isinstance(test, ast.Compare) + and len(test.ops) == 1 + and isinstance(test.ops[0], ast.Eq) + and isinstance(test.left, ast.Name) + and test.left.id == "__name__" + and len(test.comparators) == 1 + and isinstance(test.comparators[0], ast.Constant) + and test.comparators[0].value == "__main__") + + +class ConvertVisitor(ast.NodeVisitor, CppFormatter): + """AST visitor printing out C++ + Note on implementation: + - Any visit_XXX() overridden function should call self.generic_visit(node) + to continue visiting + - When controlling the visiting manually (cf visit_Call()), + self.visit(child) needs to be called since that dispatches to + visit_XXX(). This is usually done to prevent undesired output + for example from references of calls, etc. + """ + + debug = False + + def __init__(self, file_name, output_file): + ast.NodeVisitor.__init__(self) + CppFormatter.__init__(self, output_file) + self._file_name = file_name + self._class_scope = [] # List of class names + self._stack = [] # nodes + self._stack_variables = [] # variables instantiated on stack + self._debug_indent = 0 + + @staticmethod + def create_ast(filename): + """Create an Abstract Syntax Tree on which a visitor can be run""" + node = None + with tokenize.open(filename) as file: + node = ast.parse(file.read(), mode="exec") + return node + + def generic_visit(self, node): + parent = self._stack[-1] if self._stack else None + if self.debug: + self._debug_enter(node, parent) + self._stack.append(node) + try: + super().generic_visit(node) + except Exception as e: + line_no = node.lineno if hasattr(node, 'lineno') else -1 + error_message = str(e) + message = f'{self._file_name}:{line_no}: Error "{error_message}"' + warnings.warn(message) + self._output_file.write(f'\n// {error_message}\n') + del self._stack[-1] + if self.debug: + self._debug_leave(node) + + def visit_Add(self, node): + self._handle_bin_op(node, "+") + + def _is_augmented_assign(self): + """Is it 'Augmented_assign' (operators +=/-=, etc)?""" + return self._stack and isinstance(self._stack[-1], ast.AugAssign) + + def visit_AugAssign(self, node): + """'Augmented_assign', Operators +=/-=, etc.""" + self.INDENT() + self.generic_visit(node) + self._output_file.write("\n") + + def visit_Assign(self, node): + self.INDENT() + + qt_class = _is_qt_constructor(node) + on_stack = qt_class and qt_class_flags(qt_class) & ClassFlag.INSTANTIATE_ON_STACK + + # Is this a free variable and not a member assignment? Instantiate + # on stack or give a type + if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + if qt_class: + if on_stack: + # "QFile f(args)" + var = node.targets[0].id + self._stack_variables.append(var) + self._output_file.write(f"{qt_class} {var}(") + self._write_function_args(node.value.args) + self._output_file.write(");\n") + return + self._output_file.write("auto *") + + line_no = node.lineno if hasattr(node, 'lineno') else -1 + for target in node.targets: + if isinstance(target, ast.Tuple): + w = f"{self._file_name}:{line_no}: List assignment not handled." + warnings.warn(w) + elif isinstance(target, ast.Subscript): + w = f"{self._file_name}:{line_no}: Subscript assignment not handled." + warnings.warn(w) + else: + self._output_file.write(format_reference(target)) + self._output_file.write(' = ') + if qt_class and not on_stack: + self._output_file.write("new ") + self.visit(node.value) + self._output_file.write(';\n') + + def visit_Attribute(self, node): + """Format a variable reference (cf visit_Name)""" + # Default parameter (like Qt::black)? + if self._ignore_function_def_node(node): + return + self._output_file.write(format_reference(node)) + + def visit_BinOp(self, node): + # Parentheses are not exposed, so, every binary operation needs to + # be enclosed by (). + self._output_file.write('(') + self.generic_visit(node) + self._output_file.write(')') + + def _handle_bin_op(self, node, op): + """Handle a binary operator which can appear as 'Augmented Assign'.""" + self.generic_visit(node) + full_op = f" {op}= " if self._is_augmented_assign() else f" {op} " + self._output_file.write(full_op) + + def visit_BitAnd(self, node): + self._handle_bin_op(node, "&") + + def visit_BitOr(self, node): + self._handle_bin_op(node, "|") + + def _format_call(self, node): + # Decorator list? + if self._ignore_function_def_node(node): + return + f = node.func + if isinstance(f, ast.Name): + self._output_file.write(f.id) + else: + # Attributes denoting chained calls "a->b()->c()". Walk along in + # reverse order, recursing for other calls. + names = [] + n = f + while isinstance(n, ast.Attribute): + names.insert(0, n.attr) + n = n.value + + if isinstance(n, ast.Name): # Member or variable reference + if n.id != "self": + sep = "->" + if n.id in self._stack_variables: + sep = "." + elif n.id[0:1].isupper(): # Heuristics for static + sep = "::" + self._output_file.write(n.id) + self._output_file.write(sep) + elif isinstance(n, ast.Call): # A preceding call + self._format_call(n) + self._output_file.write("->") + + self._output_file.write("->".join(names)) + + self._output_file.write('(') + self._write_function_args(node.args) + self._output_file.write(')') + + def visit_Call(self, node): + self._format_call(node) + # Context manager expression? + if self._within_context_manager(): + self._output_file.write(";\n") + + def _write_function_args(self, args_node): + # Manually do visit(), skip the children of func + for i, arg in enumerate(args_node): + if i > 0: + self._output_file.write(', ') + self.visit(arg) + + def visit_ClassDef(self, node): + # Manually do visit() to skip over base classes + # and annotations + self._class_scope.append(node.name) + self.write_class_def(node) + self.indent() + for b in node.body: + self.visit(b) + self.dedent() + self.indent_line('};') + del self._class_scope[-1] + + def visit_Div(self, node): + self._handle_bin_op(node, "/") + + def visit_Eq(self, node): + self.generic_visit(node) + self._output_file.write(" == ") + + def visit_Expr(self, node): + self.INDENT() + self.generic_visit(node) + self._output_file.write(';\n') + + def visit_Gt(self, node): + self.generic_visit(node) + self._output_file.write(" > ") + + def visit_GtE(self, node): + self.generic_visit(node) + self._output_file.write(" >= ") + + def visit_For(self, node): + # Manually do visit() to get the indentation right. + # TODO: what about orelse? + self.indent_line(format_for_loop(node)) + self.indent() + for b in node.body: + self.visit(b) + self.dedent() + self.indent_line('}') + + def visit_FunctionDef(self, node): + class_context = self._class_scope[-1] if self._class_scope else None + for decorator in node.decorator_list: + func = decorator.func # (Call) + if isinstance(func, ast.Name) and func.id == "Slot": + self._output_file.write("\npublic slots:") + self.write_function_def(node, class_context) + # Find stack variables + for arg in node.args.args: + if arg.annotation and isinstance(arg.annotation, ast.Name): + type_name = arg.annotation.id + flags = qt_class_flags(type_name) + if flags & ClassFlag.PASS_ON_STACK_MASK: + self._stack_variables.append(arg.arg) + self.indent() + self.generic_visit(node) + self.dedent() + self.indent_line('}') + self._stack_variables.clear() + + def visit_If(self, node): + # Manually do visit() to get the indentation right. Note: + # elsif() is modelled as nested if. + + # Check for the main function + if _is_if_main(node): + self._output_file.write("\nint main(int argc, char *argv[])\n{\n") + self.indent() + for b in node.body: + self.visit(b) + self.indent_string("return 0;\n") + self.dedent() + self._output_file.write("}\n") + return + + self.indent_string('if (') + self.visit(node.test) + self._output_file.write(') {\n') + self.indent() + for b in node.body: + self.visit(b) + self.dedent() + self.indent_string('}') + if node.orelse: + self._output_file.write(' else {\n') + self.indent() + for b in node.orelse: + self.visit(b) + self.dedent() + self.indent_string('}') + self._output_file.write('\n') + + def visit_Import(self, node): + write_import(self._output_file, node) + + def visit_ImportFrom(self, node): + write_import_from(self._output_file, node) + + def visit_List(self, node): + # Manually do visit() to get separators right + self._output_file.write('{') + for i, el in enumerate(node.elts): + if i > 0: + self._output_file.write(', ') + self.visit(el) + self._output_file.write('}') + + def visit_LShift(self, node): + self.generic_visit(node) + self._output_file.write(" << ") + + def visit_Lt(self, node): + self.generic_visit(node) + self._output_file.write(" < ") + + def visit_LtE(self, node): + self.generic_visit(node) + self._output_file.write(" <= ") + + def visit_Mult(self, node): + self._handle_bin_op(node, "*") + + def _within_context_manager(self): + """Return whether we are within a context manager (with).""" + parent = self._stack[-1] if self._stack else None + return parent and isinstance(parent, ast.withitem) + + def _ignore_function_def_node(self, node): + """Should this node be ignored within a FunctionDef.""" + if not self._stack: + return False + parent = self._stack[-1] + # A type annotation or default value of an argument? + if isinstance(parent, (ast.arguments, ast.arg)): + return True + if not isinstance(parent, ast.FunctionDef): + return False + # Return type annotation or decorator call + return node == parent.returns or node in parent.decorator_list + + def visit_Index(self, node): + self._output_file.write("[") + self.generic_visit(node) + self._output_file.write("]") + + def visit_Name(self, node): + """Format a variable reference (cf visit_Attribute)""" + # Skip Context manager variables, return or argument type annotation + if self._within_context_manager() or self._ignore_function_def_node(node): + return + self._output_file.write(format_reference(node)) + + def visit_NameConstant(self, node): + # Default parameter? + if self._ignore_function_def_node(node): + return + self.generic_visit(node) + self._output_file.write(format_name_constant(node)) + + def visit_Not(self, node): + self.generic_visit(node) + self._output_file.write("!") + + def visit_NotEq(self, node): + self.generic_visit(node) + self._output_file.write(" != ") + + def visit_Num(self, node): + self.generic_visit(node) + self._output_file.write(format_literal(node)) + + def visit_RShift(self, node): + self.generic_visit(node) + self._output_file.write(" >> ") + + def visit_Return(self, node): + self.indent_string("return") + if node.value: + self._output_file.write(" ") + self.generic_visit(node) + self._output_file.write(";\n") + + def visit_Slice(self, node): + self._output_file.write("[") + if node.lower: + self.visit(node.lower) + self._output_file.write(":") + if node.upper: + self.visit(node.upper) + self._output_file.write("]") + + def visit_Str(self, node): + self.generic_visit(node) + self._output_file.write(format_literal(node)) + + def visit_Sub(self, node): + self._handle_bin_op(node, "-") + + def visit_UnOp(self, node): + self.generic_visit(node) + + def visit_With(self, node): + self.INDENT() + self._output_file.write("{ // Converted from context manager\n") + self.indent() + for item in node.items: + self.INDENT() + if item.optional_vars: + self._output_file.write(format_reference(item.optional_vars)) + self._output_file.write(" = ") + self.generic_visit(node) + self.dedent() + self.INDENT() + self._output_file.write("}\n") + + def _debug_enter(self, node, parent=None): + message = '{}>generic_visit({})'.format(' ' * self ._debug_indent, + debug_format_node(node)) + if parent: + message += ', parent={}'.format(debug_format_node(parent)) + message += '\n' + sys.stderr.write(message) + self._debug_indent += 1 + + def _debug_leave(self, node): + self._debug_indent -= 1 + message = '{}<generic_visit({})\n'.format(' ' * self ._debug_indent, + type(node).__name__) + sys.stderr.write(message) diff --git a/sources/pyside-tools/requirements-android.txt b/sources/pyside-tools/requirements-android.txt new file mode 100644 index 000000000..1169fd663 --- /dev/null +++ b/sources/pyside-tools/requirements-android.txt @@ -0,0 +1,2 @@ +jinja2 +pkginfo |
