Deployment: More Refactoring and minor bug fixes

- setup_python() moved to constructor of PythonExecutable.
 -install_python_dependencies() moved under PythonExecutable in
  python_helper.py.
- create_executable() of PythonExecutable removed. Instead, we call
  Nuitka.create_executable() directly. This removes unncessary import
  problems when using PythonExecutable class for Android Deployment.
- nuitka==1.8.0 changed to Nuitka=1.8 in default.spec to match with
  the installed version. Otherwise, it forces the reinstall of
  Nuitka==1.8 every time (bug).
- Remove recomputation of qt_plugins and local_libs. If the values
  exist in pysidedeploy.spec, then they should not be computed again.
  This serves the purposes of speeding up the deployment and also
  to no modifying the already existing pysidedeploy.spec.
- find_pyside_modules() moved from python_helper.py to deploy_util.py.
- Adapt tests.
- Remove os.fspath wrapping from python.exe. This is not needed as
 python.exe is already pathlib.Path.

Pick-to: 6.5 6.6
Task-number: PYSIDE-1612
Change-Id: Ic598e57cd2f2779c410b12fc9584cf60c5e94505
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>

7 files changed, 186 insertions, 206 deletions

diff --git a/sources/pyside-tools/android_deploy.py b/sources/pyside-tools/android_deploy.py
index 83350bc5f..362fb3766 100644
--- a/sources/pyside-tools/android_deploy.py
+++ b/sources/pyside-tools/android_deploy.py
@@ -8,8 +8,8 @@ import traceback
 from pathlib import Path
 from textwrap import dedent
 
-from deploy_lib import (setup_python, create_config_file, cleanup, install_python_dependencies,
-                        config_option_exists, MAJOR_VERSION)
+from deploy_lib import (create_config_file, cleanup, config_option_exists, PythonExecutable,
+                        MAJOR_VERSION)
 from deploy_lib.android import AndroidData, AndroidConfig
 from deploy_lib.android.buildozer import Buildozer
 
@@ -96,7 +96,7 @@ def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = Non
     android_data = AndroidData(wheel_pyside=pyside_wheel, wheel_shiboken=shiboken_wheel,
                                ndk_path=ndk_path, sdk_path=sdk_path)
 
-    python = setup_python(dry_run=dry_run, force=force, init=init)
+    python = PythonExecutable(dry_run=dry_run, init=init, force=force)
 
     config_file_exists = config_file and Path(config_file).exists()
 
@@ -117,8 +117,7 @@ def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = Non
 
     cleanup(config=config, is_android=True)
 
-    install_python_dependencies(config=config, python=python, init=init,
-                                packages="android_packages", is_android=True)
+    python.install_dependencies(config=config, packages="android_packages", is_android=True)
 
     # set application name
     if name:
diff --git a/sources/pyside-tools/deploy.py b/sources/pyside-tools/deploy.py
index 0aea807a8..576c01f9d 100644
--- a/sources/pyside-tools/deploy.py
+++ b/sources/pyside-tools/deploy.py
@@ -35,8 +35,7 @@ from pathlib import Path
 from textwrap import dedent
 
 from deploy_lib import (MAJOR_VERSION, Config, cleanup, config_option_exists,
-                        finalize, create_config_file, install_python_dependencies,
-                        setup_python)
+                        finalize, create_config_file, PythonExecutable, Nuitka)
 
 
 def main(main_file: Path = None, name: str = None, config_file: Path = None, init: bool = False,
@@ -57,7 +56,7 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini
     config = None
     logging.info("[DEPLOY] Start")
 
-    python = setup_python(dry_run=dry_run, force=force, init=init)
+    python = PythonExecutable(dry_run=dry_run, init=init, force=force)
     config_file_exists = config_file and Path(config_file).exists()
 
     if config_file_exists:
@@ -75,8 +74,7 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini
 
     cleanup(config=config)
 
-    install_python_dependencies(config=config, python=python, init=init,
-                                packages="packages")
+    python.install_dependencies(config=config, packages="packages")
 
     # required by Nuitka for pyenv Python
     add_arg = " --static-libpython=no"
@@ -89,6 +87,9 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini
     if not dry_run:
         config.update_config()
 
+    if config.qml_files:
+        logging.info(f"[DEPLOY] Included QML files: {config.qml_files}")
+
     if init:
         # config file created above. Exiting.
         logging.info(f"[DEPLOY]: Config file {config.config_file} created")
@@ -99,9 +100,13 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini
         if not dry_run:
             logging.info("[DEPLOY] Deploying application")
 
-        command_str = python.create_executable(source_file=config.source_file,
+        nuitka = Nuitka(nuitka=[python.exe, "-m", "nuitka"])
+        command_str = nuitka.create_executable(source_file=config.source_file,
                                                extra_args=config.extra_args,
-                                               config=config)
+                                               qml_files=config.qml_files,
+                                               excluded_qml_plugins=config.excluded_qml_plugins,
+                                               icon=config.icon,
+                                               dry_run=dry_run)
     except Exception:
         print(f"[DEPLOY] Exception occurred: {traceback.format_exc()}")
     finally:
diff --git a/sources/pyside-tools/deploy_lib/__init__.py b/sources/pyside-tools/deploy_lib/__init__.py
index 40a7535db..27d178eee 100644
--- a/sources/pyside-tools/deploy_lib/__init__.py
+++ b/sources/pyside-tools/deploy_lib/__init__.py
@@ -16,6 +16,9 @@ else:
     EXE_FORMAT = ".bin"
 
 DEFAULT_APP_ICON = str((Path(__file__).parent / f"pyside_icon{IMAGE_FORMAT}").resolve())
+IMPORT_WARNING_PYSIDE = (f"[DEPLOY] Found 'import PySide6' in file {0}"
+                         ". Use 'from PySide6 import <module>' or pass the module"
+                         " needed using --extra-modules command line argument")
 
 
 def get_all_pyside_modules():
@@ -30,7 +33,7 @@ def get_all_pyside_modules():
 
 from .commands import run_command, run_qmlimportscanner
 from .nuitka_helper import Nuitka
-from .python_helper import PythonExecutable, find_pyside_modules
 from .config import BaseConfig, Config
-from .deploy_util import (cleanup, finalize, create_config_file, setup_python,
-                          install_python_dependencies, config_option_exists)
+from .python_helper import PythonExecutable
+from .deploy_util import (cleanup, finalize, create_config_file,
+                          config_option_exists, find_pyside_modules)
diff --git a/sources/pyside-tools/deploy_lib/android/android_config.py b/sources/pyside-tools/deploy_lib/android/android_config.py
index 1ea99411f..8054ce373 100644
--- a/sources/pyside-tools/deploy_lib/android/android_config.py
+++ b/sources/pyside-tools/deploy_lib/android/android_config.py
@@ -121,20 +121,21 @@ class AndroidConfig(Config):
         self._dependency_files = []
         self._find_and_set_dependency_files()
 
-        self._qt_plugins = []
-        if self.get_value("android", "plugins"):
-            self._qt_plugins = self.get_value("android", "plugins").split(",")
-
+        dependent_plugins = []
         self._local_libs = []
         if self.get_value("buildozer", "local_libs"):
-            self.local_libs = self.get_value("buildozer", "local_libs").split(",")
+            self._local_libs = self.get_value("buildozer", "local_libs").split(",")
+        else:
+            # the local_libs can also store dependent plugins
+            local_libs, dependent_plugins = self._find_local_libs()
+            self.local_libs = list(set(local_libs))
 
-        dependent_plugins = []
-        # the local_libs can also store dependent plugins
-        local_libs, dependent_plugins = self._find_local_libs()
-        self._find_plugin_dependencies(dependent_plugins)
-        self.qt_plugins += dependent_plugins
-        self.local_libs += local_libs
+        self._qt_plugins = []
+        if self.get_value("android", "plugins"):
+            self._qt_plugins = self.get_value("android", "plugins").split(",")
+        elif dependent_plugins:
+            self._find_plugin_dependencies(dependent_plugins)
+            self.qt_plugins = list(set(dependent_plugins))
 
         recipe_dir_temp = self.get_value("buildozer", "recipe_dir")
         if recipe_dir_temp:
@@ -382,11 +383,6 @@ class AndroidConfig(Config):
                     # eg: lib<lib_name>_x86_64.so
                     file_name = Path(file).stem
 
-                    if file_name.startswith("libplugins_platforms_qtforandroid"):
-                        # the platform library is a requisite and is already added from the
-                        # configuration file
-                        continue
-
                     # we only need lib_name, because lib and arch gets re-added by
                     # python-for-android
                     match = lib_pattern.search(file_name)
diff --git a/sources/pyside-tools/deploy_lib/default.spec b/sources/pyside-tools/deploy_lib/default.spec
index 7ca2edfe7..185a2cd5a 100644
--- a/sources/pyside-tools/deploy_lib/default.spec
+++ b/sources/pyside-tools/deploy_lib/default.spec
@@ -27,7 +27,7 @@ python_path =
 # python packages to install
 # ordered-set: increase compile time performance of nuitka packaging
 # zstandard: provides final executable size optimization
-packages = nuitka==1.8.0,ordered_set,zstandard
+packages = Nuitka==1.8,ordered_set,zstandard
 
 # buildozer: for deploying Android application
 android_packages = buildozer==1.5.0,cython==0.29.33
@@ -50,7 +50,7 @@ wheel_pyside =
 wheel_shiboken =
 
 # plugins to be copied to libs folder of the packaged application. Comma separated
-plugins = platforms_qtforandroid
+plugins =
 
 [nuitka]
 
@@ -82,7 +82,7 @@ modules =
 
 # other libraries to be loaded. Comma separated.
 # loaded at app startup
-local_libs = plugins_platforms_qtforandroid
+local_libs =
 
 # architecture of deployed platform
 # possible values: ["aarch64", "armv7a", "i686", "x86_64"]
diff --git a/sources/pyside-tools/deploy_lib/deploy_util.py b/sources/pyside-tools/deploy_lib/deploy_util.py
index a8ca58611..b20c9c8cb 100644
--- a/sources/pyside-tools/deploy_lib/deploy_util.py
+++ b/sources/pyside-tools/deploy_lib/deploy_util.py
@@ -1,14 +1,18 @@
 # Copyright (C) 2023 The Qt Company Ltd.
 # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
+import ast
+import re
+import os
+import warnings
 import logging
 import shutil
 import sys
 from pathlib import Path
+from typing import List
 
-from . import EXE_FORMAT
+from . import EXE_FORMAT, IMPORT_WARNING_PYSIDE
 from .config import Config
-from .python_helper import PythonExecutable
 
 
 def config_option_exists():
@@ -62,49 +66,6 @@ def create_config_file(dry_run: bool = False, config_file: Path = None, main_fil
     return config_file
 
 
-def setup_python(dry_run: bool, force: bool, init: bool):
-    """
-        Sets up Python venv for deployment, and return a wrapper around the venv environment
-    """
-    python = None
-    response = "yes"
-    # checking if inside virtual environment
-    if not PythonExecutable.is_venv() and not force and not dry_run and not init:
-        response = input(("You are not using a virtual environment. pyside6-deploy needs to install"
-                          " a few Python packages for deployment to work seamlessly. \n"
-                          "Proceed? [Y/n]"))
-
-    if response.lower() in ["no", "n"]:
-        print("[DEPLOY] Exiting ...")
-        sys.exit(0)
-
-    python = PythonExecutable(dry_run=dry_run)
-    logging.info(f"[DEPLOY] Using python at {sys.executable}")
-
-    return python
-
-
-def install_python_dependencies(config: Config, python: PythonExecutable, init: bool,
-                                packages: str, is_android: bool = False):
-    """
-        Installs the python package dependencies for the target deployment platform
-    """
-    packages = config.get_value("python", packages).split(",")
-    if not init:
-        # install packages needed for deployment
-        logging.info("[DEPLOY] Installing dependencies")
-        python.install(packages=packages)
-        # nuitka requires patchelf to make patchelf rpath changes for some Qt files
-        if sys.platform.startswith("linux") and not is_android:
-            python.install(packages=["patchelf"])
-    elif is_android:
-        # install only buildozer
-        logging.info("[DEPLOY] Installing buildozer")
-        buildozer_package_with_version = ([package for package in packages
-                                          if package.startswith("buildozer")])
-        python.install(packages=list(buildozer_package_with_version))
-
-
 def finalize(config: Config):
     """
         Copy the executable into the final location
@@ -115,3 +76,105 @@ def finalize(config: Config):
         shutil.copy(generated_exec_path, config.exe_dir)
         print("[DEPLOY] Executed file created in "
               f"{str(config.exe_dir / (config.source_file.stem + EXE_FORMAT))}")
+
+
+def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None,
+                        project_data=None):
+    """
+    Searches all the python files in the project to find all the PySide modules used by
+    the application.
+    """
+    all_modules = set()
+    mod_pattern = re.compile("PySide6.Qt(?P<mod_name>.*)")
+
+    def pyside_imports(py_file: Path):
+        modules = []
+        contents = py_file.read_text(encoding="utf-8")
+        try:
+            tree = ast.parse(contents)
+            for node in ast.walk(tree):
+                if isinstance(node, ast.ImportFrom):
+                    main_mod_name = node.module
+                    if main_mod_name.startswith("PySide6"):
+                        if main_mod_name == "PySide6":
+                            # considers 'from PySide6 import QtCore'
+                            for imported_module in node.names:
+                                full_mod_name = imported_module.name
+                                if full_mod_name.startswith("Qt"):
+                                    modules.append(full_mod_name[2:])
+                            continue
+
+                        # considers 'from PySide6.QtCore import Qt'
+                        match = mod_pattern.search(main_mod_name)
+                        if match:
+                            mod_name = match.group("mod_name")
+                            modules.append(mod_name)
+                        else:
+                            logging.warning((
+                                f"[DEPLOY] Unable to find module name from{ast.dump(node)}"))
+
+                if isinstance(node, ast.Import):
+                    for imported_module in node.names:
+                        full_mod_name = imported_module.name
+                        if full_mod_name == "PySide6":
+                            logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file)))
+        except Exception as e:
+            raise RuntimeError(f"[DEPLOY] Finding module import failed on file {str(py_file)} with "
+                               f"error {e}")
+
+        return set(modules)
+
+    py_candidates = []
+    ignore_dirs = ["__pycache__", "env", "venv", "deployment"]
+
+    if project_data:
+        py_candidates = project_data.python_files
+        ui_candidates = project_data.ui_files
+        qrc_candidates = project_data.qrc_files
+        ui_py_candidates = None
+        qrc_ui_candidates = None
+
+        if ui_candidates:
+            ui_py_candidates = [(file.parent / f"ui_{file.stem}.py") for file in ui_candidates
+                                if (file.parent / f"ui_{file.stem}.py").exists()]
+
+            if len(ui_py_candidates) != len(ui_candidates):
+                warnings.warn("[DEPLOY] The number of uic files and their corresponding Python"
+                              " files don't match.", category=RuntimeWarning)
+
+            py_candidates.extend(ui_py_candidates)
+
+        if qrc_candidates:
+            qrc_ui_candidates = [(file.parent / f"rc_{file.stem}.py") for file in qrc_candidates
+                                 if (file.parent / f"rc_{file.stem}.py").exists()]
+
+            if len(qrc_ui_candidates) != len(qrc_candidates):
+                warnings.warn("[DEPLOY] The number of qrc files and their corresponding Python"
+                              " files don't match.", category=RuntimeWarning)
+
+            py_candidates.extend(qrc_ui_candidates)
+
+        for py_candidate in py_candidates:
+            all_modules = all_modules.union(pyside_imports(py_candidate))
+        return list(all_modules)
+
+    # incase there is not .pyproject file, search all python files in project_dir, except
+    # ignore_dirs
+    if extra_ignore_dirs:
+        ignore_dirs.extend(extra_ignore_dirs)
+
+    # find relevant .py files
+    _walk = os.walk(project_dir)
+    for root, dirs, files in _walk:
+        dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")]
+        for py_file in files:
+            if py_file.endswith(".py"):
+                py_candidates.append(Path(root) / py_file)
+
+    for py_candidate in py_candidates:
+        all_modules = all_modules.union(pyside_imports(py_candidate))
+
+    if not all_modules:
+        ValueError("[DEPLOY] No PySide6 modules were found")
+
+    return list(all_modules)
diff --git a/sources/pyside-tools/deploy_lib/python_helper.py b/sources/pyside-tools/deploy_lib/python_helper.py
index 6ec3b64f8..7cbf323ed 100644
--- a/sources/pyside-tools/deploy_lib/python_helper.py
+++ b/sources/pyside-tools/deploy_lib/python_helper.py
@@ -1,124 +1,15 @@
 # Copyright (C) 2022 The Qt Company Ltd.
 # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
-import ast
 import logging
 import os
-import re
 import sys
-import warnings
-from typing import List
+
 from importlib import util
 from importlib.metadata import version
 from pathlib import Path
 
-from . import Nuitka, run_command
-
-IMPORT_WARNING_PYSIDE = (f"[DEPLOY] Found 'import PySide6' in file {0}"
-                         ". Use 'from PySide6 import <module>' or pass the module"
-                         " needed using --extra-modules command line argument")
-
-
-def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None,
-                        project_data=None):
-    """
-    Searches all the python files in the project to find all the PySide modules used by
-    the application.
-    """
-    all_modules = set()
-    mod_pattern = re.compile("PySide6.Qt(?P<mod_name>.*)")
-
-    def pyside_imports(py_file: Path):
-        modules = []
-        contents = py_file.read_text(encoding="utf-8")
-        try:
-            tree = ast.parse(contents)
-            for node in ast.walk(tree):
-                if isinstance(node, ast.ImportFrom):
-                    main_mod_name = node.module
-                    if main_mod_name.startswith("PySide6"):
-                        if main_mod_name == "PySide6":
-                            # considers 'from PySide6 import QtCore'
-                            for imported_module in node.names:
-                                full_mod_name = imported_module.name
-                                if full_mod_name.startswith("Qt"):
-                                    modules.append(full_mod_name[2:])
-                            continue
-
-                        # considers 'from PySide6.QtCore import Qt'
-                        match = mod_pattern.search(main_mod_name)
-                        if match:
-                            mod_name = match.group("mod_name")
-                            modules.append(mod_name)
-                        else:
-                            logging.warning((
-                                f"[DEPLOY] Unable to find module name from{ast.dump(node)}"))
-
-                if isinstance(node, ast.Import):
-                    for imported_module in node.names:
-                        full_mod_name = imported_module.name
-                        if full_mod_name == "PySide6":
-                            logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file)))
-        except Exception as e:
-            raise RuntimeError(f"[DEPLOY] Finding module import failed on file {str(py_file)} with "
-                               f"error {e}")
-
-        return set(modules)
-
-    py_candidates = []
-    ignore_dirs = ["__pycache__", "env", "venv", "deployment"]
-
-    if project_data:
-        py_candidates = project_data.python_files
-        ui_candidates = project_data.ui_files
-        qrc_candidates = project_data.qrc_files
-        ui_py_candidates = None
-        qrc_ui_candidates = None
-
-        if ui_candidates:
-            ui_py_candidates = [(file.parent / f"ui_{file.stem}.py") for file in ui_candidates
-                                if (file.parent / f"ui_{file.stem}.py").exists()]
-
-            if len(ui_py_candidates) != len(ui_candidates):
-                warnings.warn("[DEPLOY] The number of uic files and their corresponding Python"
-                              " files don't match.", category=RuntimeWarning)
-
-            py_candidates.extend(ui_py_candidates)
-
-        if qrc_candidates:
-            qrc_ui_candidates = [(file.parent / f"rc_{file.stem}.py") for file in qrc_candidates
-                                 if (file.parent / f"rc_{file.stem}.py").exists()]
-
-            if len(qrc_ui_candidates) != len(qrc_candidates):
-                warnings.warn("[DEPLOY] The number of qrc files and their corresponding Python"
-                              " files don't match.", category=RuntimeWarning)
-
-            py_candidates.extend(qrc_ui_candidates)
-
-        for py_candidate in py_candidates:
-            all_modules = all_modules.union(pyside_imports(py_candidate))
-        return list(all_modules)
-
-    # incase there is not .pyproject file, search all python files in project_dir, except
-    # ignore_dirs
-    if extra_ignore_dirs:
-        ignore_dirs.extend(extra_ignore_dirs)
-
-    # find relevant .py files
-    _walk = os.walk(project_dir)
-    for root, dirs, files in _walk:
-        dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")]
-        for py_file in files:
-            if py_file.endswith(".py"):
-                py_candidates.append(Path(root) / py_file)
-
-    for py_candidate in py_candidates:
-        all_modules = all_modules.union(pyside_imports(py_candidate))
-
-    if not all_modules:
-        ValueError("[DEPLOY] No PySide6 modules were found")
-
-    return list(all_modules)
+from . import Config, run_command
 
 
 class PythonExecutable:
@@ -126,10 +17,28 @@ class PythonExecutable:
     Wrapper class around Python executable
     """
 
-    def __init__(self, python_path=None, dry_run=False):
-        self.exe = python_path if python_path else Path(sys.executable)
+    def __init__(self, python_path: Path = None, dry_run: bool = False, init: bool = False,
+                 force: bool = False):
+
         self.dry_run = dry_run
-        self.nuitka = Nuitka(nuitka=[os.fspath(self.exe), "-m", "nuitka"])
+        self.init = init
+        if not python_path:
+            response = "yes"
+            # checking if inside virtual environment
+            if not self.is_venv() and not force and not self.dry_run and not self.init:
+                response = input(("You are not using a virtual environment. pyside6-deploy needs "
+                                  "to install a few Python packages for deployment to work "
+                                  "seamlessly. \n Proceed? [Y/n]"))
+
+            if response.lower() in ["no", "n"]:
+                print("[DEPLOY] Exiting ...")
+                sys.exit(0)
+
+            self.exe = Path(sys.executable)
+        else:
+            self.exe = python_path
+
+        logging.info(f"[DEPLOY] Using Python at {str(self.exe)}")
 
     @property
     def exe(self):
@@ -193,16 +102,21 @@ class PythonExecutable:
     def is_installed(self, package):
         return bool(util.find_spec(package))
 
-    def create_executable(self, source_file: Path, extra_args: str, config):
-        if config.qml_files:
-            logging.info(f"[DEPLOY] Included QML files: {config.qml_files}")
-
-        command_str = self.nuitka.create_executable(source_file=source_file,
-                                                    extra_args=extra_args,
-                                                    qml_files=config.qml_files,
-                                                    excluded_qml_plugins=(config.
-                                                                          excluded_qml_plugins),
-                                                    icon=config.icon,
-                                                    dry_run=self.dry_run)
-
-        return command_str
+    def install_dependencies(self, config: Config, packages: str, is_android: bool = False):
+        """
+        Installs the python package dependencies for the target deployment platform
+        """
+        packages = config.get_value("python", packages).split(",")
+        if not self.init:
+            # install packages needed for deployment
+            logging.info("[DEPLOY] Installing dependencies")
+            self.install(packages=packages)
+            # nuitka requires patchelf to make patchelf rpath changes for some Qt files
+            if sys.platform.startswith("linux") and not is_android:
+                self.install(packages=["patchelf"])
+        elif is_android:
+            # install only buildozer
+            logging.info("[DEPLOY] Installing buildozer")
+            buildozer_package_with_version = ([package for package in packages
+                                               if package.startswith("buildozer")])
+            self.install(packages=list(buildozer_package_with_version))