Deployment tool: Code fixes

- `--config-file` now defaults to pysidedeploy.spec. This enables to
  automatically find the config file, in case if it is present,
  without specifying it through the `--config-file` option.
- Use `pip freeze` also to check if a Python package in installed. This
  is useful for packages like `patchelf` which does not provide
  a Python module, but only an executable.
- Change some `logging.exception()` to actual exceptions because the
  the exception requires some manual intervention for resolution.
- Some of the config options were earlier reevaluated even when an
  existing config file exists. This is now adapted to skip the
  reevaluation when a config file exists.
- In case of pyenv python, add `--static-libpython=no` as extra argument
  accepted by Nuitka. This is because pyenv Python uses `--enable-shared`
  by default since release 2.3.10 -
  https://github.com/pyenv/pyenv/pull/2554
- Some general fixes related to logging.

Task-number: PYSIDE-1612
Pick-to: 6.6
Change-Id: Ib0fa0ca0ec3a08c07140c0c2fa32f422658d04d8
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

6 files changed, 57 insertions, 33 deletions

diff --git a/sources/pyside-tools/deploy.py b/sources/pyside-tools/deploy.py
index 51c10664b..e13d97385 100644
--- a/sources/pyside-tools/deploy.py
+++ b/sources/pyside-tools/deploy.py
@@ -75,6 +75,10 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini
     install_python_dependencies(config=config, python=python, init=init,
                                 packages="packages")
 
+    # required by Nuitka for pyenv Python
+    if python.is_pyenv_python():
+        config.extra_args += " --static-libpython=no"
+
     # writing config file
     # in the case of --dry-run, we use default.spec as reference. Do not save the changes
     # for --dry-run
@@ -93,7 +97,7 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini
 
         command_str = python.create_executable(
                         source_file=source_file,
-                        extra_args=config.get_value("nuitka", "extra_args"),
+                        extra_args=config.extra_args,
                         config=config,
                     )
     except Exception:
@@ -116,6 +120,7 @@ if __name__ == "__main__":
     )
 
     parser.add_argument("-c", "--config-file", type=lambda p: Path(p).absolute(),
+                        default=(Path.cwd() / "pysidedeploy.spec"),
                         help="Path to the .spec config file")
 
     parser.add_argument(
diff --git a/sources/pyside-tools/deploy_lib/android/android_helper.py b/sources/pyside-tools/deploy_lib/android/android_helper.py
index c10bdd994..58efa0943 100644
--- a/sources/pyside-tools/deploy_lib/android/android_helper.py
+++ b/sources/pyside-tools/deploy_lib/android/android_helper.py
@@ -54,7 +54,7 @@ def create_recipe(version: str, component: str, wheel_path: str, generated_files
 
     recipe_path = generated_files_path / "recipes" / f"{component}"
     recipe_path.mkdir(parents=True, exist_ok=True)
-    logging.info(f"[DEPLOY] Writing {component} recipe into {recipe_path}")
+    logging.info(f"[DEPLOY] Writing {component} recipe into {str(recipe_path)}")
     with open(recipe_path / "__init__.py", mode="w", encoding="utf-8") as recipe:
         recipe.write(content)
 
diff --git a/sources/pyside-tools/deploy_lib/commands.py b/sources/pyside-tools/deploy_lib/commands.py
index 53ad633ea..44785533a 100644
--- a/sources/pyside-tools/deploy_lib/commands.py
+++ b/sources/pyside-tools/deploy_lib/commands.py
@@ -27,17 +27,15 @@ def run_command(command, dry_run: bool, fetch_output: bool = False):
         else:
             print(command_str + "\n")
     except FileNotFoundError as error:
-        logging.exception(f"[DEPLOY] {error.filename} not found")
-        raise
+        raise FileNotFoundError(f"[DEPLOY] {error.filename} not found")
     except subprocess.CalledProcessError as error:
-        logging.exception(
+        raise RuntimeError(
             f"[DEPLOY] Command {command_str} failed with error {error} and return_code"
             f"{error.returncode}"
         )
-        raise
     except Exception as error:
-        logging.exception(f"[DEPLOY] Command {command_str} failed with error {error}")
-        raise
+        raise RuntimeError(f"[DEPLOY] Command {command_str} failed with error {error}")
+
     return command_str, output
 
 
diff --git a/sources/pyside-tools/deploy_lib/config.py b/sources/pyside-tools/deploy_lib/config.py
index 5fcc54795..b2b074a70 100644
--- a/sources/pyside-tools/deploy_lib/config.py
+++ b/sources/pyside-tools/deploy_lib/config.py
@@ -1,10 +1,11 @@
 # Copyright (C) 2022 The Qt Company Ltd.
 # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
-from pathlib import Path
 import configparser
-from configparser import ConfigParser
 import logging
+import warnings
+from pathlib import Path
+from configparser import ConfigParser
 
 from project import ProjectData
 from .commands import run_qmlimportscanner
@@ -19,11 +20,11 @@ ANDROID_DEPLOY_CACHE = Path.home() / ".pyside6_android_deploy"
 
 class BaseConfig:
 
-    def __init__(self, config_file: Path, comment_prefixes: str = "/") -> None:
+    def __init__(self, config_file: Path, comment_prefixes: str = "/",
+                 existing_config_file: bool = False) -> None:
         self.config_file = config_file
+        self.existing_config_file = existing_config_file
         self.parser = ConfigParser(comment_prefixes=comment_prefixes, allow_no_value=True)
-        if not config_file.exists():
-            raise RuntimeError(f"[DEPLOY] {config_file} does not exist")
         self.parser.read(self.config_file)
 
     def update_config(self):
@@ -61,8 +62,8 @@ class Config(BaseConfig):
     """
 
     def __init__(self, config_file: Path, source_file: Path, python_exe: Path, dry_run: bool,
-                 android_data, is_android: bool):
-        super().__init__(config_file)
+                 android_data, is_android: bool, existing_config_file: bool = False):
+        super().__init__(config_file=config_file, existing_config_file=existing_config_file)
 
         self._dry_run = dry_run
         # set source_file
@@ -102,13 +103,13 @@ class Config(BaseConfig):
 
         self.qml_files = []
         config_qml_files = self.get_value("qt", "qml_files")
-        if config_qml_files and self.project_dir:
+        if config_qml_files and self.project_dir and self.existing_config_file:
             self.qml_files = [Path(self.project_dir) / file for file in config_qml_files.split(",")]
         else:
             self._find_and_set_qml_files()
 
         self.excluded_qml_plugins = []
-        if self.get_value("qt", "excluded_qml_plugins"):
+        if self.get_value("qt", "excluded_qml_plugins") and self.existing_config_file:
             self.excluded_qml_plugins = self.get_value("qt", "excluded_qml_plugins").split(",")
         else:
             self._find_and_set_excluded_qml_plugins()
@@ -119,13 +120,17 @@ class Config(BaseConfig):
                 self.wheel_pyside = android_data.wheel_pyside
             else:
                 wheel_pyside_temp = self.get_value("qt", "wheel_pyside")
-                self.wheel_pyside = Path(wheel_pyside_temp) if wheel_pyside_temp else None
+                if not wheel_pyside_temp:
+                    raise RuntimeError("[DEPLOY] Unable to find PySide6 Android wheel")
+                self.wheel_pyside = Path(wheel_pyside_temp).resolve()
 
             if android_data.wheel_shiboken:
                 self.wheel_shiboken = android_data.wheel_shiboken
             else:
                 wheel_shiboken_temp = self.get_value("qt", "wheel_shiboken")
-                self.wheel_shiboken = Path(wheel_shiboken_temp) if wheel_shiboken_temp else None
+                if not wheel_shiboken_temp:
+                    raise RuntimeError("[DEPLOY] Unable to find shiboken6 Android wheel")
+                self.wheel_shiboken = Path(wheel_shiboken_temp).resolve()
 
             if android_data.ndk_path:
                 self.ndk_path = android_data.ndk_path
@@ -241,6 +246,14 @@ class Config(BaseConfig):
         self._python_path = python_path
 
     @property
+    def extra_args(self):
+        return self.get_value("nuitka", "extra_args")
+
+    @extra_args.setter
+    def extra_args(self, extra_args):
+        self.set_value("nuitka", "extra_args", extra_args)
+
+    @property
     def excluded_qml_plugins(self):
         return self._excluded_qml_plugins
 
@@ -401,12 +414,12 @@ class Config(BaseConfig):
 
                 if len(qml_files_temp) > 500:
                     if "site-packages" in str(qml_files_temp[-1]):
-                        logging.warning(
-                            "You seem to include a lot of QML files from a local virtual env."
-                            "This can lead to errors in deployment."
+                        raise RuntimeError(
+                            "You are including a lot of QML files from a local virtual env."
+                            " This can lead to errors in deployment."
                         )
                     else:
-                        logging.warning(
+                        warnings.warn(
                             "You seem to include a lot of QML files. This can lead to errors in "
                             "deployment."
                         )
@@ -429,18 +442,14 @@ class Config(BaseConfig):
         self.project_dir = self.source_file.parent
 
         # update input_file path
-        logging.info("[DEPLOY] Update input_file path")
         self.set_value("app", "input_file", str(self.source_file.relative_to(self.project_dir)))
 
-        logging.info("[DEPLOY] Update project_dir path")
         if self.project_dir != Path.cwd():
             self.set_value("app", "project_dir", str(self.project_dir))
         else:
             self.set_value("app", "project_dir", str(self.project_dir.relative_to(Path.cwd())))
 
     def _find_and_set_project_file(self):
-        logging.info("[DEPLOY] Searching for .pyproject file")
-
         if self.project_dir:
             files = list(self.project_dir.glob("*.pyproject"))
         else:
diff --git a/sources/pyside-tools/deploy_lib/deploy_util.py b/sources/pyside-tools/deploy_lib/deploy_util.py
index bb986344c..02a7ed39e 100644
--- a/sources/pyside-tools/deploy_lib/deploy_util.py
+++ b/sources/pyside-tools/deploy_lib/deploy_util.py
@@ -26,8 +26,6 @@ def cleanup(generated_files_path: Path, config: Config, is_android: bool = False
     if generated_files_path.exists():
         shutil.rmtree(generated_files_path)
         logging.info("[DEPLOY] Deployment directory purged")
-    elif not config.dry_run:
-        logging.info(f"[DEPLOY] {generated_files_path} does not exist")
 
     if is_android:
         buildozer_spec: Path = config.project_dir / "buildozer.spec"
@@ -68,7 +66,8 @@ def get_config(python_exe: Path, dry_run: bool = False, config_file: Path = None
         config_file = Path(__file__).parent / "default.spec"
 
     config = Config(config_file=config_file, source_file=main_file, python_exe=python_exe,
-                    dry_run=dry_run, android_data=android_data, is_android=is_android)
+                    dry_run=dry_run, android_data=android_data, is_android=is_android,
+                    existing_config_file=config_file_exists)
 
     return config
 
@@ -102,7 +101,7 @@ def install_python_dependencies(config: Config, python: PythonExecutable, init:
     """
     if not init:
         # install packages needed for deployment
-        logging.info("[DEPLOY] Installing dependencies \n")
+        logging.info("[DEPLOY] Installing dependencies")
         packages = config.get_value("python", packages).split(",")
         python.install(packages=packages)
         # nuitka requires patchelf to make patchelf rpath changes for some Qt files
diff --git a/sources/pyside-tools/deploy_lib/python_helper.py b/sources/pyside-tools/deploy_lib/python_helper.py
index 1a48b0e9a..32bf04f2c 100644
--- a/sources/pyside-tools/deploy_lib/python_helper.py
+++ b/sources/pyside-tools/deploy_lib/python_helper.py
@@ -148,7 +148,20 @@ class PythonExecutable:
         venv = os.environ.get("VIRTUAL_ENV")
         return True if venv else False
 
+    def is_pyenv_python(self):
+        pyenv_root = os.environ.get("PYENV_ROOT")
+
+        if pyenv_root:
+            resolved_exe = self.exe.resolve()
+            if str(resolved_exe).startswith(pyenv_root):
+                return True
+
+        return False
+
     def install(self, packages: list = None):
+        _, installed_packages = run_command(command=[str(self.exe), "-m", "pip", "freeze"], dry_run=False
+                                            , fetch_output=True)
+        installed_packages = [p.decode().split('==')[0] for p in installed_packages.split()]
         for package in packages:
             package_info = package.split('==')
             package_components_len = len(package_info)
@@ -160,7 +173,7 @@ class PythonExecutable:
                 package_version = package_info[1]
             else:
                 raise ValueError(f"{package} should be of the format 'package_name'=='version'")
-            if not self.is_installed(package=package_name):
+            if (package_name not in installed_packages) and (not self.is_installed(package_name)):
                 logging.info(f"[DEPLOY] Installing package: {package}")
                 run_command(
                     command=[self.exe, "-m", "pip", "install", package],
@@ -177,7 +190,7 @@ class PythonExecutable:
                     )
                 else:
                     logging.info(f"[DEPLOY] package: {package_name}=={package_version}"
-                                 "already installed")
+                                 " already installed")
             else:
                 logging.info(f"[DEPLOY] package: {package_name} already installed")