Deployment: add permission support and create macOS bundle application

- Look at the ast of the python files of the application to identify
  the permissions used by the application. Once the permissions
  are identified, pass the necessary NS property list key to be added
  to the Info.plist file to Nuitka.
- For macOS, when deploying create a macOS application bundle (.app)
  by default. This makes it align more with Apple recommendations
  and Qt deployment.
- Fix tests.
- Fix wheel_tester.py to consider .app for macOS.

Task-number: PYSIDE-1612
Task-number: PYSIDE-2468
Change-Id: Ie225c9a92c845b432a8e7eaa791a8aeb86ecd988
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>

7 files changed, 214 insertions, 68 deletions

diff --git a/sources/pyside-tools/deploy.py b/sources/pyside-tools/deploy.py
index b54943ddf..aa03d13d0 100644
--- a/sources/pyside-tools/deploy.py
+++ b/sources/pyside-tools/deploy.py
@@ -39,6 +39,17 @@ from deploy_lib import (MAJOR_VERSION, DesktopConfig, cleanup, config_option_exi
                         HELP_EXTRA_MODULES, HELP_EXTRA_IGNORE_DIRS)
 
 
+TOOL_DESCRIPTION = dedent(f"""
+                          This tool deploys PySide{MAJOR_VERSION} to desktop (Windows, Linux,
+                          macOS) platforms. The following types of executables are produced as per
+                          the platform:
+
+                          Windows = .exe
+                          macOS = .app
+                          Linux = .bin
+                          """)
+
+
 def main(main_file: Path = None, name: str = None, config_file: Path = None, init: bool = False,
          loglevel=logging.WARNING, dry_run: bool = False, keep_deployment_files: bool = False,
          force: bool = False, extra_ignore_dirs: str = None, extra_modules_grouped: str = None):
@@ -123,7 +134,8 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini
                                                qt_plugins=config.qt_plugins,
                                                excluded_qml_plugins=config.excluded_qml_plugins,
                                                icon=config.icon,
-                                               dry_run=dry_run)
+                                               dry_run=dry_run,
+                                               permissions=config.permissions)
     except Exception:
         print(f"[DEPLOY] Exception occurred: {traceback.format_exc()}")
     finally:
@@ -137,11 +149,7 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description=(f"This tool deploys PySide{MAJOR_VERSION} to desktop (Windows, Linux, macOS)"
-                     "platforms"),
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
+    parser = argparse.ArgumentParser(description=TOOL_DESCRIPTION)
 
     parser.add_argument("-c", "--config-file", type=lambda p: Path(p).absolute(),
                         default=(Path.cwd() / "pysidedeploy.spec"),
diff --git a/sources/pyside-tools/deploy_lib/__init__.py b/sources/pyside-tools/deploy_lib/__init__.py
index b3dedd40c..a40d0838b 100644
--- a/sources/pyside-tools/deploy_lib/__init__.py
+++ b/sources/pyside-tools/deploy_lib/__init__.py
@@ -11,7 +11,7 @@ if sys.platform == "win32":
     EXE_FORMAT = ".exe"
 elif sys.platform == "darwin":
     IMAGE_FORMAT = ".icns"
-    EXE_FORMAT = ".bin"
+    EXE_FORMAT = ".app"
 else:
     IMAGE_FORMAT = ".jpg"
     EXE_FORMAT = ".bin"
@@ -52,7 +52,7 @@ def get_all_pyside_modules():
 
 
 from .commands import run_command, run_qmlimportscanner
-from .dependency_util import find_pyside_modules, QtDependencyReader
+from .dependency_util import find_pyside_modules, find_permission_categories, QtDependencyReader
 from .nuitka_helper import Nuitka
 from .config import BaseConfig, Config, DesktopConfig
 from .python_helper import PythonExecutable
diff --git a/sources/pyside-tools/deploy_lib/config.py b/sources/pyside-tools/deploy_lib/config.py
index f1c877cac..2ca6ff895 100644
--- a/sources/pyside-tools/deploy_lib/config.py
+++ b/sources/pyside-tools/deploy_lib/config.py
@@ -1,5 +1,7 @@
 # Copyright (C) 2022 The Qt Company Ltd.
 # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+import sys
 import configparser
 import logging
 import warnings
@@ -8,12 +10,23 @@ from typing import List
 from pathlib import Path
 
 from project import ProjectData
-from . import DEFAULT_APP_ICON, find_pyside_modules, run_qmlimportscanner, QtDependencyReader
+from . import (DEFAULT_APP_ICON, find_pyside_modules, find_permission_categories,
+               QtDependencyReader, run_qmlimportscanner)
 
 # Some QML plugins like QtCore are excluded from this list as they don't contribute much to
 # executable size. Excluding them saves the extra processing of checking for them in files
 EXCLUDED_QML_PLUGINS = {"QtQuick", "QtQuick3D", "QtCharts", "QtWebEngine", "QtTest", "QtSensors"}
 
+PERMISSION_MAP = {"Bluetooth": "NSBluetoothAlwaysUsageDescription:BluetoothAccess",
+                  "Camera": "NSCameraUsageDescription:CameraAccess",
+                  "Microphone": "NSMicrophoneUsageDescription:MicrophoneAccess",
+                  "Contacts": "NSContactsUsageDescription:ContactsAccess",
+                  "Calendar": "NSCalendarsUsageDescription:CalendarAccess",
+                  # for iOS NSLocationWhenInUseUsageDescription and
+                  # NSLocationAlwaysAndWhenInUseUsageDescription are also required.
+                  "Location": "NSLocationUsageDescription:LocationAccess",
+                  }
+
 
 class BaseConfig:
     """Wrapper class around any .spec file with function to read and set values for the .spec file
@@ -22,7 +35,8 @@ class BaseConfig:
                  existing_config_file: bool = False) -> None:
         self.config_file = config_file
         self.existing_config_file = existing_config_file
-        self.parser = ConfigParser(comment_prefixes=comment_prefixes, allow_no_value=True)
+        self.parser = ConfigParser(comment_prefixes=comment_prefixes, strict=False,
+                                   allow_no_value=True)
         self.parser.read(self.config_file)
 
     def update_config(self):
@@ -379,6 +393,14 @@ class DesktopConfig(Config):
         else:
             self.qt_plugins = self.dependency_reader.find_plugin_dependencies(self.modules)
 
+        self._permissions = []
+        if sys.platform == "darwin":
+            nuitka_macos_permissions = self.get_value("nuitka", "macos.permissions")
+            if nuitka_macos_permissions:
+                self._permissions = nuitka_macos_permissions.split(",")
+            else:
+                self._find_and_set_permissions()
+
     @property
     def qt_plugins(self):
         return self._qt_plugins
@@ -388,6 +410,15 @@ class DesktopConfig(Config):
         self._qt_plugins = qt_plugins
         self.set_value("qt", "plugins", ",".join(qt_plugins))
 
+    @property
+    def permissions(self):
+        return self._permissions
+
+    @permissions.setter
+    def permissions(self, permissions):
+        self._permissions = permissions
+        self.set_value("nuitka", "macos.permissions", ",".join(permissions))
+
     def _find_dependent_qt_modules(self):
         """
         Given pysidedeploy_config.modules, find all the other dependent Qt modules.
@@ -404,3 +435,24 @@ class DesktopConfig(Config):
             self.dependency_reader.find_dependencies(module=module_name, used_modules=all_modules)
 
         self.modules = list(all_modules)
+
+    def _find_and_set_permissions(self):
+        """
+        Finds and sets the usage description string required for each permission requested by the
+        macOS application.
+        """
+        permissions = []
+        perm_categories = find_permission_categories(project_dir=self.project_dir,
+                                                     extra_ignore_dirs=self.extra_ignore_dirs,
+                                                     project_data=self.project_data)
+
+        perm_categories_str = ",".join(perm_categories)
+        logging.info(f"[DEPLOY] Usage descriptions for the {perm_categories_str} will be added to "
+                     "the Info.plist file of the macOS application bundle")
+
+        # handling permissions
+        for perm_category in perm_categories:
+            if perm_category in PERMISSION_MAP:
+                permissions.append(PERMISSION_MAP[perm_category])
+
+        self.permissions = permissions
diff --git a/sources/pyside-tools/deploy_lib/default.spec b/sources/pyside-tools/deploy_lib/default.spec
index 8c0697afd..cb488af95 100644
--- a/sources/pyside-tools/deploy_lib/default.spec
+++ b/sources/pyside-tools/deploy_lib/default.spec
@@ -60,6 +60,11 @@ plugins =
 
 [nuitka]
 
+# usage description for permissions requested by the app as found in the Info.plist file
+# of the app bundle
+# eg: NSCameraUsageDescription:CameraAccess
+macos.permissions =
+
 # (str) specify any extra nuitka arguments
 # eg: extra_args = --show-modules --follow-stdlib
 extra_args = --quiet --noinclude-qt-translations
diff --git a/sources/pyside-tools/deploy_lib/dependency_util.py b/sources/pyside-tools/deploy_lib/dependency_util.py
index 53c12ad92..d71640ed0 100644
--- a/sources/pyside-tools/deploy_lib/dependency_util.py
+++ b/sources/pyside-tools/deploy_lib/dependency_util.py
@@ -12,10 +12,120 @@ import shutil
 import sys
 from pathlib import Path
 from typing import List, Set
+from functools import lru_cache
 
 from . import IMPORT_WARNING_PYSIDE, run_command
 
 
+@lru_cache(maxsize=None)
+def get_py_files(project_dir: Path, extra_ignore_dirs: List[Path] = None, project_data=None):
+    """Finds and returns all the Python files in the project
+    """
+    py_candidates = []
+    ignore_dirs = ["__pycache__", "env", "venv", "deployment"]
+
+    if project_data:
+        py_candidates = project_data.python_files
+        ui_candidates = project_data.ui_files
+        qrc_candidates = project_data.qrc_files
+
+        def add_uic_qrc_candidates(candidates, candidate_type):
+            possible_py_candidates = [(file.parent / f"{candidate_type}_{file.stem}.py")
+                                      for file in candidates
+                                      if (file.parent / f"{candidate_type}_{file.stem}.py").exists()
+                                      ]
+
+            if len(possible_py_candidates) != len(candidates):
+                warnings.warn(f"[DEPLOY] The number of {candidate_type} files and their "
+                              "corresponding Python files don't match.",
+                              category=RuntimeWarning)
+
+            py_candidates.extend(possible_py_candidates)
+
+        if ui_candidates:
+            add_uic_qrc_candidates(ui_candidates, "ui")
+
+        if qrc_candidates:
+            add_uic_qrc_candidates(qrc_candidates, "qrc")
+
+        return py_candidates
+
+    # incase there is not .pyproject file, search all python files in project_dir, except
+    # ignore_dirs
+    if extra_ignore_dirs:
+        ignore_dirs.extend(extra_ignore_dirs)
+
+    # find relevant .py files
+    _walk = os.walk(project_dir)
+    for root, dirs, files in _walk:
+        dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")]
+        for py_file in files:
+            if py_file.endswith(".py"):
+                py_candidates.append(Path(root) / py_file)
+
+    return py_candidates
+
+
+@lru_cache(maxsize=None)
+def get_ast(py_file: Path):
+    """Given a Python file returns the abstract syntax tree
+    """
+    contents = py_file.read_text(encoding="utf-8")
+    try:
+        tree = ast.parse(contents)
+    except SyntaxError:
+        print(f"[DEPLOY] Unable to parse {py_file}")
+    return tree
+
+
+def find_permission_categories(project_dir: Path, extra_ignore_dirs: List[Path] = None,
+                               project_data=None):
+    """Given the project directory, finds all the permission categories required by the
+    project. eg: Camera, Bluetooth, Contacts etc.
+
+    Note: This function is only relevant for mac0S deployment.
+    """
+    all_perm_categories = set()
+    mod_pattern = re.compile("Q(?P<mod_name>.*)Permission")
+
+    def pyside_permission_imports(py_file: Path):
+        perm_categories = []
+        try:
+            tree = get_ast(py_file)
+            for node in ast.walk(tree):
+                if isinstance(node, ast.ImportFrom):
+                    main_mod_name = node.module
+                    if main_mod_name == "PySide6.QtCore":
+                        # considers 'from PySide6.QtCore import QtMicrophonePermission'
+                        for imported_module in node.names:
+                            full_mod_name = imported_module.name
+                            match = mod_pattern.search(full_mod_name)
+                            if match:
+                                mod_name = match.group("mod_name")
+                                perm_categories.append(mod_name)
+                        continue
+
+                if isinstance(node, ast.Import):
+                    for imported_module in node.names:
+                        full_mod_name = imported_module.name
+                        if full_mod_name == "PySide6":
+                            logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file)))
+        except Exception as e:
+            raise RuntimeError(f"[DEPLOY] Finding permission categories failed on file "
+                               f"{str(py_file)} with error {e}")
+
+        return set(perm_categories)
+
+    py_candidates = get_py_files(project_dir, extra_ignore_dirs, project_data)
+    for py_candidate in py_candidates:
+        all_perm_categories = all_perm_categories.union(pyside_permission_imports(py_candidate))
+
+    if not all_perm_categories:
+        ValueError("[DEPLOY] No permission categories were found for macOS app bundle creation.")
+
+    return all_perm_categories
+
+
 def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None,
                         project_data=None):
     """
@@ -25,11 +135,10 @@ def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None,
     all_modules = set()
     mod_pattern = re.compile("PySide6.Qt(?P<mod_name>.*)")
 
-    def pyside_imports(py_file: Path):
+    def pyside_module_imports(py_file: Path):
         modules = []
-        contents = py_file.read_text(encoding="utf-8")
         try:
-            tree = ast.parse(contents)
+            tree = get_ast(py_file)
             for node in ast.walk(tree):
                 if isinstance(node, ast.ImportFrom):
                     main_mod_name = node.module
@@ -62,55 +171,9 @@ def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None,
 
         return set(modules)
 
-    py_candidates = []
-    ignore_dirs = ["__pycache__", "env", "venv", "deployment"]
-
-    if project_data:
-        py_candidates = project_data.python_files
-        ui_candidates = project_data.ui_files
-        qrc_candidates = project_data.qrc_files
-        ui_py_candidates = None
-        qrc_ui_candidates = None
-
-        if ui_candidates:
-            ui_py_candidates = [(file.parent / f"ui_{file.stem}.py") for file in ui_candidates
-                                if (file.parent / f"ui_{file.stem}.py").exists()]
-
-            if len(ui_py_candidates) != len(ui_candidates):
-                warnings.warn("[DEPLOY] The number of uic files and their corresponding Python"
-                              " files don't match.", category=RuntimeWarning)
-
-            py_candidates.extend(ui_py_candidates)
-
-        if qrc_candidates:
-            qrc_ui_candidates = [(file.parent / f"rc_{file.stem}.py") for file in qrc_candidates
-                                 if (file.parent / f"rc_{file.stem}.py").exists()]
-
-            if len(qrc_ui_candidates) != len(qrc_candidates):
-                warnings.warn("[DEPLOY] The number of qrc files and their corresponding Python"
-                              " files don't match.", category=RuntimeWarning)
-
-            py_candidates.extend(qrc_ui_candidates)
-
-        for py_candidate in py_candidates:
-            all_modules = all_modules.union(pyside_imports(py_candidate))
-        return list(all_modules)
-
-    # incase there is not .pyproject file, search all python files in project_dir, except
-    # ignore_dirs
-    if extra_ignore_dirs:
-        ignore_dirs.extend(extra_ignore_dirs)
-
-    # find relevant .py files
-    _walk = os.walk(project_dir)
-    for root, dirs, files in _walk:
-        dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")]
-        for py_file in files:
-            if py_file.endswith(".py"):
-                py_candidates.append(Path(root) / py_file)
-
+    py_candidates = get_py_files(project_dir, extra_ignore_dirs, project_data)
     for py_candidate in py_candidates:
-        all_modules = all_modules.union(pyside_imports(py_candidate))
+        all_modules = all_modules.union(pyside_module_imports(py_candidate))
 
     if not all_modules:
         ValueError("[DEPLOY] No PySide6 modules were found")
@@ -197,10 +260,15 @@ class QtDependencyReader:
         dependent_modules = set()
         for line in output.splitlines():
             line = line.decode("utf-8").lstrip()
-            if sys.platform == "darwin" and line.startswith(f"Qt{module} [arm64]"):
-                # macOS Qt frameworks bundles have both x86_64 and arm64 architectures
-                # We only need to consider one as the dependencies are redundant
-                break
+            if sys.platform == "darwin":
+                if line.endswith(f"Qt{module} [arm64]:"):
+                    # macOS Qt frameworks bundles have both x86_64 and arm64 architectures
+                    # We only need to consider one as the dependencies are redundant
+                    break
+                elif line.endswith(f"Qt{module} [X86_64]:"):
+                    # this line needs to be skipped because it matches with the pattern
+                    # and is related to the module itself, not the dependencies of the module
+                    continue
             elif sys.platform == "win32" and line.startswith("Summary"):
                 # the dependencies would be found before the `Summary` line
                 break
diff --git a/sources/pyside-tools/deploy_lib/deploy_util.py b/sources/pyside-tools/deploy_lib/deploy_util.py
index 274b41905..e8b05e990 100644
--- a/sources/pyside-tools/deploy_lib/deploy_util.py
+++ b/sources/pyside-tools/deploy_lib/deploy_util.py
@@ -68,6 +68,10 @@ def finalize(config: Config):
     """
     generated_exec_path = config.generated_files_path / (config.source_file.stem + EXE_FORMAT)
     if generated_exec_path.exists() and config.exe_dir:
-        shutil.copy(generated_exec_path, config.exe_dir)
+        if sys.platform == "darwin":
+            shutil.copytree(generated_exec_path, config.exe_dir / (config.title + EXE_FORMAT),
+                            dirs_exist_ok=True)
+        else:
+            shutil.copy(generated_exec_path, config.exe_dir)
         print("[DEPLOY] Executed file created in "
               f"{str(config.exe_dir / (config.source_file.stem + EXE_FORMAT))}")
diff --git a/sources/pyside-tools/deploy_lib/nuitka_helper.py b/sources/pyside-tools/deploy_lib/nuitka_helper.py
index 721701f70..d202db25e 100644
--- a/sources/pyside-tools/deploy_lib/nuitka_helper.py
+++ b/sources/pyside-tools/deploy_lib/nuitka_helper.py
@@ -46,9 +46,19 @@ class Nuitka:
 
     def create_executable(self, source_file: Path, extra_args: str, qml_files: List[Path],
                           qt_plugins: List[str], excluded_qml_plugins: List[str], icon: str,
-                          dry_run: bool):
+                          dry_run: bool, permissions: List[str]):
         qt_plugins = [plugin for plugin in qt_plugins if plugin not in self.qt_plugins_to_ignore]
         extra_args = extra_args.split()
+
+        if sys.platform == "darwin":
+            # create an app bundle
+            extra_args.extend(["--standalone", "--macos-create-app-bundle"])
+            permission_pattern = "--macos-app-protected-resource={permission}"
+            for permission in permissions:
+                extra_args.append(permission_pattern.format(permission=permission))
+        else:
+            extra_args.append("--onefile")
+
         qml_args = []
         if qml_files:
             # This will generate options for each file using:
@@ -78,7 +88,6 @@ class Nuitka:
         command = self.nuitka + [
             os.fspath(source_file),
             "--follow-imports",
-            "--onefile",
             "--enable-plugin=pyside6",
             f"--output-dir={output_dir}",
         ]