Desktop Deployment: Optimize the plugins included

- Applications that use certain modules like Multimedia does not work
  because the plugins for it were not included. However, including all
  the plugins can make the application executable huge. This patch
  filters out the necessary plugins by looking at
  PySide6_Essentials.json and PySide6_Addons.json shipped with the
  wheels and only bundles these necessary plugins with the application.
- Adjust tests.

Task-number: PYSIDE-1612
Task-number: PYSIDE-2597
Change-Id: I35c74907a1782ae5101fb7c0861adcb97db5792d
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>

5 files changed, 100 insertions, 29 deletions

diff --git a/sources/pyside-tools/deploy.py b/sources/pyside-tools/deploy.py
index 6cb6d4d9c..b54943ddf 100644
--- a/sources/pyside-tools/deploy.py
+++ b/sources/pyside-tools/deploy.py
@@ -120,6 +120,7 @@ def main(main_file: Path = None, name: str = None, config_file: Path = None, ini
         command_str = nuitka.create_executable(source_file=config.source_file,
                                                extra_args=config.extra_args,
                                                qml_files=config.qml_files,
+                                               qt_plugins=config.qt_plugins,
                                                excluded_qml_plugins=config.excluded_qml_plugins,
                                                icon=config.icon,
                                                dry_run=dry_run)
diff --git a/sources/pyside-tools/deploy_lib/config.py b/sources/pyside-tools/deploy_lib/config.py
index 44b4ded06..f1c877cac 100644
--- a/sources/pyside-tools/deploy_lib/config.py
+++ b/sources/pyside-tools/deploy_lib/config.py
@@ -365,7 +365,7 @@ class DesktopConfig(Config):
                  existing_config_file: bool = False, extra_ignore_dirs: List[str] = None):
         super().__init__(config_file, source_file, python_exe, dry_run, existing_config_file,
                          extra_ignore_dirs)
-
+        self.dependency_reader = QtDependencyReader(dry_run=self.dry_run)
         if self.get_value("qt", "modules"):
             self.modules = self.get_value("qt", "modules").split(",")
         else:
@@ -373,20 +373,34 @@ class DesktopConfig(Config):
             self._find_and_set_qtquick_modules()
             self._find_dependent_qt_modules()
 
+        self._qt_plugins = []
+        if self.get_value("qt", "plugins"):
+            self._qt_plugins = self.get_value("qt", "plugins").split(",")
+        else:
+            self.qt_plugins = self.dependency_reader.find_plugin_dependencies(self.modules)
+
+    @property
+    def qt_plugins(self):
+        return self._qt_plugins
+
+    @qt_plugins.setter
+    def qt_plugins(self, qt_plugins):
+        self._qt_plugins = qt_plugins
+        self.set_value("qt", "plugins", ",".join(qt_plugins))
+
     def _find_dependent_qt_modules(self):
         """
         Given pysidedeploy_config.modules, find all the other dependent Qt modules.
         """
-        dependency_reader = QtDependencyReader(dry_run=self.dry_run)
         all_modules = set(self.modules)
 
-        if not dependency_reader.lib_reader:
-            warnings.warn(f"[DEPLOY] Unable to find {dependency_reader.lib_reader_name}. This tool"
-                          " helps to find the Qt module dependencies of the application. Skipping "
-                          " checking for dependencies.", category=RuntimeWarning)
+        if not self.dependency_reader.lib_reader:
+            warnings.warn(f"[DEPLOY] Unable to find {self.dependency_reader.lib_reader_name}. This "
+                          "tool helps to find the Qt module dependencies of the application. "
+                          "Skipping checking for dependencies.", category=RuntimeWarning)
             return
 
         for module_name in self.modules:
-            dependency_reader.find_dependencies(module=module_name, used_modules=all_modules)
+            self.dependency_reader.find_dependencies(module=module_name, used_modules=all_modules)
 
         self.modules = list(all_modules)
diff --git a/sources/pyside-tools/deploy_lib/default.spec b/sources/pyside-tools/deploy_lib/default.spec
index 2276fa496..8c0697afd 100644
--- a/sources/pyside-tools/deploy_lib/default.spec
+++ b/sources/pyside-tools/deploy_lib/default.spec
@@ -44,6 +44,9 @@ excluded_qml_plugins =
 # Qt modules used. Comma separated
 modules =
 
+# Qt plugins used by the application
+plugins =
+
 [android]
 
 # path to PySide wheel
diff --git a/sources/pyside-tools/deploy_lib/dependency_util.py b/sources/pyside-tools/deploy_lib/dependency_util.py
index c7821794f..53c12ad92 100644
--- a/sources/pyside-tools/deploy_lib/dependency_util.py
+++ b/sources/pyside-tools/deploy_lib/dependency_util.py
@@ -5,6 +5,7 @@ import ast
 import re
 import os
 import site
+import json
 import warnings
 import logging
 import shutil
@@ -15,25 +16,6 @@ from typing import List, Set
 from . import IMPORT_WARNING_PYSIDE, run_command
 
 
-def get_qt_libs_dir():
-    """
-    Finds the path to the Qt libs directory inside PySide6 package installation
-    """
-    pyside_install_dir = None
-    for possible_site_package in site.getsitepackages():
-        if possible_site_package.endswith("site-packages"):
-            pyside_install_dir = Path(possible_site_package) / "PySide6"
-
-    if not pyside_install_dir:
-        print("Unable to find site-packages. Exiting ...")
-        sys.exit(-1)
-
-    if sys.platform == "win32":
-        return pyside_install_dir
-
-    return pyside_install_dir / "Qt" / "lib"  # for linux and macOS
-
-
 def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None,
                         project_data=None):
     """
@@ -164,9 +146,27 @@ class QtDependencyReader:
             print(f"[DEPLOY] Deployment on unsupported platfrom {sys.platform}")
             sys.exit(1)
 
-        self.qt_libs_dir = get_qt_libs_dir()
+        self.pyside_install_dir = None
+        self.qt_libs_dir = self.get_qt_libs_dir()
         self._lib_reader = shutil.which(self.lib_reader_name)
 
+    def get_qt_libs_dir(self):
+        """
+        Finds the path to the Qt libs directory inside PySide6 package installation
+        """
+        for possible_site_package in site.getsitepackages():
+            if possible_site_package.endswith("site-packages"):
+                self.pyside_install_dir = Path(possible_site_package) / "PySide6"
+
+        if not self.pyside_install_dir:
+            print("Unable to find site-packages. Exiting ...")
+            sys.exit(-1)
+
+        if sys.platform == "win32":
+            return self.pyside_install_dir
+
+        return self.pyside_install_dir / "Qt" / "lib"  # for linux and macOS
+
     @property
     def lib_reader(self):
         return self._lib_reader
@@ -216,3 +216,27 @@ class QtDependencyReader:
             logging.info(f"[DEPLOY] Following dependencies found for {module}: {dependent_modules}")
         else:
             logging.info(f"[DEPLOY] No Qt dependencies found for {module}")
+
+    def find_plugin_dependencies(self, used_modules: List[str]) -> List[str]:
+        """
+        Given the modules used by the application, returns all the required plugins
+        """
+        plugins = set()
+        pyside_mod_plugin_jsons = ["PySide6_Essentials.json", "PySide6_Addons.json"]
+        for pyside_mod_plugin_json_name in pyside_mod_plugin_jsons:
+            pyside_mod_plugin_json_file = self.pyside_install_dir / pyside_mod_plugin_json_name
+            if not pyside_mod_plugin_json_file.exists():
+                warnings.warn(f"[DEPLOY] Unable to find {pyside_mod_plugin_json_file}.",
+                              category=RuntimeWarning)
+                continue
+
+            # convert the json to dict
+            pyside_mod_dict = {}
+            with open(pyside_mod_plugin_json_file) as pyside_json:
+                pyside_mod_dict = json.load(pyside_json)
+
+            # find all the plugins in the modules
+            for module in used_modules:
+                plugins.update(pyside_mod_dict.get(module, []))
+
+        return list(plugins)
diff --git a/sources/pyside-tools/deploy_lib/nuitka_helper.py b/sources/pyside-tools/deploy_lib/nuitka_helper.py
index ae5834b6b..721701f70 100644
--- a/sources/pyside-tools/deploy_lib/nuitka_helper.py
+++ b/sources/pyside-tools/deploy_lib/nuitka_helper.py
@@ -17,6 +17,23 @@ class Nuitka:
 
     def __init__(self, nuitka):
         self.nuitka = nuitka
+        # plugins to ignore. The sensible plugins are include by default by Nuitka for PySide6
+        # application deployment
+        self.qt_plugins_to_ignore = ["imageformats",  # being Nuitka `sensible`` plugins
+                                     "iconengines",
+                                     "mediaservice",
+                                     "printsupport",
+                                     "platforms",
+                                     "platformthemes",
+                                     "styles",
+                                     "wayland-shell-integration",
+                                     "wayland-decoration-client",
+                                     "wayland-graphics-integration-client",
+                                     "egldeviceintegrations",
+                                     "xcbglintegrations",
+                                     "tls",  # end Nuitka `sensible` plugins
+                                     "generic"  # plugins that error with Nuitka
+                                     ]
 
     @staticmethod
     def icon_option():
@@ -28,11 +45,12 @@ class Nuitka:
             return "--macos-app-icon"
 
     def create_executable(self, source_file: Path, extra_args: str, qml_files: List[Path],
-                          excluded_qml_plugins: List[str], icon: str, dry_run: bool):
+                          qt_plugins: List[str], excluded_qml_plugins: List[str], icon: str,
+                          dry_run: bool):
+        qt_plugins = [plugin for plugin in qt_plugins if plugin not in self.qt_plugins_to_ignore]
         extra_args = extra_args.split()
         qml_args = []
         if qml_files:
-            qml_args.append("--include-qt-plugins=all")
             # This will generate options for each file using:
             #     --include-data-files=ABSOLUTE_PATH_TO_FILE=RELATIVE_PATH_TO ROOT
             # for each file. This will preserve the directory structure of QML resources.
@@ -41,6 +59,11 @@ class Nuitka:
                  f"./{qml_file.resolve().relative_to(source_file.parent)}"
                  for qml_file in qml_files]
             )
+            # add qml plugin. The `qml`` plugin name is not present in the module json files shipped
+            # with Qt and hence not in `qt_plugins``. However, Nuitka uses the 'qml' plugin name to
+            # include the necessary qml plugins. There we have to add it explicitly for a qml
+            # application
+            qt_plugins.append("qml")
 
             if excluded_qml_plugins:
                 prefix = "lib" if sys.platform != "win32" else ""
@@ -59,8 +82,14 @@ class Nuitka:
             "--enable-plugin=pyside6",
             f"--output-dir={output_dir}",
         ]
+
         command.extend(extra_args + qml_args)
         command.append(f"{self.__class__.icon_option()}={icon}")
+        if qt_plugins:
+            # sort qt_plugins so that the result is definitive when testing
+            qt_plugins.sort()
+            qt_plugins_str = ",".join(qt_plugins)
+            command.append(f"--include-qt-plugins={qt_plugins_str}")
 
         command_str, _ = run_command(command=command, dry_run=dry_run)
         return command_str