Android Deployment: copy required plugins to libs

- Copy the required Qt plugins from `site_packages` of the python
  bundled with the application to the `libs` folder of the Android
  gradle project. Android looks for required libraries in this `libs`
  folder. A similar step is also done by `androiddeployqt` when it
  created an Android gradle project from a C++ application.

- Dependent Qt libraries found during processing of
  pyside6-android-deploy are also copied into the `libs` folder, if it
  does not exist already.

- `plugins` key added to `pysidedeploy.spec`, which represents the
  plugins to be copied.

- The Android dependency files shipped with Qt for Android platforms,
  are prased to obtain all the dependent Qt plugins of an application.

- Some code refactoring to facilitate the plugin and library copy,
  by passing the plugin and library names to the PySide6 recipe
  template. `jinja2` does the job of using this template to create
  the PySide6 recipe to be used by python-for-android.

- As an addition, fix some minor code issues and add extra logging.

Task-number: PYSIDE-1612
Pick-to: 6.6
Change-Id: I63ca1e48aa1e4c98c912a87e68f3ae912ce89ca4
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

7 files changed, 156 insertions, 51 deletions

diff --git a/sources/pyside-tools/android_deploy.py b/sources/pyside-tools/android_deploy.py
index fbc613069..3c1dd5925 100644
--- a/sources/pyside-tools/android_deploy.py
+++ b/sources/pyside-tools/android_deploy.py
@@ -8,11 +8,9 @@ import traceback
 from pathlib import Path
 from textwrap import dedent
 
-from pkginfo import Wheel
-
 from deploy_lib import (setup_python, get_config, cleanup, install_python_dependencies,
                         config_option_exists, find_pyside_modules, MAJOR_VERSION)
-from deploy_lib.android import (create_recipe, extract_and_copy_jar, get_wheel_android_arch,
+from deploy_lib.android import (extract_and_copy_jar, get_wheel_android_arch,
                                 Buildozer, AndroidData)
 
 
@@ -126,24 +124,8 @@ def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = Non
                          f"the project {config.modules}")
         config.modules.extend(extra_modules)
 
-        # create recipes
-        # https://python-for-android.readthedocs.io/en/latest/recipes/
-        # These recipes are manually added through buildozer.spec file to be used by
-        # python_for_android while building the distribution
-        if not config.recipes_exist():
-            logging.info("[DEPLOY] Creating p4a recipes for PySide6 and shiboken6")
-            version = Wheel(config.wheel_pyside).version
-            create_recipe(version=version, component=f"PySide{MAJOR_VERSION}",
-                          wheel_path=config.wheel_pyside,
-                          generated_files_path=generated_files_path,
-                          qt_modules=config.modules)
-            create_recipe(version=version, component=f"shiboken{MAJOR_VERSION}",
-                          wheel_path=config.wheel_shiboken,
-                          generated_files_path=generated_files_path)
-            config.recipe_dir = (generated_files_path / "recipes").resolve()
-
         # extract out and copy .jar files to {generated_files_path}
-        if not config.jars_dir or not Path(config.jars_dir).exists():
+        if not config.jars_dir or not Path(config.jars_dir).exists() and not dry_run:
             logging.info("[DEPLOY] Extract and copy jar files from PySide6 wheel to "
                          f"{generated_files_path}")
             extract_and_copy_jar(wheel_path=config.wheel_pyside,
@@ -154,9 +136,8 @@ def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = Non
         if not config.arch:
             arch = get_wheel_android_arch(wheel=config.wheel_pyside)
             if not arch:
-                logging.exception("[DEPLOY] PySide wheel corrupted. Wheel name should end with"
+                raise RuntimeError("[DEPLOY] PySide wheel corrupted. Wheel name should end with"
                                   "platform name")
-                raise
             config.arch = arch
 
         # writing config file
@@ -174,7 +155,7 @@ def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = Non
         # init buildozer
         Buildozer.dry_run = dry_run
         logging.info("[DEPLOY] Creating buildozer.spec file")
-        Buildozer.initialize(pysidedeploy_config=config)
+        Buildozer.initialize(pysidedeploy_config=config, generated_files_path=generated_files_path)
 
         # run buildozer
         logging.info("[DEPLOY] Running buildozer deployment")
diff --git a/sources/pyside-tools/deploy_lib/android/android_helper.py b/sources/pyside-tools/deploy_lib/android/android_helper.py
index 3e74a7e79..c10bdd994 100644
--- a/sources/pyside-tools/deploy_lib/android/android_helper.py
+++ b/sources/pyside-tools/deploy_lib/android/android_helper.py
@@ -25,17 +25,31 @@ class AndroidData:
 
 
 def create_recipe(version: str, component: str, wheel_path: str, generated_files_path: Path,
-                  qt_modules: List[str] = None):
+                  qt_modules: List[str] = None, local_libs: List[str] = None,
+                  plugins: List[str] = None):
     '''
     Create python_for_android recipe for PySide6 and shiboken6
     '''
+    qt_plugins  = []
+    if plugins:
+        #split plugins based on category
+        for plugin in plugins:
+            plugin_category, plugin_name = plugin.split('_', 1)
+            qt_plugins.append((plugin_category, plugin_name))
+
+    qt_local_libs = []
+    if local_libs:
+        qt_local_libs = [local_lib for local_lib in local_libs if local_lib.startswith("Qt6") ]
+
     rcp_tmpl_path = Path(__file__).parent / "recipes" / f"{component}"
     environment = Environment(loader=FileSystemLoader(rcp_tmpl_path))
     template = environment.get_template("__init__.tmpl.py")
     content = template.render(
         version=version,
         wheel_path=wheel_path,
-        qt_modules=qt_modules
+        qt_modules=qt_modules,
+        qt_local_libs=qt_local_libs,
+        qt_plugins=qt_plugins
     )
 
     recipe_path = generated_files_path / "recipes" / f"{component}"
diff --git a/sources/pyside-tools/deploy_lib/android/buildozer.py b/sources/pyside-tools/deploy_lib/android/buildozer.py
index e6a954c62..647951a09 100644
--- a/sources/pyside-tools/deploy_lib/android/buildozer.py
+++ b/sources/pyside-tools/deploy_lib/android/buildozer.py
@@ -5,17 +5,19 @@ import re
 import logging
 import tempfile
 import xml.etree.ElementTree as ET
+from typing import List
+from pkginfo import Wheel
 
 import zipfile
 from pathlib import Path
-from typing import List
 
 from .. import run_command, BaseConfig, Config, MAJOR_VERSION
-from . import get_llvm_readobj, find_lib_dependencies, find_qtlibs_in_wheel
+from . import get_llvm_readobj, find_lib_dependencies, find_qtlibs_in_wheel, create_recipe
 
 
 class BuildozerConfig(BaseConfig):
-    def __init__(self, buildozer_spec_file: Path, pysidedeploy_config: Config):
+    def __init__(self, buildozer_spec_file: Path, pysidedeploy_config: Config,
+                 generated_files_path: Path):
         super().__init__(buildozer_spec_file, comment_prefixes="#")
         self.set_value("app", "title", pysidedeploy_config.title)
         self.set_value("app", "package.name", pysidedeploy_config.title)
@@ -47,7 +49,6 @@ class BuildozerConfig(BaseConfig):
                      "https://github.com/shyamnathp/python-for-android/tree/pyside_support")
         self.set_value("app", "p4a.fork", "shyamnathp")
         self.set_value("app", "p4a.branch", "pyside_support_2")
-        self.set_value('app', "p4a.local_recipes", str(pysidedeploy_config.recipe_dir))
         self.set_value("app", "p4a.bootstrap", "qt")
 
         self.qt_libs_path: zipfile.Path = (
@@ -68,14 +69,38 @@ class BuildozerConfig(BaseConfig):
         dependency_files = self.__get_dependency_files(modules=pysidedeploy_config.modules,
                                                        arch=self.arch)
 
-        local_libs = self.__find_local_libs(dependency_files)
+        dependent_plugins = []
+        # the local_libs can also store dependent plugins
+        local_libs, dependent_plugins = self.__find_local_libs(dependency_files)
         pysidedeploy_config.local_libs += local_libs
 
-        if local_libs:
+        self.__find_plugin_dependencies(dependency_files, dependent_plugins)
+        pysidedeploy_config.qt_plugins += dependent_plugins
+
+        if local_libs or dependent_plugins:
             pysidedeploy_config.update_config()
 
         local_libs = ",".join(pysidedeploy_config.local_libs)
 
+        # create recipes
+        # https://python-for-android.readthedocs.io/en/latest/recipes/
+        # These recipes are manually added through buildozer.spec file to be used by
+        # python_for_android while building the distribution
+        if not pysidedeploy_config.recipes_exist() and not pysidedeploy_config.dry_run:
+            logging.info("[DEPLOY] Creating p4a recipes for PySide6 and shiboken6")
+            version = Wheel(pysidedeploy_config.wheel_pyside).version
+            create_recipe(version=version, component=f"PySide{MAJOR_VERSION}",
+                          wheel_path=pysidedeploy_config.wheel_pyside,
+                          generated_files_path=generated_files_path,
+                          qt_modules=pysidedeploy_config.modules,
+                          local_libs=pysidedeploy_config.local_libs,
+                          plugins=pysidedeploy_config.qt_plugins)
+            create_recipe(version=version, component=f"shiboken{MAJOR_VERSION}",
+                          wheel_path=pysidedeploy_config.wheel_shiboken,
+                          generated_files_path=generated_files_path)
+            pysidedeploy_config.recipe_dir = (generated_files_path / "recipes").resolve()
+        self.set_value('app', "p4a.local_recipes", str(pysidedeploy_config.recipe_dir))
+
         # add permissions
         permissions = self.__find_permissions(dependency_files)
         permissions = ", ".join(permissions)
@@ -165,6 +190,7 @@ class BuildozerConfig(BaseConfig):
 
     def __find_local_libs(self, dependency_files: List[zipfile.Path]):
         local_libs = set()
+        plugins = set()
         lib_pattern = re.compile(f"lib(?P<lib_name>.*)_{self.arch}")
         for dependency_file in dependency_files:
             xml_content = dependency_file.read_text()
@@ -172,6 +198,9 @@ class BuildozerConfig(BaseConfig):
             for local_lib in root.iter("lib"):
 
                 if 'file' not in local_lib.attrib:
+                    if 'name' not in local_lib.attrib:
+                        logging.warning("[DEPLOY] Invalid android dependency file"
+                                        f" {str(dependency_file)}")
                     continue
 
                 file = local_lib.attrib['file']
@@ -191,8 +220,59 @@ class BuildozerConfig(BaseConfig):
                     if match:
                         lib_name = match.group("lib_name")
                         local_libs.add(lib_name)
+                        if lib_name.startswith("plugins"):
+                            plugin_name = lib_name.split('plugins_', 1)[1]
+                            plugins.add(plugin_name)
+
+        return list(local_libs), list(plugins)
+
+    def __find_plugin_dependencies(self, dependency_files: List[zipfile.Path],
+                                   dependent_plugins: List[str]):
+        # The `bundled` element in the dependency xml files points to the folder where
+        # additional dependencies for the application exists. Inspecting the depenency files
+        # in android, this always points to the specific Qt plugin dependency folder.
+        # eg: for application using Qt Multimedia, this looks like:
+        # <bundled file="./plugins/multimedia" />
+        # The code recusively checks all these dependent folders and adds the necessary plugins
+        # as dependencies
+        lib_pattern = re.compile(f"libplugins_(?P<plugin_name>.*)_{self.arch}.so")
+        for dependency_file in dependency_files:
+            xml_content = dependency_file.read_text()
+            root = ET.fromstring(xml_content)
+            for bundled_element in root.iter("bundled"):
+                # the attribute 'file' can be misleading, but it always points to the plugin
+                # folder on inspecting the dependency files
+                if 'file' not in bundled_element.attrib:
+                    logging.warning("[DEPLOY] Invalid Android dependency file"
+                                    f" {str(dependency_file)}")
+                    continue
+
+                # from "./plugins/multimedia" to absolute path in wheel
+                plugin_module_folder = bundled_element.attrib['file']
+                # they all should start with `./plugins`
+                if plugin_module_folder.startswith("./plugins"):
+                    plugin_module_folder = plugin_module_folder.partition("./plugins/")[2]
+                else:
+                    continue
 
-        return list(local_libs)
+                absolute_plugin_module_folder = (self.qt_libs_path.parent / "plugins" /
+                                                plugin_module_folder)
+
+                if not absolute_plugin_module_folder.is_dir():
+                    logging.warning(f"[DEPLOY] Qt plugin folder '{plugin_module_folder}' does not"
+                                    " exist or is not a directory for this Android platform")
+                    continue
+
+                for plugin in absolute_plugin_module_folder.iterdir():
+                    plugin_name = plugin.name
+                    if plugin_name.endswith(".so") and plugin_name.startswith("libplugins"):
+                        # we only need part of plugin_name, because `lib` prefix and `arch` suffix
+                        # gets re-added by python-for-android
+                        match = lib_pattern.search(plugin_name)
+                        if match:
+                            plugin_infix_name = match.group("plugin_name")
+                            if plugin_infix_name not in dependent_plugins:
+                                dependent_plugins.append(plugin_infix_name)
 
     def __find_dependent_qt_modules(self, pysidedeploy_config: Config):
         """
@@ -232,6 +312,10 @@ class BuildozerConfig(BaseConfig):
                 if module not in pysidedeploy_config.modules:
                     dependent_modules.add(module)
 
+        dependent_modules_str = ",".join(dependent_modules)
+        logging.info("[DEPLOY] The following extra dependencies were found:"
+                     f" {dependent_modules_str}")
+
         return list(dependent_modules)
 
 
@@ -239,7 +323,7 @@ class Buildozer:
     dry_run = False
 
     @staticmethod
-    def initialize(pysidedeploy_config: Config):
+    def initialize(pysidedeploy_config: Config, generated_files_path: Path):
         project_dir = Path(pysidedeploy_config.project_dir)
         buildozer_spec = project_dir / "buildozer.spec"
         if buildozer_spec.exists():
@@ -253,7 +337,7 @@ class Buildozer:
         if not Buildozer.dry_run:
             if not buildozer_spec.exists():
                 raise RuntimeError(f"buildozer.spec not found in {Path.cwd()}")
-            BuildozerConfig(buildozer_spec, pysidedeploy_config)
+            BuildozerConfig(buildozer_spec, pysidedeploy_config, generated_files_path)
 
     @staticmethod
     def create_executable(mode: str):
diff --git a/sources/pyside-tools/deploy_lib/android/recipes/PySide6/__init__.tmpl.py b/sources/pyside-tools/deploy_lib/android/recipes/PySide6/__init__.tmpl.py
index 76d79710e..d79f03676 100644
--- a/sources/pyside-tools/deploy_lib/android/recipes/PySide6/__init__.tmpl.py
+++ b/sources/pyside-tools/deploy_lib/android/recipes/PySide6/__init__.tmpl.py
@@ -18,37 +18,46 @@ class PySideRecipe(PythonRecipe):
     def build_arch(self, arch):
         """Unzip the wheel and copy into site-packages of target"""
 
-        info("Installing {} into site-packages".format(self.name))
+        info("Copying libc++_shared.so from SDK to be loaded on startup")
+        libcpp_path = f"{self.ctx.ndk.sysroot_lib_dir}/{arch.command_prefix}/libc++_shared.so"
+        shutil.copyfile(libcpp_path, Path(self.ctx.get_libs_dir(arch.arch)) / "libc++_shared.so")
+
+        info(f"Installing {self.name} into site-packages")
         with zipfile.ZipFile(self.wheel_path, "r") as zip_ref:
             info("Unzip wheels and copy into {}".format(self.ctx.get_python_install_dir(arch.arch)))
             zip_ref.extractall(self.ctx.get_python_install_dir(arch.arch))
 
         lib_dir = Path(f"{self.ctx.get_python_install_dir(arch.arch)}/PySide6/Qt/lib")
+
         info("Copying Qt libraries to be loaded on startup")
         shutil.copytree(lib_dir, self.ctx.get_libs_dir(arch.arch), dirs_exist_ok=True)
         shutil.copyfile(lib_dir.parent.parent / "libpyside6.abi3.so",
                         Path(self.ctx.get_libs_dir(arch.arch)) / "libpyside6.abi3.so")
 
-        {%- for module in qt_modules %}
+        {% for module in qt_modules %}
         shutil.copyfile(lib_dir.parent.parent / f"Qt{{ module }}.abi3.so",
-                        Path(self.ctx.get_libs_dir(arch.arch)) / f"Qt{{ module }}.abi3.so")
+                        Path(self.ctx.get_libs_dir(arch.arch)) / "Qt{{ module }}.abi3.so")
         {% if module == "Qml" -%}
         shutil.copyfile(lib_dir.parent.parent / "libpyside6qml.abi3.so",
                         Path(self.ctx.get_libs_dir(arch.arch)) / "libpyside6qml.abi3.so")
         {% endif %}
-        {%- endfor -%}
-
-        info("Copying libc++_shared.so from SDK to be loaded on startup")
-        libcpp_path = f"{self.ctx.ndk.sysroot_lib_dir}/{arch.command_prefix}/libc++_shared.so"
-        shutil.copyfile(libcpp_path, Path(self.ctx.get_libs_dir(arch.arch)) / "libc++_shared.so")
-
-        info("Copying Qt platform plugin to be loaded on startup from SDK to be loaded on startup")
-        shutil.copyfile(
-            Path(self.ctx.get_python_install_dir(arch.arch))
-            / "PySide6" / "Qt" / "plugins" / "platforms"
-            / f"libplugins_platforms_qtforandroid_{arch.arch}.so",
-            Path(self.ctx.get_libs_dir(arch.arch)) / f"libplugins_platforms_qtforandroid_{arch.arch}.so",
-        )
+        {% endfor %}
+
+        {% for lib in qt_local_libs %}
+        lib_path = lib_dir / f"lib{{ lib }}_{arch.arch}.so"
+        if lib_path.exists():
+            shutil.copyfile(lib_path,
+                            Path(self.ctx.get_libs_dir(arch.arch)) / f"lib{{ lib }}_{arch.arch}.so")
+        {% endfor %}
+
+        {% for plugin_category,plugin_name in qt_plugins %}
+        plugin_path = (lib_dir.parent / "plugins" / "{{ plugin_category }}" /
+                      f"libplugins_{{ plugin_category }}_{{ plugin_name }}_{arch.arch}.so")
+        if plugin_path.exists():
+            shutil.copyfile(plugin_path,
+                            (Path(self.ctx.get_libs_dir(arch.arch)) /
+                             f"libplugins_{{ plugin_category }}_{{ plugin_name }}_{arch.arch}.so"))
+        {% endfor %}
 
 
 recipe = PySideRecipe()
diff --git a/sources/pyside-tools/deploy_lib/config.py b/sources/pyside-tools/deploy_lib/config.py
index 8af341bad..5fcc54795 100644
--- a/sources/pyside-tools/deploy_lib/config.py
+++ b/sources/pyside-tools/deploy_lib/config.py
@@ -172,6 +172,10 @@ class Config(BaseConfig):
             if self.get_value("buildozer", "local_libs"):
                 self.local_libs = self.get_value("buildozer", "local_libs").split(",")
 
+            self._qt_plugins = []
+            if self.get_value("qt", "plugins"):
+                self._qt_plugins = self.get_value("qt", "plugins").split(",")
+
             self._mode = self.get_value("buildozer", "mode")
 
     def set_or_fetch(self, config_property_val, config_property_key, config_property_group="app"):
@@ -292,6 +296,15 @@ class Config(BaseConfig):
         self.set_value("buildozer", "local_libs", ",".join(local_libs))
 
     @property
+    def qt_plugins(self):
+        return self._qt_plugins
+
+    @qt_plugins.setter
+    def qt_plugins(self, qt_plugins):
+        self._qt_plugins = qt_plugins
+        self.set_value("qt", "plugins", ",".join(qt_plugins))
+
+    @property
     def ndk_path(self):
         return self._ndk_path
 
diff --git a/sources/pyside-tools/deploy_lib/default.spec b/sources/pyside-tools/deploy_lib/default.spec
index fdfb5397d..c49ef53bf 100644
--- a/sources/pyside-tools/deploy_lib/default.spec
+++ b/sources/pyside-tools/deploy_lib/default.spec
@@ -44,6 +44,9 @@ wheel_pyside =
 # path to Shiboken wheel
 wheel_shiboken =
 
+# plugins to be copied to libs folder of the packaged application. Comma separated
+plugins = platforms_qtforandroid
+
 [nuitka]
 
 # (str) specify any extra nuitka arguments
@@ -73,6 +76,7 @@ sdk_path =
 modules =
 
 # other libraries to be loaded. Comma separated.
+# loaded at app startup
 local_libs = plugins_platforms_qtforandroid
 
 # architecture of deployed platform
diff --git a/sources/pyside-tools/deploy_lib/python_helper.py b/sources/pyside-tools/deploy_lib/python_helper.py
index 0174a8fcf..af8753257 100644
--- a/sources/pyside-tools/deploy_lib/python_helper.py
+++ b/sources/pyside-tools/deploy_lib/python_helper.py
@@ -63,7 +63,7 @@ def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None,
                             logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file)))
 
         except Exception as e:
-            logging.error(f"Finding module import failed on file {str(py_file)}")
+            logging.error(f"[DEPLOY] Finding module import failed on file {str(py_file)}")
             raise e
 
         return set(modules)