Android Deployment: find PySide and Qt dependencies

- Use llvm-readelf to recursively find the dependencies of a dependent
  Qt binary.
  All the Qt dependencies are loaded at startup when loading the Android
  application.
- Parse the revelant Python files of the project into ast, and
  find the used Python modules. Once the Python file is parsed
  into an ast, we find the imports of the following form:
      from PySide6 import Qt<module>
      from PySide6.Qt<module> import <classname>
  This is then used to identify the module used, and we try to
  load the binaries of this module. If the modules does not exist
  in Qt for Android, then an error is thrown.
- The easiest way to find the relevant Python files in the project is
  using a .pyproject file which lists all the relevant files. If this
  is not there, then we find all the Python files in the project
  folder excluding the following folders:
  [".hg", ".svn", ".git", ".tox", "__pycache__", "env", "venv",
    "deployment",".buildozer"]
- A new cli argument --extra-ignore-dirs, that lists the extra
  directories to ignore when searching for all the relevant python
  files in the project.
- A new cli argument --extra-modules, that lists the extra modules
  to be added manually to the application incase they are not found
  by `pyside6-android-deploy` automatically. Adding a module using
  this argument means that the module binary is loaded by the Android
  application on startup.
- sdk and ndk cli options are now mandatory to find the dependencies.
  These two options will be removed later when pyside6-android-deploy
  can automatically download them.

Task-number: PYSIDE-1612
Change-Id: Ifbdc20cbc70ab0935a23157ccc8cb7fde6992df2
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

6 files changed, 275 insertions, 47 deletions

diff --git a/sources/pyside-tools/android_deploy.py b/sources/pyside-tools/android_deploy.py
index b1ea32064..97ac523b4 100644
--- a/sources/pyside-tools/android_deploy.py
+++ b/sources/pyside-tools/android_deploy.py
@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 import argparse
-import sys
 import logging
 import shutil
 import traceback
@@ -12,10 +11,9 @@ from textwrap import dedent
 from pkginfo import Wheel
 
 from deploy_lib import (setup_python, get_config, cleanup, install_python_dependencies,
-                        config_option_exists, MAJOR_VERSION)
+                        config_option_exists, find_pyside_modules, MAJOR_VERSION)
 from deploy_lib.android import (create_recipe, extract_and_copy_jar, get_wheel_android_arch,
-                                Buildozer, AndroidData, WIDGET_APPLICATION_MODULES,
-                                QUICK_APPLICATION_MODULES)
+                                Buildozer, AndroidData)
 
 
 """ pyside6-android-deploy deployment tool
@@ -39,8 +37,6 @@ from deploy_lib.android import (create_recipe, extract_and_copy_jar, get_wheel_a
 
     Platforms Supported: aarch64, armv7a, i686, x86_64
 
-    Supported Modules: Core, Gui, Widgets, Network, OpenGL, Qml, Quick, QuickControls2
-
     Config file:
         On the first run of the tool, it creates a config file called pysidedeploy.spec which
         controls the various characteristic of the deployment. Users can simply change the value
@@ -50,13 +46,46 @@ from deploy_lib.android import (create_recipe, extract_and_copy_jar, get_wheel_a
         Note: This file is used by both pyside6-deploy and pyside6-android-deploy
 """
 
+HELP_EXTRA_IGNORE_DIRS = dedent("""
+                                Comma separated directory names inside the project dir. These
+                                directories will be skipped when searching for python files
+                                relevant to the project.
+
+                                Example usage: --extra-ignore-dirs=doc,translations
+                                """)
+
+HELP_EXTRA_MODULES = dedent("""
+                            Comma separated list of Qt modules to be added to the application,
+                            in case they are not found automatically.
+
+                            This occurs when you have 'import PySide6' in your code instead
+                            'from PySide6 import <module>'. The module name is specified
+                            with either omitting the prefix of Qt or with it.
+
+                            Example usage 1: --extra-modules=Network,Svg
+                            Example usage 2: --extra-modules=QtNetwork,QtSvg
+                            """)
+
 
 def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = None, ndk_path: Path = None,
          sdk_path: Path = None, config_file: Path = None, init: bool = False,
          loglevel=logging.WARNING, dry_run: bool = False, keep_deployment_files: bool = False,
-         force: bool = False):
+         force: bool = False, extra_ignore_dirs: str = None, extra_modules_grouped: str = None):
 
     logging.basicConfig(level=loglevel)
+
+    if extra_ignore_dirs:
+        extra_ignore_dirs = extra_ignore_dirs.split(",")
+
+    extra_modules = []
+    if extra_modules_grouped:
+        tmp_extra_modules = extra_modules_grouped.split(",")
+        for extra_module in tmp_extra_modules:
+            if extra_module.startswith("Qt"):
+                extra_modules.append(extra_module[2:])
+            else:
+                extra_modules.append(extra_module)
+
     main_file = Path.cwd() / "main.py"
     generated_files_path = None
     if not main_file.exists():
@@ -65,12 +94,6 @@ def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = Non
         'main.py' and it should be run from the application directory
         """))
 
-    # check if ndk and sdk path given, else use default
-    if ndk_path and sdk_path:
-        logging.warning("[DEPLOY] May not work with custom Ndk and Sdk versions."
-                        "Use the default by leaving out --ndk-path and --sdk-path cl"
-                        "arguments")
-
     android_data = AndroidData(wheel_pyside=pyside_wheel, wheel_shiboken=shiboken_wheel,
                                ndk_path=ndk_path, sdk_path=sdk_path)
 
@@ -121,8 +144,13 @@ def main(name: str = None, pyside_wheel: Path = None, shiboken_wheel: Path = Non
         # TODO: Optimize this based on the modules needed
         # check if other modules not supported by Android used and raise error
         if not config.modules:
-            config.modules = (QUICK_APPLICATION_MODULES if config.qml_files else
-                              WIDGET_APPLICATION_MODULES)
+            config.modules = find_pyside_modules(project_dir=config.project_dir,
+                                                 extra_ignore_dirs=extra_ignore_dirs,
+                                                 project_data=config.project_data)
+            logging.info("The following PySide modules were found from the python files of "
+                         f"the project {config.modules}")
+
+        config.modules.extend(extra_modules)
 
         # find architecture from wheel name
         if not config.arch:
@@ -211,16 +239,21 @@ if __name__ == "__main__":
                         help=f"Path to shiboken{MAJOR_VERSION} Android Wheel",
                         required=not config_option_exists())
 
+    #TODO: --ndk-path and --sdk-path will be removed when automatic download of sdk and ndk is added
     parser.add_argument("--ndk-path", type=lambda p: Path(p).resolve(),
                         help=("Path to Android Ndk. If omitted, the default from buildozer is used")
-                        , required="--sdk-path" in sys.argv)
+                        , required=True)
 
     parser.add_argument("--sdk-path", type=lambda p: Path(p).resolve(),
                         help=("Path to Android Sdk. If omitted, the default from buildozer is used")
-                        , required="--ndk-path" in sys.argv)
+                        , required=True)
+
+    parser.add_argument("--extra-ignore-dirs", type=str, help=HELP_EXTRA_IGNORE_DIRS)
+
+    parser.add_argument("--extra-modules", type=str, help=HELP_EXTRA_MODULES)
 
     args = parser.parse_args()
 
     main(args.name, args.wheel_pyside, args.wheel_shiboken, args.ndk_path, args.sdk_path,
          args.config_file, args.init, args.loglevel, args.dry_run, args.keep_deployment_files,
-         args.force)
+         args.force, args.extra_ignore_dirs, args.extra_modules)
diff --git a/sources/pyside-tools/deploy_lib/__init__.py b/sources/pyside-tools/deploy_lib/__init__.py
index 1aa7ef9cc..003bac5f4 100644
--- a/sources/pyside-tools/deploy_lib/__init__.py
+++ b/sources/pyside-tools/deploy_lib/__init__.py
@@ -8,6 +8,6 @@ EXE_FORMAT = ".exe" if sys.platform == "win32" else ".bin"
 from .commands import run_command
 from .nuitka_helper import Nuitka
 from .config import BaseConfig, Config
-from .python_helper import PythonExecutable
+from .python_helper import PythonExecutable, find_pyside_modules
 from .deploy_util import (cleanup, finalize, get_config, setup_python, install_python_dependencies,
                           config_option_exists)
diff --git a/sources/pyside-tools/deploy_lib/android/__init__.py b/sources/pyside-tools/deploy_lib/android/__init__.py
index 12eb6830c..e53f41848 100644
--- a/sources/pyside-tools/deploy_lib/android/__init__.py
+++ b/sources/pyside-tools/deploy_lib/android/__init__.py
@@ -1,10 +1,7 @@
 # Copyright (C) 2023 The Qt Company Ltd.
 # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
-WIDGET_APPLICATION_MODULES = ["Core", "Gui", "Widgets"]
-QUICK_APPLICATION_MODULES = ["Core", "Gui", "Widgets", "Network", "OpenGL", "Qml", "Quick",
-                             "QuickControls2"]
-
 from .android_helper import (create_recipe, extract_and_copy_jar, get_wheel_android_arch,
-                             AndroidData)
+                             AndroidData, get_llvm_readobj, find_lib_dependencies,
+                             find_qtlibs_in_wheel)
 from .buildozer import Buildozer
diff --git a/sources/pyside-tools/deploy_lib/android/android_helper.py b/sources/pyside-tools/deploy_lib/android/android_helper.py
index 321fc3734..514ac8b5d 100644
--- a/sources/pyside-tools/deploy_lib/android/android_helper.py
+++ b/sources/pyside-tools/deploy_lib/android/android_helper.py
@@ -2,10 +2,15 @@
 # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 import logging
-from pathlib import Path
-from jinja2 import Environment, FileSystemLoader
+import zipfile
 from zipfile import ZipFile
 from dataclasses import dataclass
+from typing import Set
+
+from pathlib import Path
+from jinja2 import Environment, FileSystemLoader
+
+from .. import run_command
 
 
 @dataclass
@@ -62,3 +67,67 @@ def get_wheel_android_arch(wheel: Path):
             return arch
 
     return None
+
+
+def get_llvm_readobj(ndk_path: Path) -> Path:
+    '''
+    Return the path to llvm_readobj from the Android Ndk
+    '''
+    if not ndk_path:
+        # fetch ndk path from buildozer
+        raise FileNotFoundError("[DEPLOY] Unable to find Ndk path. Please pass the Ndk path either"
+                                " from the CLI or from pysidedeploy.spec")
+
+    # TODO: Requires change if Windows platform supports Android Deployment or if we
+    # support host other than linux-x86_64
+    return (ndk_path / "toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-readobj")
+
+
+def find_lib_dependencies(llvm_readobj: Path, lib_path: Path, used_dependencies: Set[str] = None,
+                          dry_run: bool = False):
+    """
+    Find all the Qt dependencies of a library using llvm_readobj
+    """
+    if lib_path.name in used_dependencies:
+        return
+
+    command = [str(llvm_readobj), "--needed-libs", str(lib_path)]
+    _, output = run_command(command=command, dry_run=dry_run, fetch_output=True)
+
+    dependencies = set()
+    neededlibraries_found = False
+    for line in output.splitlines():
+        line = line.decode("utf-8").lstrip()
+        if line.startswith("NeededLibraries") and not neededlibraries_found:
+            neededlibraries_found = True
+        if neededlibraries_found and line.startswith("libQt"):
+            dependencies.add(line)
+            used_dependencies.add(line)
+            dependent_lib_path = lib_path.parent / line
+            find_lib_dependencies(llvm_readobj, dependent_lib_path, used_dependencies, dry_run)
+
+    if dependencies:
+        logging.info(f"[DEPLOY] Following dependencies found for {lib_path.stem}: {dependencies}")
+    else:
+        logging.info(f"[DEPLOY] No Qt dependencies found for {lib_path.stem}")
+
+
+def find_qtlibs_in_wheel(wheel_pyside: Path):
+    """
+    Find the path to Qt/lib folder inside the wheel.
+    """
+    archive = ZipFile(wheel_pyside)
+    qt_libs_path = wheel_pyside / "PySide6/Qt/lib"
+    qt_libs_path = zipfile.Path(archive, at=qt_libs_path)
+    if not qt_libs_path.exists():
+        for file in archive.namelist():
+            # the dependency files are inside the libs folder
+            if file.endswith("android-dependencies.xml"):
+                qt_libs_path = zipfile.Path(archive, at=file).parent
+                # all dependency files are in the same path
+                break
+
+    if not qt_libs_path:
+        raise FileNotFoundError("[DEPLOY] Unable to find Qt libs folder inside the wheel")
+
+    return qt_libs_path
diff --git a/sources/pyside-tools/deploy_lib/android/buildozer.py b/sources/pyside-tools/deploy_lib/android/buildozer.py
index 45c62075c..2eeb2d261 100644
--- a/sources/pyside-tools/deploy_lib/android/buildozer.py
+++ b/sources/pyside-tools/deploy_lib/android/buildozer.py
@@ -3,12 +3,15 @@
 
 import re
 import logging
+import tempfile
 import xml.etree.ElementTree as ET
+
 import zipfile
 from pathlib import Path
 from typing import List
 
-from .. import MAJOR_VERSION, BaseConfig, Config, run_command
+from .. import run_command, BaseConfig, Config, MAJOR_VERSION
+from . import get_llvm_readobj, find_lib_dependencies, find_qtlibs_in_wheel
 
 
 class BuildozerConfig(BaseConfig):
@@ -47,10 +50,24 @@ class BuildozerConfig(BaseConfig):
         self.set_value('app', "p4a.local_recipes", str(pysidedeploy_config.recipe_dir))
         self.set_value("app", "p4a.bootstrap", "qt")
 
-        # gets the xml dependency files from Qt installation path
-        dependency_files = self.__get_dependency_files(pysidedeploy_config)
+        self.qt_libs_path: zipfile.Path = (
+            find_qtlibs_in_wheel(wheel_pyside=pysidedeploy_config.wheel_pyside))
+        logging.info(f"[DEPLOY] Found Qt libs path inside wheel: {str(self.qt_libs_path)}")
+
+        extra_modules = self.__find_dependent_qt_modules(pysidedeploy_config)
+        logging.info(f"[DEPLOY] Other dependent modules to be added: {extra_modules}")
+        pysidedeploy_config.modules = pysidedeploy_config.modules + extra_modules
+
+        # update the config file with the extra modules
+        if extra_modules:
+            pysidedeploy_config.update_config()
 
         modules = ",".join(pysidedeploy_config.modules)
+
+        # gets the xml dependency files from Qt installation path
+        dependency_files = self.__get_dependency_files(modules=pysidedeploy_config.modules,
+                                                       arch=self.arch)
+
         local_libs = self.__find_local_libs(dependency_files)
         pysidedeploy_config.local_libs += local_libs
 
@@ -80,31 +97,18 @@ class BuildozerConfig(BaseConfig):
 
         self.update_config()
 
-    def __get_dependency_files(self, pysidedeploy_config: Config) -> List[zipfile.Path]:
+    def __get_dependency_files(self, modules: List[str], arch: str) -> List[zipfile.Path]:
         """
         Based on pysidedeploy_config.modules, returns the
         Qt6{module}_{arch}-android-dependencies.xml file, which contains the various
         dependencies of the module, like permissions, plugins etc
         """
         dependency_files = []
-        needed_dependency_files = [(f"Qt{MAJOR_VERSION}{module}_{self.arch}"
-                                   "-android-dependencies.xml") for module in
-                                   pysidedeploy_config.modules]
-        archive = zipfile.ZipFile(pysidedeploy_config.wheel_pyside)
-
-        # find parent path to dependency files in the wheel
-        dependency_parent_path = None
-        for file in archive.namelist():
-            if file.endswith("android-dependencies.xml"):
-                dependency_parent_path = Path(file).parent
-                # all dependency files are in the same path
-                break
+        needed_dependency_files = [(f"Qt{MAJOR_VERSION}{module}_{arch}"
+                                    "-android-dependencies.xml") for module in modules]
 
         for dependency_file_name in needed_dependency_files:
-            dependency_file = dependency_parent_path / dependency_file_name
-            # convert from pathlib.Path to zipfile.Path
-            dependency_file = zipfile.Path(archive, at=str(dependency_file))
-
+            dependency_file = self.qt_libs_path / dependency_file_name
             if dependency_file.exists():
                 dependency_files.append(dependency_file)
 
@@ -180,6 +184,46 @@ class BuildozerConfig(BaseConfig):
 
         return list(local_libs)
 
+    def __find_dependent_qt_modules(self, pysidedeploy_config: Config):
+        """
+        Given pysidedeploy_config.modules, find all the other dependent Qt modules. This is
+        done by using llvm-readobj (readelf) to find the dependent libraries from the module
+        library.
+        """
+        dependent_modules = set()
+        all_dependencies = set()
+        lib_pattern = re.compile(f"libQt6(?P<mod_name>.*)_{self.arch}")
+
+        llvm_readobj = get_llvm_readobj(pysidedeploy_config.ndk_path)
+        if not llvm_readobj.exists():
+            raise FileNotFoundError(f"[DEPLOY] {llvm_readobj} does not exist."
+                                    "Finding Qt dependencies failed")
+
+        archive = zipfile.ZipFile(pysidedeploy_config.wheel_pyside)
+        lib_path_suffix = Path(str(self.qt_libs_path)).relative_to(pysidedeploy_config.wheel_pyside)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            archive.extractall(tmpdir)
+            qt_libs_tmpdir = Path(tmpdir) / lib_path_suffix
+            # find the lib folder where Qt libraries are stored
+            for module_name in pysidedeploy_config.modules:
+                qt_module_path = qt_libs_tmpdir / f"libQt6{module_name}_{self.arch}.so"
+                if not qt_module_path.exists():
+                    raise FileNotFoundError(f"[DEPLOY] libQt6{module_name}_{self.arch}.so not found"
+                                            " inside the wheel")
+                find_lib_dependencies(llvm_readobj=llvm_readobj, lib_path=qt_module_path,
+                                      dry_run=pysidedeploy_config.dry_run,
+                                      used_dependencies=all_dependencies)
+
+        for dependency in all_dependencies:
+            match = lib_pattern.search(dependency)
+            if match:
+                module = match.group("mod_name")
+                if module not in pysidedeploy_config.modules:
+                    dependent_modules.add(module)
+
+        return list(dependent_modules)
+
 
 class Buildozer:
     dry_run = False
diff --git a/sources/pyside-tools/deploy_lib/python_helper.py b/sources/pyside-tools/deploy_lib/python_helper.py
index e86ce2e1c..0174a8fcf 100644
--- a/sources/pyside-tools/deploy_lib/python_helper.py
+++ b/sources/pyside-tools/deploy_lib/python_helper.py
@@ -1,9 +1,12 @@
 # Copyright (C) 2022 The Qt Company Ltd.
 # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
-import sys
+import ast
 import os
+import re
+import sys
 import logging
+from typing import List
 from importlib import util
 if sys.version_info >= (3, 8):
     from importlib.metadata import version
@@ -13,6 +16,88 @@ from pathlib import Path
 
 from . import Nuitka, run_command, Config
 
+IMPORT_WARNING_PYSIDE = (f"[DEPLOY] Found 'import PySide6' in file {0}"
+                         ". Use 'from PySide6 import <module>' or pass the module"
+                         " needed using --extra-modules command line argument")
+
+
+def find_pyside_modules(project_dir: Path, extra_ignore_dirs: List[Path] = None,
+                        project_data=None):
+    """
+    Searches all the python files in the project to find all the PySide modules used by
+    the application.
+    """
+    all_modules = set()
+    mod_pattern = re.compile("PySide6.Qt(?P<mod_name>.*)")
+
+    def pyside_imports(py_file: Path):
+        modules = []
+        contents = py_file.read_text(encoding="utf-8")
+        try:
+            tree = ast.parse(contents)
+            for node in ast.walk(tree):
+                if isinstance(node, ast.ImportFrom):
+                    main_mod_name = node.module
+                    if main_mod_name.startswith("PySide6"):
+                        if main_mod_name == "PySide6":
+                            # considers 'from PySide6 import QtCore'
+                            for imported_module in node.names:
+                                full_mod_name = imported_module.name
+                                if full_mod_name.startswith("Qt"):
+                                    modules.append(full_mod_name[2:])
+                            continue
+
+                        # considers 'from PySide6.QtCore import Qt'
+                        match = mod_pattern.search(main_mod_name)
+                        if match:
+                            mod_name = match.group("mod_name")
+                            modules.append(mod_name)
+                        else:
+                            logging.warning((
+                                f"[DEPLOY] Unable to find module name from{ast.dump(node)}"))
+
+                if isinstance(node, ast.Import):
+                    for imported_module in node.names:
+                        full_mod_name = imported_module.name
+                        if full_mod_name == "PySide6":
+                            logging.warning(IMPORT_WARNING_PYSIDE.format(str(py_file)))
+
+        except Exception as e:
+            logging.error(f"Finding module import failed on file {str(py_file)}")
+            raise e
+
+        return set(modules)
+
+    py_candidates = []
+    ignore_dirs = ["__pycache__", "env", "venv", "deployment"]
+
+    if project_data:
+        py_candidates = project_data.python_files
+        for py_candidate in py_candidates:
+            all_modules = all_modules.union(pyside_imports(py_candidate))
+        return list(all_modules)
+
+    # incase there is not .pyproject file, search all python files in project_dir, except
+    # ignore_dirs
+    if extra_ignore_dirs:
+        ignore_dirs.extend(extra_ignore_dirs)
+
+    # find relevant .py files
+    _walk = os.walk(project_dir)
+    for root, dirs, files in _walk:
+        dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith(".")]
+        for py_file in files:
+            if py_file.endswith(".py"):
+                py_candidates.append(Path(root) / py_file)
+
+    for py_candidate in py_candidates:
+        all_modules = all_modules.union(pyside_imports(py_candidate))
+
+    if not all_modules:
+        ValueError("[DEPLOY] No PySide6 modules were found")
+
+    return list(all_modules)
+
 
 class PythonExecutable:
     """