Android Deployment: auto download the required Android NDK and SDK

- Enable automatic download of Android NDK and SDK for creating
  Android PySide6 and shiboken6 wheels. Earlier, the user was
  required to manually download it and supply as a command line
  argument.

- They are downloaded into the HOME directory of the user into the
  folder .pyside6_android_deploy. This is also similar to buildozer
  which downloads them into .buildozer.

- The downloaded NDK is set to r25c, same as buildozer. The latest
  Android SDK packages are downloaded.

- For the deployment tool, incase the NDK and SDK path are not
  provided either through the command line or through the configuration
  .spec file, the it will look for the NDK and SDK inside
  .pyside6_android_deploy, before falling back to the default Android
   NDK and SDK that buildozer supplies. buildozer will download them.

- As a addition, a small code rearranging is also done, along with
  changing the deprecated subprocess.call() to subprocess.run().

Task-number: PYSIDE-1612
Change-Id: Ie8d51f8c7634b5b320b1dce2d4164985cbfbaaca
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

4 files changed, 335 insertions, 36 deletions

diff --git a/sources/pyside-tools/android_deploy.py b/sources/pyside-tools/android_deploy.py
index f10834c12..fbc613069 100644
--- a/sources/pyside-tools/android_deploy.py
+++ b/sources/pyside-tools/android_deploy.py
@@ -237,14 +237,13 @@ if __name__ == "__main__":
                         help=f"Path to shiboken{MAJOR_VERSION} Android Wheel",
                         required=not config_option_exists())
 
-    #TODO: --ndk-path and --sdk-path will be removed when automatic download of sdk and ndk is added
     parser.add_argument("--ndk-path", type=lambda p: Path(p).resolve(),
                         help=("Path to Android Ndk. If omitted, the default from buildozer is used")
-                        , required=True)
+                        )
 
     parser.add_argument("--sdk-path", type=lambda p: Path(p).resolve(),
                         help=("Path to Android Sdk. If omitted, the default from buildozer is used")
-                        , required=True)
+                        )
 
     parser.add_argument("--extra-ignore-dirs", type=str, help=HELP_EXTRA_IGNORE_DIRS)
 
diff --git a/sources/pyside-tools/deploy_lib/config.py b/sources/pyside-tools/deploy_lib/config.py
index af019a093..8af341bad 100644
--- a/sources/pyside-tools/deploy_lib/config.py
+++ b/sources/pyside-tools/deploy_lib/config.py
@@ -12,6 +12,9 @@ from .commands import run_qmlimportscanner
 # Some QML plugins like QtCore are excluded from this list as they don't contribute much to
 # executable size. Excluding them saves the extra processing of checking for them in files
 EXCLUDED_QML_PLUGINS = {"QtQuick", "QtQuick3D", "QtCharts", "QtWebEngine", "QtTest", "QtSensors"}
+# TODO: Move this to android module. Fix circular import.
+ANDROID_NDK_VERSION = "25c"
+ANDROID_DEPLOY_CACHE = Path.home() / ".pyside6_android_deploy"
 
 
 class BaseConfig:
@@ -128,13 +131,30 @@ class Config(BaseConfig):
                 self.ndk_path = android_data.ndk_path
             else:
                 ndk_path_temp = self.get_value("buildozer", "ndk_path")
-                self.ndk_path = Path(ndk_path_temp) if ndk_path_temp else None
+                if ndk_path_temp:
+                    self.ndk_path = Path(ndk_path_temp)
+                else:
+                    self.ndk_path = (ANDROID_DEPLOY_CACHE / "android-ndk"
+                                     / f"android-ndk-r{ANDROID_NDK_VERSION}")
+                    if not self.ndk_path.exists():
+                        logging.info("[DEPLOY] Use default NDK from buildoer")
+
+            if self.ndk_path:
+                print(f"Using Android NDK: {str(self.ndk_path)}")
 
             if android_data.sdk_path:
                 self.sdk_path = android_data.sdk_path
             else:
                 sdk_path_temp = self.get_value("buildozer", "sdk_path")
-                self.sdk_path = Path(sdk_path_temp) if sdk_path_temp else None
+                if sdk_path_temp:
+                    self.sdk_path = Path(sdk_path_temp)
+                else:
+                    self.sdk_path = ANDROID_DEPLOY_CACHE / "android-sdk"
+                    if not self.sdk_path.exists():
+                        logging.info("[DEPLOY] Use default SDK from buildozer")
+
+            if self.sdk_path:
+                print(f"Using Android SDK: {str(self.sdk_path)}")
 
             recipe_dir_temp = self.get_value("buildozer", "recipe_dir")
             self.recipe_dir = Path(recipe_dir_temp) if recipe_dir_temp else None
diff --git a/tools/cross_compile_android/android_utilities.py b/tools/cross_compile_android/android_utilities.py
new file mode 100644
index 000000000..e93631173
--- /dev/null
+++ b/tools/cross_compile_android/android_utilities.py
@@ -0,0 +1,256 @@
+# Copyright (C) 2023 The Qt Company Ltd.
+# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+import logging
+import shutil
+import os
+import stat
+import sys
+import subprocess
+
+from urllib import request
+from pathlib import Path
+from typing import List
+from packaging import version
+from tqdm import tqdm
+
+# the tag number does not matter much since we update the sdk later
+DEFAULT_SDK_TAG = 6514223
+ANDROID_NDK_VERSION = "25c"
+
+
+def run_command(command: List[str], cwd: str = None, ignore_fail: bool = False,
+                dry_run: bool = False, accept_prompts: bool = False, show_stdout: bool = False,
+                capture_stdout: bool = False):
+
+    if capture_stdout and not show_stdout:
+        raise RuntimeError("capture_stdout should always be used together with show_stdout")
+
+    if dry_run:
+        print(" ".join(command))
+        return
+
+    input = None
+    if accept_prompts:
+        input = str.encode("y")
+
+    if show_stdout:
+        stdout = None
+    else:
+        stdout = subprocess.DEVNULL
+
+    result = subprocess.run(command, cwd=cwd, input=input, stdout=stdout,
+                            capture_output=capture_stdout)
+
+    if result.returncode != 0 and not ignore_fail:
+        sys.exit(result.returncode)
+
+    if capture_stdout and not result.returncode:
+        return result.stdout.decode("utf-8")
+
+    return None
+
+
+class DownloadProgressBar(tqdm):
+    def update_to(self, b=1, bsize=1, tsize=None):
+        if tsize is not None:
+            self.total = tsize
+        self.update(b * bsize - self.n)
+
+
+class SdkManager:
+    def __init__(self, android_sdk_dir: Path, dry_run: bool = False):
+        self._sdk_manager = android_sdk_dir / "tools" / "bin" / "sdkmanager"
+
+        if not self._sdk_manager.exists():
+            raise RuntimeError(f"Unable to find SdkManager in {str(self._sdk_manager)}")
+
+        if not os.access(self._sdk_manager, os.X_OK):
+            current_permissions = stat.S_IMODE(os.lstat(self._sdk_manager).st_mode)
+            os.chmod(self._sdk_manager, current_permissions | stat.S_IEXEC)
+
+        self._android_sdk_dir = android_sdk_dir
+        self._dry_run = dry_run
+
+    def list_packages(self):
+        command = [self._sdk_manager, f"--sdk_root={self._android_sdk_dir}", "--list"]
+        return run_command(command=command, dry_run=self._dry_run, show_stdout=True,
+                           capture_stdout=True)
+
+    def install(self, *args, accept_license: bool = False, show_stdout=False):
+        command = [str(self._sdk_manager), f"--sdk_root={self._android_sdk_dir}", *args]
+        run_command(command=command, dry_run=self._dry_run,
+                    accept_prompts=accept_license, show_stdout=show_stdout)
+
+
+def _unpack(zip_file: Path, destination: Path):
+    """
+    Unpacks the zip_file into destination preserving all permissions
+
+    TODO: Try to use zipfile module. Currently we cannot use zipfile module here because
+    extractAll() does not preserve permissions.
+
+    In case `unzip` is not available, the user is requested to install it manually
+    """
+    unzip = shutil.which("unzip")
+    if not unzip:
+        raise RuntimeError("Unable to find program unzip. Use `sudo apt-get install unzip`"
+                           "to install it")
+
+    command = [unzip, zip_file, "-d", destination]
+    run_command(command=command, show_stdout=True)
+
+
+def _download(url: str, destination: Path):
+    """
+    Download url to destination
+    """
+    headers, download_path = None, None
+    # https://github.com/tqdm/tqdm#hooks-and-callbacks
+    with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
+        download_path, headers = request.urlretrieve(url=url, filename=destination,
+                                                     reporthook=t.update_to)
+    assert headers["Content-Type"] == "application/zip"
+    assert Path(download_path).resolve() == destination
+
+
+def download_android_ndk(ndk_path: Path):
+    """
+    Downloads the given ndk_version into ndk_path
+    """
+    ndk_path = ndk_path / "android-ndk"
+    ndk_zip_path = ndk_path / f"android-ndk-r{ANDROID_NDK_VERSION}-linux.zip"
+    ndk_version_path = ndk_path / f"android-ndk-r{ANDROID_NDK_VERSION}"
+
+    if ndk_version_path.exists():
+        print(f"NDK path found in {str(ndk_version_path)}")
+    else:
+        ndk_path.mkdir(parents=True, exist_ok=True)
+        url = f"https://dl.google.com/android/repository/android-ndk-r{ANDROID_NDK_VERSION}-linux.zip"
+
+        print(f"Downloading Android Ndk version r{ANDROID_NDK_VERSION}")
+        _download(url=url, destination=ndk_zip_path)
+
+        print("Unpacking Android Ndk")
+        _unpack(zip_file=(ndk_path /
+                f"android-ndk-r{ANDROID_NDK_VERSION}-linux.zip"),
+                destination=ndk_path)
+
+    return ndk_version_path
+
+
+def download_android_commandlinetools(android_sdk_dir: Path):
+    """
+    Downloads Android commandline tools into cltools_path.
+    """
+    android_sdk_dir = android_sdk_dir / "android-sdk"
+    url = ("https://dl.google.com/android/repository/"
+           f"commandlinetools-linux-{DEFAULT_SDK_TAG}_latest.zip")
+    cltools_zip_path = android_sdk_dir / f"commandlinetools-linux-{DEFAULT_SDK_TAG}_latest.zip"
+    cltools_path = android_sdk_dir / "tools"
+
+    if cltools_path.exists():
+        print(f"Command-line tools found in {str(cltools_path)}")
+    else:
+        android_sdk_dir.mkdir(parents=True, exist_ok=True)
+
+        print("Download Android Command Line Tools: "
+              f"commandlinetools-linux-{DEFAULT_SDK_TAG}_latest.zip")
+        _download(url=url, destination=cltools_zip_path)
+
+        print("Unpacking Android Command Line Tools")
+        _unpack(zip_file=cltools_zip_path, destination=android_sdk_dir)
+
+    return android_sdk_dir
+
+
+def android_list_build_tools_versions(sdk_manager: SdkManager):
+    """
+    List all the build-tools versions available for download
+    """
+    available_packages = sdk_manager.list_packages()
+    build_tools_versions = []
+    lines = available_packages.split('\n')
+
+    for line in lines:
+        if not line.strip().startswith('build-tools;'):
+            continue
+        package_name = line.strip().split(' ')[0]
+        if package_name.count(';') != 1:
+            raise RuntimeError(f"Unable to parse build-tools version: {package_name}")
+        ver = package_name.split(';')[1]
+
+        build_tools_versions.append(version.Version(ver))
+
+    return build_tools_versions
+
+
+def find_installed_buildtools_version(build_tools_dir: Path):
+    """
+    It is possible that the user has multiple build-tools installed. The newer  version is generally
+    used. This function find the newest among the installed build-tools
+    """
+    versions = [version.Version(bt_dir.name) for bt_dir in build_tools_dir.iterdir()
+                if bt_dir.is_dir()]
+    return max(versions)
+
+
+def find_latest_buildtools_version(sdk_manager: SdkManager):
+    """
+    Uses sdk manager to find the latest build-tools version
+    """
+    available_build_tools_v = android_list_build_tools_versions(sdk_manager=sdk_manager)
+
+    if not available_build_tools_v:
+        raise RuntimeError('Unable to find any build tools available for download')
+
+    return max(available_build_tools_v)
+
+
+def install_android_packages(android_sdk_dir: Path, android_api: str, dry_run: bool = False,
+                             accept_license: bool = False, skip_update: bool = False):
+    """
+    Use the sdk manager to install build-tools, platform-tools and platform API
+    """
+    tools_dir = android_sdk_dir / "tools"
+    if not tools_dir.exists():
+        raise RuntimeError("Unable to find Android command-line tools in "
+                           f"{str(tools_dir)}")
+
+    # incase of --verbose flag
+    show_output = (logging.getLogger().getEffectiveLevel() == logging.INFO)
+
+    sdk_manager = SdkManager(android_sdk_dir=android_sdk_dir, dry_run=dry_run)
+
+    # install/upgrade platform-tools
+    if not (android_sdk_dir / "platform-tools").exists():
+        print("Installing/Updating Android platform-tools")
+        sdk_manager.install("platform-tools", accept_license=accept_license,
+                            show_stdout=show_output)
+        # The --update command is only relevant for platform tools
+        if not skip_update:
+            sdk_manager.install("--update", show_stdout=show_output)
+
+    # install/upgrade build-tools
+    buildtools_dir = android_sdk_dir / "build-tools"
+
+    if not buildtools_dir.exists():
+        latest_build_tools_v = find_latest_buildtools_version(sdk_manager=sdk_manager)
+        print(f"Installing Android build-tools version {latest_build_tools_v}")
+        sdk_manager.install(f"build-tools;{latest_build_tools_v}", show_stdout=show_output)
+    else:
+        if not skip_update:
+            latest_build_tools_v = find_latest_buildtools_version(sdk_manager=sdk_manager)
+            installed_build_tools_v = find_installed_buildtools_version(buildtools_dir)
+            if latest_build_tools_v > installed_build_tools_v:
+                print(f"Updating Android build-tools version to {latest_build_tools_v}")
+                sdk_manager.install(f"build-tools;{latest_build_tools_v}", show_stdout=show_output)
+                installed_build_tools_v = latest_build_tools_v
+
+    # install the platform API
+    platform_api_dir = android_sdk_dir / "platforms" / f"android-{android_api}"
+    if not platform_api_dir.exists():
+        print(f"Installing Android platform API {android_api}")
+        sdk_manager.install(f"platforms;android-{android_api}", show_stdout=show_output)
+
+    print("Android packages installation done")
diff --git a/tools/cross_compile_android/main.py b/tools/cross_compile_android/main.py
index 073fe74e8..97f4030ac 100644
--- a/tools/cross_compile_android/main.py
+++ b/tools/cross_compile_android/main.py
@@ -16,9 +16,31 @@ from git import Repo, RemoteProgress
 from tqdm import tqdm
 from jinja2 import Environment, FileSystemLoader
 
+from android_utilities import (run_command, download_android_commandlinetools,
+                                download_android_ndk, install_android_packages)
+
 # Note: Does not work with PyEnv. Your Host Python should contain openssl.
 PYTHON_VERSION = "3.10"
 
+APIC_HELP = ('''
+Points to the installation path of Python for the specific Android
+platform. If the path given does not exist, then Python for Android
+is cross compiled for the specific platform and installed into this
+path as <path>/Python-'plat_name'/_install.
+
+If this path is not given, then Python for Android is cross-compiled
+into a temportary directory, which is deleted when the Qt for Python
+Android wheels are created.
+''')
+
+SKIP_UPDATE_HELP = ("skip the updation of SDK packages build-tools, platform-tools to"
+                    " latest version")
+
+ACCEPT_LICENSE_HELP = ('''
+Accepts license automatically for Android SDK installation. Otherwise,
+accept the license manually through command line.
+''')
+
 
 @dataclass
 class PlatformData:
@@ -48,20 +70,10 @@ class CloneProgress(RemoteProgress):
         self.pbar.refresh()
 
 
-def run_command(command: List[str], cwd: str = None, ignore_fail: bool = False,
-                dry_run: bool = False):
-    if dry_run:
-        print(" ".join(command))
-        return
-    ex = subprocess.call(command, cwd=cwd)
-    if ex != 0 and not ignore_fail:
-        sys.exit(ex)
-
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        description="This tool cross builds cpython for android and uses that Python to cross build"
-                    "android Qt for Python wheels",
+        description="This tool cross builds CPython for Android and uses that Python to cross build"
+                    "Android Qt for Python wheels",
         formatter_class=argparse.RawTextHelpFormatter,
     )
 
@@ -72,11 +84,9 @@ if __name__ == "__main__":
     parser.add_argument("-v", "--verbose", help="run in verbose mode", action="store_const",
                         dest="loglevel", const=logging.INFO)
     parser.add_argument("--api-level", type=str, default="31", help="Android API level to use")
-    parser.add_argument("--ndk-path", type=str, required=True,
-                        help="Path to Android NDK (Preferred 25b)")
+    parser.add_argument("--ndk-path", type=str, help="Path to Android NDK (Preferred 25b)")
     # sdk path is needed to compile all the Qt Java Acitivity files into Qt6AndroidBindings.jar
-    parser.add_argument("--sdk-path", type=str, required=True,
-                        help="Path to Android SDK")
+    parser.add_argument("--sdk-path", type=str, help="Path to Android SDK")
     parser.add_argument("--qt-install-path", type=str, required=not occp_exists(),
                         help="Qt installation path eg: /home/Qt/6.5.0")
 
@@ -85,19 +95,16 @@ if __name__ == "__main__":
 
     parser.add_argument("-apic", "--android-python-install-path", type=str, default=None,
                         required=occp_exists(),
-                        help='''
-                        Points to the installation path of Python for the specific Android
-                        platform. If the path given does not exist, then Python for android
-                        is cross compiled for the specific platform and installed into this
-                        path as <path>/Python-'plat_name'/_install.
-
-                        If this path is not given, then Python for android is cross-compiled
-                        into a temportary directory, which is deleted when the Qt for Python
-                        android wheels are created.
-                        ''')
+                        help=APIC_HELP)
 
     parser.add_argument("--dry-run", action="store_true", help="show the commands to be run")
 
+    parser.add_argument("--skip-update", action="store_true",
+                        help=SKIP_UPDATE_HELP)
+
+    parser.add_argument("--auto-accept-license", action="store_true",
+                        help=ACCEPT_LICENSE_HELP)
+
     args = parser.parse_args()
 
     logging.basicConfig(level=args.loglevel)
@@ -114,6 +121,24 @@ if __name__ == "__main__":
     gcc_march = None
     plat_bits = None
     dry_run = args.dry_run
+    plat_name = args.plat_name
+    api_level = args.api_level
+    skip_update = args.skip_update
+    auto_accept_license = args.auto_accept_license
+
+    # auto download Android NDK and SDK
+    pyside6_deploy_cache = Path.home() / ".pyside6_android_deploy"
+
+    if not ndk_path:
+        # Download android ndk
+        ndk_path = download_android_ndk(pyside6_deploy_cache)
+
+    if not sdk_path:
+        # download and unzip command-line tools
+        sdk_path = download_android_commandlinetools(pyside6_deploy_cache)
+        # install and update required android packages
+        install_android_packages(android_sdk_dir=sdk_path, android_api=api_level, dry_run=dry_run,
+                                 accept_license=auto_accept_license, skip_update=skip_update)
 
     # python path is valid, if Python for android installation exists in python_path
     valid_python_path = True
@@ -130,8 +155,6 @@ if __name__ == "__main__":
                 break
 
     templates_path = Path(__file__).parent / "templates"
-    plat_name = args.plat_name
-    api_level = args.api_level
 
     # for armv7a the API level dependent binaries like clang are named
     # armv7a-linux-androideabi27-clang, as opposed to other platforms which
@@ -187,7 +210,7 @@ if __name__ == "__main__":
 
             # run the cross compile script
             logging.info(f"Running Python cross-compile for platform {platform_data.plat_name}")
-            run_command(["./cross_compile.sh"], cwd=cpython_dir, dry_run=dry_run)
+            run_command(["./cross_compile.sh"], cwd=cpython_dir, dry_run=dry_run, show_stdout=True)
 
             python_path = (f"{android_py_install_path_prefix}/Python-{platform_data.plat_name}-linux-android/"
                            "_install")
@@ -196,7 +219,8 @@ if __name__ == "__main__":
             # libpython3.x.so, to match with python_for_android's Python library. Otherwise,
             # the Qfp binaries won't be able to link to Python
             run_command(["patchelf", "--set-soname", f"libpython{PYTHON_VERSION}.so",
-                         f"libpython{PYTHON_VERSION}.so.1.0"], cwd=Path(python_path) / "lib")
+                        f"libpython{PYTHON_VERSION}.so.1.0"], cwd=Path(python_path) / "lib",
+                        dry_run=dry_run)
 
             logging.info(
                 f"Cross compile Python for Android platform {platform_data.plat_name}. "
@@ -242,4 +266,4 @@ if __name__ == "__main__":
                             (f"--qt-target-path={qt_install_path}/"
                              f"android_{platform_data.qt_plat_name}"),
                             "--no-qt-tools", "--skip-docs", "--unity"]
-        run_command(qfp_ccompile_cmd, cwd=pyside_setup_dir, dry_run=dry_run)
+        run_command(qfp_ccompile_cmd, cwd=pyside_setup_dir, dry_run=dry_run, show_stdout=True)