notarize.py: refactor, use notarytool for notarization

-Altool will be deprecated, move to using notarytool (Xcode 13+).
-Support App Store Connect API keys in addition to developer Apple ID.
-Allow to notarize file formats other than DMG,
create a ZIP to notarize when the upload file type isn't supported.
-Keep the old API for now, to be removed in a future patch

Task-number: QTBUG-106822
Change-Id: Ib4f52257929dda64f79c6be1e515c60d2d61471d
Reviewed-by: Iikka Eklund <iikka.eklund@qt.io>

3 files changed, 342 insertions, 157 deletions

diff --git a/packaging-tools/build_wrapper.py b/packaging-tools/build_wrapper.py
index cb5e96fac..07e456d16 100644
--- a/packaging-tools/build_wrapper.py
+++ b/packaging-tools/build_wrapper.py
@@ -44,6 +44,7 @@ from contextlib import suppress
 from getpass import getuser
 from glob import glob
 from io import TextIOWrapper
+from pathlib import Path
 from time import gmtime, strftime
 from typing import Any, Dict, List, Optional, Tuple
 from urllib.parse import urlparse
@@ -77,11 +78,6 @@ from read_remote_config import get_pkg_value
 from runner import run_cmd
 from threadedwork import Task, ThreadedWork
 
-if sys.version_info < (3, 7):
-    from asyncio_backport import run as asyncio_run
-else:
-    from asyncio import run as asyncio_run
-
 log = init_logger(__name__, debug_mode=False)
 
 SCRIPT_ROOT_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -920,7 +916,8 @@ def handle_qt_creator_build(option_dict: Dict[str, str], qtcreator_plugins: List
 
     # notarize
     if is_macos() and do_notarize:
-        notarize_dmg(os.path.join(work_dir, 'qt-creator_build', 'qt-creator.dmg'), 'Qt Creator')
+        with ch_dir(SCRIPT_ROOT_DIR):
+            notarize(path=Path(work_dir, 'qt-creator_build', 'qt-creator.dmg'))
 
     # Upload
     file_upload_list: List[Tuple[str, str]] = []  # pairs (source, dest), source relative to WORK_DIR, dest relative to server + dir_path
@@ -1074,14 +1071,6 @@ def handle_sdktool_build(option_dict: Dict[str, str]) -> None:
     update_job_link(unversioned_base_path, base_path, option_dict)
 
 
-def notarize_dmg(dmg_path: str, installer_name_base: str) -> None:
-    # this is just a unique id without any special meaning, used to track the notarization progress
-    bundle_id = installer_name_base + "-" + strftime('%Y-%m-%d', gmtime())
-    bundle_id = bundle_id.replace('_', '-').replace(' ', '')  # replace illegal chars for bundle_id
-    with ch_dir(SCRIPT_ROOT_DIR):
-        asyncio_run(notarize(dmg=dmg_path, bundle_id=bundle_id))
-
-
 ###############################
 # Update latest link
 ###############################
diff --git a/packaging-tools/notarize.py b/packaging-tools/notarize.py
index 3590ec5ea..31d389bac 100755
--- a/packaging-tools/notarize.py
+++ b/packaging-tools/notarize.py
@@ -30,165 +30,368 @@
 #############################################################################
 
 import argparse
-import asyncio
+import json
+import shutil
 import sys
+from pathlib import Path
+from platform import mac_ver
 from shutil import which
-from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired
-from time import gmtime, sleep, strftime
-from typing import List
+from subprocess import CalledProcessError
+from typing import Optional
 
+from bld_utils import is_macos
 from logging_util import init_logger
 from read_remote_config import get_pkg_value
-
-if sys.version_info < (3, 7):
-    from asyncio_backport import run as asyncio_run
-else:
-    from asyncio import run as asyncio_run
+from runner import run_cmd, run_cmd_silent
 
 log = init_logger(__name__, debug_mode=False)
 
 
 class NotarizationError(Exception):
-    pass
+    """NotarizationError exception class derived from the Exception class"""
 
 
-def parse_value_from_data(key: str, data: str) -> str:
-    for line in data.split("\n"):
-        if line.strip().startswith(key):
-            return line.split(key)[-1].strip()
-    return ""
+def store_credentials_api_key(profile_name: str, key: str, key_id: str, issuer: str) -> None:
+    """
+    Store App Store Connect API Keys to keychain and validate them with notarytool.
 
+    Args:
+        profile_name: The profile name in keychain to which to save the credentials
+        key: App Store Connect API key file system path
+        key_id: App Store Connect API Key ID specific to a team (~10 chars)
+        issuer: App Store Connect API Issuer ID in UUID format
 
-async def request_cmd(timeout: float, cmd: List[str]) -> str:
-    proc = await asyncio.create_subprocess_exec(*cmd, stdout=PIPE, stderr=STDOUT)
-    attempts = 3
+    Raises:
+        NotarizationError: When the command saving and validating the credentials fails
 
-    while attempts:
-        try:
-            data = await asyncio.wait_for(proc.communicate(), timeout=timeout)
-            break
-        except (asyncio.TimeoutError, TimeoutExpired):
-            log.warning("Timeout (%ss)", str(timeout))
-            attempts -= 1
-            if attempts:
-                log.info("Waiting a bit before next attempt..")
-                await asyncio.sleep(60)
-        except CalledProcessError as command_err:
-            log.critical("Failed to run command: %s", str(command_err))
-            raise
-        except Exception as error:
-            log.critical("Something failed: %s", str(error))
-            raise
-
-    return data[0].decode('utf-8')
-
-
-async def request_notarization(
-    user: str, passwd: str, bundle_id: str, dmg: str, timeout: float
-) -> str:
-    # long lasting command, it uploads the binary to Apple server
-    cmd = ["xcrun", "altool", "-u", user, "-p", passwd, "--notarize-app", "-t", "osx"]
-    cmd += ["--primary-bundle-id", bundle_id, "-f", dmg]
-
-    data = await request_cmd(timeout, cmd)
-    request_uuid = parse_value_from_data("RequestUUID", data)
-    if not request_uuid:
-        raise NotarizationError(f"Failed to notarize app:\n\n{data}")
-    return request_uuid.split("=")[-1].strip()
-
-
-async def poll_notarization_completed(
-    user: str, passwd: str, dmg: str, timeout: float, uuid: str
-) -> bool:
-    cmd = ["xcrun", "altool", "-u", user, "-p", passwd, "--notarization-info", uuid]
-
-    attempts = 180
-    poll_interval = 60  # attempts * poll_interval = 3h
-    while attempts:
-        data = await request_cmd(timeout, cmd)
-        status_code = parse_value_from_data("Status Code:", data)
-
-        if status_code == "0":
-            log.info("Notarization succeeded for: %s", dmg)
-            log.info("%s", data)
-            return True
-        if status_code == "2":
-            log.info("Notarization failed for: %s", dmg)
-            raise NotarizationError(f"Notarization failed:\n\n{data}")
-        log.info("Notarization not ready yet for: %s", dmg)
-        log.info("%s", data)
-
-        attempts -= 1
-        log.info("Sleeping %is before next poll attempt (attempts left: %i)", poll_interval, attempts)
-        await asyncio.sleep(poll_interval)
-
-    log.warning("Notarization poll timeout..")
-    return False
-
-
-async def embed_notarization(dmg: str, timeout: float) -> None:
-    # Embed the notarization in the dmg package
-    cmd = ["xcrun", "stapler", "staple", dmg]
-    retry_count = 10
-    delay: float = 60
-    while retry_count:
-        retry_count -= 1
-        data = await request_cmd(timeout, cmd)
-        status = parse_value_from_data("The staple and validate action", data)
-
-        if status.lower().startswith("worked"):
-            log.info("The [%s] was notirized successfully!", dmg)
-            break
-
-        log.error("Failed to 'staple' the %s - Reason:\n\n%s", dmg, data)
-
-        if retry_count:
-            log.warning("Trying again after %ss", delay)
-            sleep(delay)
-            delay = delay + delay / 2  # 60, 90, 135, 202, 303
-        else:
-            log.critical("Execution of the remote script probably failed!")
-            raise NotarizationError(f"Failed to 'staple' the: {dmg}")
-
-
-async def notarize(
-    dmg: str,
-    user: str = "",
-    passwd: str = "",
-    bundle_id: str = strftime("%Y-%m-%d-%H-%M-%S", gmtime()),
-    timeout: float = 60 * 60 * 3,
+    """
+    log.info("Store App Store Connect API Keys to keychain profile '%s'", profile_name)
+    cmd = ["xcrun", "notarytool", "store-credentials", profile_name]
+    cmd += ["--key", key, "--key-id", key_id, "--issuer", issuer]
+    if not run_cmd_silent(cmd=cmd):
+        raise NotarizationError("Failed to save or validate API key credentials")
+
+
+def store_credentials_apple_id(
+    profile_name: str, apple_id: str, password: str, team_id: str
 ) -> None:
-    """Notarize"""
-    user = user or get_pkg_value("AC_USERNAME")
-    passwd = passwd or get_pkg_value("AC_PASSWORD")
-    uuid = await request_notarization(user, passwd, bundle_id, dmg, timeout)
-    if not await poll_notarization_completed(user, passwd, dmg, timeout, uuid):
-        raise NotarizationError(f"Notarization failed for: {dmg}")
-    await embed_notarization(dmg, timeout)
+    """
+    Store Developer Apple ID credentials to keychain and validate them with notarytool.
 
+    Args:
+        profile_name: The profile name in keychain to which to save the credentials
+        apple_id: The Apple ID login username for accessing Developer ID services
+        password: App-specific password generated for the aforementioned Apple ID
+        team_id: The Developer Team identifier to use with the credentials (~10 chars)
+
+    Raises:
+        NotarizationError: When the command saving and validating the credentials fails
+
+    """
+    log.info("Store Developer Apple ID credentials to keychain profile '%s'", profile_name)
+    cmd = ["xcrun", "notarytool", "store-credentials", profile_name]
+    cmd += ["--apple-id", apple_id, "--password", password, "--team-id", team_id]
+    if not run_cmd_silent(cmd=cmd):
+        raise NotarizationError("Failed to save or validate Apple ID credentials")
 
-def main() -> None:
-    """Main"""
-    parser = argparse.ArgumentParser(prog="Helper script to notarize given macOS disk image (.dmg)")
-    parser.add_argument("--dmg", dest="dmg", required=True, type=str, help=".dmg file")
-    parser.add_argument("--user", dest="user", type=str, default=get_pkg_value("AC_USERNAME"), help="App Store Connect Username")
-    parser.add_argument("--passwd", dest="passwd", type=str, default=get_pkg_value("AC_PASSWORD"), help="App Store Connect Password")
-    parser.add_argument("--bundle-id", dest="bundle_id", default=strftime('%Y-%m-%d-%H-%M-%S', gmtime()), type=str, help="Give unique id for this bundle")
-    parser.add_argument("--timeout", dest="timeout", type=int, default=60 * 60 * 3, help="Timeout value for the remote requests")
-    args = parser.parse_args(sys.argv[1:])
 
+def process_notarize_result(
+    json_data: str, profile_name: str, timeout: Optional[int] = None
+) -> None:
+    """
+    Parse and log notarization results from json_data and raise the encountered errors.
+
+    Args:
+        json_data: Output data from notarizetool with the --output-format json specified
+        profile_name: The keychain profile containing the auth credentials for notarytool
+        timeout: Specify a timeout for notarytool command executed via run_cmd
+
+    Raises:
+        JSONDecodeError: If json_data formatting is malformed and cannot be decoded to json
+        NotarizationError: When the notarization is determined to have failed
+        CalledProcessError: When the command requesting additional logs for the UUID fails
+
+    """
+    try:
+        data = json.loads(json_data)
+    except json.JSONDecodeError as err:
+        log.exception("Error processing json response")
+        raise NotarizationError(f"Notarization failed: {str(err)}") from err
+    uuid = data.get("id", "")
+    status = data.get("status", "")
+    if status != "Accepted":
+        log.error("Notarization error: %s, UUID: %s", status, uuid)
+        log.error("Gathering more details about the errors...")
+        cmd = ["xcrun", "notarytool", "log", uuid, "-p", profile_name]
+        try:
+            run_cmd(cmd=cmd, timeout=timeout)
+        except CalledProcessError as err:
+            log.exception("Failed to log additional details", exc_info=err)
+        raise NotarizationError(f"Notarization failed, UUID: {uuid}")
+    log.info("Notarization complete, UUID: %s", uuid)
+
+
+def submit_for_notarization(
+    path: Path, profile_name: str, timeout: Optional[int] = None, acceleration: bool = True
+) -> None:
+    """
+    Submit a file to Apple's Notary service for notarization.
+    Wait for the server to return the result and pass the output to process_notarize_result.
+
+    Args:
+        path: The file system path for the file to notarize
+        profile_name: The keychain profile containing the auth credentials for notarytool
+        timeout: Specify a timeout for the notarytool command and run_cmd
+        acceleration: Whether to enable S3 transfer acceleration in the notarytool upload
+
+    """
+    cmd = ["xcrun", "notarytool", "submit", str(path), "-p", profile_name]
+    cmd += ["--wait", "--timeout", str(timeout), "--output-format", "json"]
+    if not acceleration:
+        cmd += ["--no-s3-acceleration"]
+    try:
+        result = run_cmd(cmd=cmd, timeout=timeout)
+    except CalledProcessError as err:
+        raise NotarizationError("Notarytool command failed, invalid arguments?") from err
+    process_notarize_result(result, profile_name, timeout=timeout)
+
+
+def prepare_payload(path: Path) -> Path:
+    """
+    Pack payload into a .zip archive if the path does not point to file supported by notarytool.
+    The Apple Notary service accepts the following formats:
+        - disk images (UDIF format), ending in .dmg
+        - signed flat installer packages, ending in .pkg
+        - ZIP compressed archives, ending in .zip
+
+    Args:
+        path: The file system path to the file or folder to process
+
+    Returns:
+        path: The file system path to the created zip file or the path originally passed in
+
+    Raises:
+        CalledProcessError: When the compress command via run_cmd returns a non-zero exit status.
+        NotarizationError: Raised on CalledProcessError
+
+    """
+    if path.suffix not in (".dmg", ".pkg", ".zip"):
+        zip_path: Path = path.with_suffix(".zip")
+        log.info("Compress to .zip before upload: %s", zip_path)
+        ditto_tool = shutil.which("ditto") or "/usr/bin/ditto"
+        cmd = [ditto_tool, "-c", "-k", "--keepParent", str(path), str(zip_path)]
+        try:
+            run_cmd(cmd=cmd)
+        except CalledProcessError as err:
+            raise NotarizationError(f"Failed to compress {path} to .zip") from err
+        path = zip_path
+    log.info("Ready to upload: %s", path)
+    return path
+
+
+def embed_notarization(path: Path) -> None:
+    """
+    Embed the ticket in the notarized package, supported file formats by stapler are:
+        - UDIF disk images (.dmg)
+        - code-signed executable bundles (.app)
+        - signed "flat" installer packages (.pkg)
+
+    Args:
+        path: The file system path to the previously notarized file
+
+    Raises:
+        CalledProcessError: When the stapler command via run_cmd returns a non-zero exit status.
+        NotarizationError: Raised on CalledProcessError on a supported file type.
+
+    """
+    log.info("Stapling package: %s", path)
+    try:
+        cmd = ['xcrun', 'stapler', 'staple', str(path)]
+        run_cmd(cmd=cmd)
+    except CalledProcessError as err:
+        if path.suffix in (".dmg", ".app", ".pkg"):
+            raise NotarizationError(f"Error embedding ticket: Stapler failed for {path}") from err
+        # Do not raise when file format is not known to support stapling, but log the error instead
+        log.exception("Ignored error while stapling %s: %s", str(path), str(err), exc_info=err)
+
+
+def key_from_remote_env(key: str) -> str:
+    """
+    Get value from remote environment with get_pkg_value if it exists, or return an empty string
+
+    Args:
+        key: The key for the value to look for in the environment
+
+    Returns:
+        Returned value from get_pkg_value if no exception was handled or an empty string (str)
+
+    Raises:
+        Exception: Raised by get_pkg_value, handled by the function
+
+    """
+    try:
+        return get_pkg_value(key)
+    except Exception:
+        return ""
+
+
+def check_notarize_reqs() -> None:
+    """
+    Check if the system supports notarization via notarytool and has the required tools installed.
+
+    Raises:
+        NotarizationError: If there are missing tools or the platform is not supported.
+
+    """
+    if not is_macos():
+        raise NotarizationError("Only macOS is supported. For other platforms see Notary API.")
+    if not [int(x) for x in mac_ver()[0].split(".")] >= [10, 15, 7]:
+        raise NotarizationError("Only macOS version 10.15.7+ is supported by notarytool")
+    if not shutil.which("ditto") and not Path("/usr/bin/ditto").exists():
+        raise NotarizationError("Couldn't find 'ditto': '/usr/bin/ditto' missing or not in PATH")
     if not which("xcrun"):
-        raise SystemExit("Could not find 'xcrun' from the system for notarization. Aborting..")
+        raise NotarizationError("Couldn't find 'xcrun'. Xcode Command Line Tools is required")
+    try:
+        run_cmd(["xcrun", "--find", "stapler"])
+    except CalledProcessError as err:
+        raise NotarizationError("Couldn't find 'stapler'. Xcode is required") from err
+    try:
+        run_cmd(["xcrun", "--find", "notarytool"])
+    except CalledProcessError as err:
+        raise NotarizationError("Couldn't find 'notarytool'. Xcode 13+ is required") from err
+
+
+def notarize(
+    path: Path,
+    apple_id: str = key_from_remote_env("AC_USERNAME"),
+    password: str = key_from_remote_env("AC_PASSWORD"),
+    team_id: str = key_from_remote_env("QT_CODESIGN_IDENTITY_KEY"),
+    key: str = key_from_remote_env("AC_KEY"),
+    key_id: str = key_from_remote_env("AC_KEYID"),
+    issuer: str = key_from_remote_env("AC_ISSUER"),
+    profile: str = key_from_remote_env("AC_NOTARY") or "AC_NOTARY",
+    timeout: Optional[int] = 60 * 60 * 3,
+    acceleration: bool = True
+) -> None:
+    """
+    Run notarize and staple actions for the given file with arguments specified
+    Notarytool authentication options:
+        - apple_id, password, team_id -> saved to profile
+        - key, key_id, issuer -> saved to profile
+        - profile -> use from profile directly
+
+    Args:
+        path: The file system path to the file or folder to notarize
+        apple_id: The Apple ID login username to save
+        password: App-specific password to save
+        team_id: The Developer Team identifier to save
+        key: App Store Connect API key file system path to save
+        key_id: App Store Connect API Key ID to save
+        issuer: App Store Connect API Issuer UUID to save
+        profile: Profile name in keychain for saving and accessing credentials
+        timeout: Timeout for all of the notarytool commands executed
+        acceleration: Whether to enable transfer acceleration in notarytool uploads
+
+    Raises:
+        NotarizationError: When conditions required for notarization are not met or it fails
+
+    """
+    # Check system requirements
+    check_notarize_reqs()
+    # Store credentials for later
+    if not profile:
+        raise NotarizationError("Keychain profile name is empty?")
+    if key and key_id and issuer:
+        store_credentials_api_key(profile, key, key_id, issuer)
+    elif apple_id and password and team_id:
+        store_credentials_apple_id(profile, apple_id, password, team_id)
+    else:
+        log.warning("App Store Connect API keys or Apple ID credentials not provided.")
+        log.info("Attempting to use previously saved credentials from profile '%s'", profile)
+    # Pack the file if necessary, return the new path
+    notarize_path = prepare_payload(path)
+    # Submit payload for notarization and wait for it to complete
+    log.info("Notarize %s", notarize_path)
+    submit_for_notarization(notarize_path, profile, timeout, acceleration)
+    # Embed the notarization ticket to the file if file type supports it (DMG, PKG)
+    # If required, stapler needs to be run separately for each file inside the ZIP archive
+    if notarize_path.suffix != ".zip":
+        embed_notarization(notarize_path)
+    # Remove the zipped archive as this is no longer needed, keeping only the original data
+    if path != notarize_path:
+        notarize_path.unlink()
 
-    asyncio_run(
+
+def main() -> None:
+    """Main function, parse arguments with ArgumentParser and call the notarize function."""
+    parser = argparse.ArgumentParser(prog="Helper script to notarize content from given path")
+    parser.add_argument(
+        "--bundle-id",  # TODO: remove
+        dest="bundle_id", required=False, type=str, help="Deprecated"
+    )
+    parser.add_argument(
+        "--path",
+        "--dmg",  # TODO: remove
+        dest="path", required=True, type=str, help="Path to a file or a folder"
+    )
+    parser.add_argument(
+        "--apple-id",
+        "--user",  # TODO: remove
+        dest="apple_id", type=str, default=key_from_remote_env("AC_USERNAME"),
+        help="Developer Apple ID login username"
+    )
+    parser.add_argument(
+        "--password",
+        "--passwd",  # TODO: remove
+        dest="password", type=str, default=key_from_remote_env("AC_PASSWORD"),
+        help="App-specific password for Apple ID"
+    )
+    parser.add_argument(
+        "--team-id", dest="team_id", type=str,
+        default=key_from_remote_env("QT_CODESIGN_IDENTITY_KEY"), help="Developer Team identifier"
+    )
+    parser.add_argument(
+        "--key", dest="key", type=str, default=key_from_remote_env("AC_KEY"),
+        help="Path for App Store Connect API key"
+    )
+    parser.add_argument(
+        "--key-id", dest="key_id", type=str, default=key_from_remote_env("AC_KEYID"),
+        help="App Store Connect API Key ID"
+    )
+    parser.add_argument(
+        "--issuer", dest="issuer", type=str, default=key_from_remote_env("AC_ISSUER"),
+        help="App Store Connect API Issuer UUID"
+    )
+    parser.add_argument(
+        "--timeout", dest="timeout", type=int, default=10800,
+        help="Timeout value for remote requests"
+    )
+    parser.add_argument(
+        "--profile", dest="profile", type=str,
+        default=key_from_remote_env("AC_NOTARY") or "AC_NOTARY",
+        help="Notarytool profile name for saved credentials"
+    )
+    parser.add_argument(
+        "--acceleration", dest="acceleration", action="store_true", default=True,
+        help="Enable S3 Acceleration"
+    )
+    args = parser.parse_args(sys.argv[1:])
+    try:
         notarize(
-            dmg=args.dmg,
-            user=args.user,
-            passwd=args.passwd,
-            bundle_id=args.bundle_id,
+            path=Path(args.path),
+            apple_id=args.apple_id,
+            password=args.password,
+            team_id=args.team_id,
+            key=args.key,
+            key_id=args.key_id,
+            issuer=args.issuer,
             timeout=args.timeout,
+            profile=args.profile,
+            acceleration=args.acceleration,
         )
-    )
+    except NotarizationError as err:
+        log.exception("Notarize script failed: %s", str(err), exc_info=err)
+        raise SystemExit from err
 
 
 if __name__ == "__main__":
diff --git a/packaging-tools/release_repo_updater.py b/packaging-tools/release_repo_updater.py
index febfe17d6..980ba88e5 100755
--- a/packaging-tools/release_repo_updater.py
+++ b/packaging-tools/release_repo_updater.py
@@ -701,18 +701,11 @@ async def sign_offline_installer(installer_path: str, installer_name: str) -> No
         log.info("Create macOS dmg file")
         create_mac_dmg(os.path.join(installer_path, installer_name) + '.app')
         log.info("Notarize macOS installer")
-        await notarize_dmg(os.path.join(installer_path, installer_name + '.dmg'), installer_name)
+        notarize(path=Path(installer_path, installer_name + '.dmg'))
     else:
         log.info("No signing available for this host platform: %s", platform.system())
 
 
-async def notarize_dmg(dmg_path: str, installer_basename: str) -> None:
-    # this is just a unique id without any special meaning, used to track the notarization progress
-    bundle_id = installer_basename + "-" + strftime('%Y-%m-%d-%H-%M', gmtime())
-    bundle_id = bundle_id.replace('_', '-').replace(' ', '')  # replace illegal chars for bundle_id
-    await notarize(dmg=dmg_path, bundle_id=bundle_id)
-
-
 async def build_offline_tasks(staging_server: str, staging_server_root: str, tasks: List[ReleaseTask], license_: str,
                               installer_config_base_dir: str, artifact_share_base_url: str,
                               ifw_tools: str, installer_build_id: str, update_staging: bool,