Refactor partial installer and dry-run options

Add --allow-broken-components option for partial installer creation.
Skip downloading ifw tools + payload, online repo creation in dry run.
Add more generic uri_exists function to replace is_content_url_valid.

Add two dry run modes (--dry-run=configs|payload):
- Config validation mode: check only the configuration for errors
- Payload URI validation mode: in addition to the above,
checks if payload archive uris exist on the server (or file system)

Make these dry run options available in release_repo_updater.py and
create_installer.py with a separate parser.

Change-Id: Ie77cf57cbae3302ed3b9f7cd5d3a61351f0e406d
Reviewed-by: Iikka Eklund <iikka.eklund@qt.io>

6 files changed, 231 insertions, 100 deletions

diff --git a/Pipfile b/Pipfile
index f7ea59c11..3ae452e1d 100644
--- a/Pipfile
+++ b/Pipfile
@@ -17,6 +17,7 @@ pywin32 = {version = "==303", os_name = "== 'nt'"}
 configparser = "==5.2.0"
 dataclasses = {version = "==0.8", markers="(python_version >= '3.6' and python_version < '3.7')"}
 asyncio-backport = {version = "==0.1.1", markers="(python_version >= '3.6' and python_version < '3.7')"}
+requests = "==2.27.1"
 
 [dev-packages]
 ddt = "==1.6.0"
@@ -32,6 +33,7 @@ pep8-naming = "==0.13.1"
 types-dataclasses = "==0.6.6"
 pytest = "==7.0.1"
 pytest-xdist = "==2.5.0"
+types-requests = "==2.28.2"
 
 [requires]
 python_version = "3"
diff --git a/packaging-tools/bld_ifw_tools.py b/packaging-tools/bld_ifw_tools.py
index 3e53fe506..48fbf9876 100755
--- a/packaging-tools/bld_ifw_tools.py
+++ b/packaging-tools/bld_ifw_tools.py
@@ -45,12 +45,12 @@ from bldinstallercommon import (
     clone_repository,
     extract_file,
     get_tag_from_branch,
-    is_content_url_valid,
     locate_executable,
     locate_path,
     move_tree,
     remove_tree,
     retrieve_url,
+    uri_exists,
 )
 from installer_utils import PackagingError, ch_dir
 from logging_util import init_logger
@@ -253,8 +253,7 @@ class IfwOptions:
 
     def sanity_check(self) -> None:
         # check qt src package url
-        res = is_content_url_valid(self.qt_source_package_uri)
-        if not res:
+        if not uri_exists(self.qt_source_package_uri):
             raise SystemExit(f"Qt src package uri is invalid: {self.qt_source_package_uri}")
         if self.product_key_checker_pri:
             if os.path.isfile(self.product_key_checker_pri):
@@ -354,7 +353,7 @@ def prepare_qt_sources(options: IfwOptions) -> None:
 def prepare_compressed_package(src_pkg_uri: str, src_pkg_saveas: str, destination_dir: str) -> None:
     log.info("Fetching package from: %s", src_pkg_uri)
     if not os.path.isfile(src_pkg_saveas):
-        if not is_content_url_valid(src_pkg_uri):
+        if not uri_exists(src_pkg_uri):
             raise SystemExit("Src package uri is invalid! Abort!")
         retrieve_url(src_pkg_uri, src_pkg_saveas)
     else:
diff --git a/packaging-tools/bldinstallercommon.py b/packaging-tools/bldinstallercommon.py
index a3b3f3860..5f21ec420 100644
--- a/packaging-tools/bldinstallercommon.py
+++ b/packaging-tools/bldinstallercommon.py
@@ -46,7 +46,9 @@ from traceback import print_exc
 from types import TracebackType
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 from urllib.parse import urlparse
-from urllib.request import urlcleanup, urlopen, urlretrieve
+from urllib.request import urlcleanup, urlretrieve
+
+import requests
 
 from bld_utils import download, is_linux, is_macos, is_windows, run_command
 from installer_utils import PackagingError
@@ -63,20 +65,31 @@ log = init_logger(__name__, debug_mode=False)
 MAX_DEBUG_PRINT_LENGTH = 10000
 
 
-###############################
-# function
-###############################
-def is_content_url_valid(url: str) -> bool:
+def uri_exists(uri: str) -> bool:
+    """
+    Check URI and return whether the location exists, log the errors if any
+    HTTP URLs will be validated using the response headers from a HEAD request
+    Timeout for remote requests is set to 30 seconds
+
+    Args:
+        uri: An URI pointing to a local file or a remote file (HTTP)
+
+    Returns:
+        True if the file exists at the given URI location, otherwise False
+    """
     # check first if the url points to file on local file system
-    if os.path.isfile(url):
+    if Path(uri.replace("file://", "")).resolve().is_file():
         return True
-    # throws error if url does not point to valid object
     try:
-        with urlopen(url) as response:
-            total_size = response.info().get('Content-Length').strip()
-            return int(total_size) > 0
-    except Exception:
-        pass
+        with requests.head(uri, timeout=30, stream=True) as res:
+            res.raise_for_status()
+            if int(res.headers["content-length"]) > 0:
+                return True
+            log.error("Invalid content length: %s", res.headers['content-length'])
+    except requests.exceptions.HTTPError as err:  # HTTP error status codes
+        log.exception("HTTP %s: %s", err.response.status_code, err.response.text)
+    except requests.exceptions.RequestException as err:  # Other errors
+        log.exception("Error while checking URI: %s", str(err))
     return False
 
 
@@ -628,7 +641,7 @@ def create_qt_download_task(module_urls: List[str], target_qt5_path: str, temp_p
     unzip_task = Task(f'extracting packages to "{target_qt5_path}"', function=None)
     # add Qt modules
     for module_url in module_urls:
-        if is_content_url_valid(module_url):
+        if uri_exists(module_url):
             (download_task, extract_task) = create_download_and_extract_tasks(
                 module_url, target_qt5_path, temp_path
             )
diff --git a/packaging-tools/create_installer.py b/packaging-tools/create_installer.py
index 49414e5ae..a0ecc314b 100644
--- a/packaging-tools/create_installer.py
+++ b/packaging-tools/create_installer.py
@@ -37,6 +37,7 @@ import sys
 from argparse import ArgumentParser, ArgumentTypeError
 from configparser import ConfigParser, ExtendedInterpolation
 from dataclasses import dataclass, field
+from enum import Enum
 from multiprocessing import cpu_count
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -48,7 +49,6 @@ from bldinstallercommon import (
     copy_tree,
     extract_file,
     handle_component_rpath,
-    is_content_url_valid,
     locate_executable,
     locate_path,
     locate_paths,
@@ -57,13 +57,14 @@ from bldinstallercommon import (
     replace_in_files,
     retrieve_url,
     safe_config_key_fetch,
+    uri_exists,
 )
 from installer_utils import PackagingError
 from logging_util import init_logger
 from patch_qt import patch_files, patch_qt_edition
 from pkg_constants import INSTALLER_OUTPUT_DIR_NAME, PKG_TEMPLATE_BASE_DIR_NAME
 from runner import run_cmd
-from sdkcomponent import IfwPayloadItem, IfwSdkComponent, IfwSdkError, parse_ifw_sdk_comp
+from sdkcomponent import IfwPayloadItem, IfwSdkComponent, parse_ifw_sdk_comp
 from threadedwork import ThreadedWork
 
 if is_windows():
@@ -89,6 +90,39 @@ class CreateInstallerError(Exception):
     pass
 
 
+class DryRunMode(Enum):
+    PAYLOAD = "payload"
+    CONFIGS = "configs"
+
+    @classmethod
+    def as_list(cls) -> List[str]:
+        """
+        Get the list of string mode names
+
+        Returns:
+            A list of supported values for dry run mode
+        """
+        return [mode.name for mode in cls]
+
+    @classmethod
+    def get_parser(cls) -> ArgumentParser:
+        """
+        Get a parser (without help) for specifying dry run modes
+
+        Returns:
+            An argparse.Argumentparser parser containing the --dry-run argument
+        """
+        choices = cls.as_list()
+        mode_parser = ArgumentParser(
+            prog=f"Test run w/o payload download, validation modes: {'|'.join(choices)}",
+            add_help=False
+        )
+        mode_parser.add_argument(
+            "--dry-run", dest="dry_run", type=str.upper, default=None, choices=choices
+        )
+        return mode_parser
+
+
 ##############################################################
 # Check that valid tools are present in the build environment
 ##############################################################
@@ -256,19 +290,17 @@ def parse_component_data(
                     substitutions=task.substitutions,
                     file_share_base_url=task.archive_base_url,
                 )
-                try:
-                    # Validate component
-                    sdk_comp.validate()
-                    # Skip archive download if dry run
-                    if task.dry_run:
-                        sdk_comp.archive_skip = True
-                except IfwSdkError as err:
-                    if not task.strict_mode:
-                        raise CreateInstallerError from err
-                    log.warning(
-                        "Skip invalid component (missing payload/metadata?): [%s]",
-                        sdk_comp.ifw_sdk_comp_name
-                    )
+                # validate the component
+                # - payload URIs are always checked when not in dry_run or when mode is 'payload'
+                # - errors are not raised in dry_run, so we are able to log all the errors at once
+                component_is_valid = sdk_comp.validate(
+                    uri_check=not task.dry_run or task.dry_run == DryRunMode.PAYLOAD,
+                    ignore_errors=bool(task.dry_run) or task.partial_installer,
+                )
+                # invalid components are skipped when in partial_installer mode
+                # all component data is skipped when a dry_run mode is specified
+                if (task.partial_installer and not component_is_valid) or task.dry_run:
+                    log.warning("Skipping component: [%s]", sdk_comp.ifw_sdk_comp_name)
                     sdk_comp.archive_skip = True
                 # if include filter defined for component it is included only if LICENSE_TYPE
                 # matches to include_filter
@@ -1061,7 +1093,8 @@ def create_installer(task: QtInstallerTaskType) -> None:
         set_config_directory(task)
         set_config_xml(task)
     # install Installer Framework tools
-    task.install_ifw_tools()
+    if not task.dry_run:
+        task.install_ifw_tools()
     # parse SDK components
     parse_components(task)
     # create components
@@ -1069,13 +1102,14 @@ def create_installer(task: QtInstallerTaskType) -> None:
     # substitute global tags
     substitute_global_tags(task)
     # create the installer binary
-    if task.online_installer or task.offline_installer:
-        create_installer_binary(task)
-        # for mac we need some extra work
-        if is_macos():
-            create_mac_disk_image(task)
-    if task.create_repository:
-        create_online_repository(task)
+    if not task.dry_run:
+        if task.online_installer or task.offline_installer:
+            create_installer_binary(task)
+            # for mac we need some extra work
+            if is_macos():
+                create_mac_disk_image(task)
+        if task.create_repository:
+            create_online_repository(task)
 
 
 def str2bool(value: str) -> bool:
@@ -1121,8 +1155,8 @@ class QtInstallerTask(Generic[QtInstallerTaskType]):
     offline_installer: bool = False
     online_installer: bool = False
     create_repository: bool = False
-    strict_mode: bool = True
-    dry_run: Optional[str] = None
+    partial_installer: bool = False  # Invalid IfwSdkComponents will be excluded from the installer
+    dry_run: Optional[DryRunMode] = None
     license_type: str = "opensource"
     build_timestamp: str = strftime("%Y-%m-%d", gmtime())
     force_version_number_increase: bool = False
@@ -1240,7 +1274,7 @@ class QtInstallerTask(Generic[QtInstallerTaskType]):
             # create needed dirs
             Path(self.ifw_tools_dir).mkdir(parents=True, exist_ok=True)
             log.info("Downloading: %s", self.ifw_tools_uri)
-            if not is_content_url_valid(self.ifw_tools_uri):
+            if not uri_exists(self.ifw_tools_uri):
                 raise CreateInstallerError(f"Package URL is invalid: {self.ifw_tools_uri}")
             retrieve_url(self.ifw_tools_uri, package_save_as_temp)
             if not os.path.isfile(package_save_as_temp):
@@ -1252,7 +1286,10 @@ class QtInstallerTask(Generic[QtInstallerTaskType]):
 
 def main() -> None:
     """Main"""
-    parser = ArgumentParser(prog="Script to create Qt Installer Framework based installers.")
+    parser = ArgumentParser(
+        prog="Script to create Qt Installer Framework based installers.",
+        parents=[DryRunMode.get_parser()]  # add parser for --dry-run argument as parent
+    )
     parser.add_argument("-c", "--configurations-dir", dest="configurations_dir", type=str, default="configurations",
                         help="define configurations directory where to read installer configuration files")
     parser.add_argument("-f", "--configuration-file", dest="configuration_file", type=str,
@@ -1263,12 +1300,10 @@ def main() -> None:
                         help="Create online installer")
     parser.add_argument("-r", "--create-repo", dest="create_repository", action='store_true', default=False,
                         help="Create online repository")
-    parser.add_argument("-s", "--strict", dest="strict_mode", action='store_true', default=True,
-                        help="Use strict mode, abort on any error")
-    parser.add_argument("-S", "--non-strict", dest="strict_mode", action='store_true', default=False,
-                        help="Non strict mode, try to keep on going despite of errors")
-    parser.add_argument("--dry-run", dest="dry_run", action='store_true', default=False,
-                        help="For testing purposes (faster testing), skip downloading archives")
+    parser.add_argument(
+        "--allow-broken-components", dest="partial_installer", action="store_true", default=False,
+        help="Skip invalid components and missing payload, create a partial installer",
+    )
 
     parser.add_argument("-u", "--archive-base-url", dest="archive_base_url", type=str,
                         help="Define server base url where to look for archives (.7z)")
@@ -1313,8 +1348,8 @@ def main() -> None:
         offline_installer=args.offline_installer,
         online_installer=args.online_installer,
         create_repository=args.create_repository,
-        strict_mode=args.strict_mode,
-        dry_run=args.dry_run,
+        partial_installer=args.partial_installer,
+        dry_run=DryRunMode[args.dry_run] if args.dry_run else None,
         archive_base_url=args.archive_base_url,
         ifw_tools_uri=args.ifw_tools_uri,
         license_type=args.license_type,
diff --git a/packaging-tools/release_repo_updater.py b/packaging-tools/release_repo_updater.py
index 4b50fbe4a..8b477ad8a 100755
--- a/packaging-tools/release_repo_updater.py
+++ b/packaging-tools/release_repo_updater.py
@@ -52,7 +52,7 @@ from urllib.request import urlopen, urlretrieve
 
 from bld_utils import is_linux
 from bldinstallercommon import locate_path
-from create_installer import QtInstallerTask, create_installer
+from create_installer import DryRunMode, QtInstallerTask, create_installer
 from installer_utils import PackagingError, download_archive, extract_archive, is_valid_url_path
 from logging_util import init_logger
 from notarize import notarize
@@ -492,8 +492,15 @@ async def update_repository(staging_server: str, update_strategy: RepoUpdateStra
         trigger_rta(rta, task)
 
 
-async def build_online_repositories(tasks: List[ReleaseTask], license_: str, installer_config_base_dir: str, artifact_share_base_url: str,
-                                    ifw_tools: str, build_repositories: bool) -> List[str]:
+async def build_online_repositories(
+    tasks: List[ReleaseTask],
+    license_: str,
+    installer_config_base_dir: str,
+    artifact_share_base_url: str,
+    ifw_tools: str,
+    build_repositories: bool,
+    dry_run: Optional[DryRunMode] = None,
+) -> List[str]:
     log.info("Building online repositories: %i", len(tasks))
     # create base tmp dir
     tmp_base_dir = os.path.join(os.getcwd(), "_repo_update_jobs")
@@ -535,6 +542,7 @@ async def build_online_repositories(tasks: List[ReleaseTask], license_: str, ins
             force_version_number_increase=True,
             substitution_list=task.get_installer_string_replacement_list(),
             build_timestamp=job_timestamp,
+            dry_run=dry_run,
         )
         log.info(str(installer_task))
         try:
@@ -545,12 +553,13 @@ async def build_online_repositories(tasks: List[ReleaseTask], license_: str, ins
         except Exception as exc:
             log.exception("Repository build failed!")
             raise PackagingError from exc
-
-        online_repository_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "online_repository"))
-        assert os.path.isdir(online_repository_path), f"Not a valid path: {online_repository_path}"
-        shutil.move(online_repository_path, task.source_online_repository_path)
-        log.info("Repository created at: %s", task.source_online_repository_path)
-        done_repositories.append(task.source_online_repository_path)
+        if not dry_run:
+            script_dir = os.path.dirname(__file__)
+            online_repo_path = os.path.abspath(os.path.join(script_dir, "online_repository"))
+            assert os.path.isdir(online_repo_path), f"Not a valid path: {online_repo_path}"
+            shutil.move(online_repo_path, task.source_online_repository_path)
+            log.info("Repository created at: %s", task.source_online_repository_path)
+            done_repositories.append(task.source_online_repository_path)
     return done_repositories
 
 
@@ -592,20 +601,38 @@ async def sync_production(tasks: List[ReleaseTask], repo_layout: QtRepositoryLay
     log.info("Production sync trigger done!")
 
 
-async def handle_update(staging_server: str, staging_server_root: str, license_: str, tasks: List[ReleaseTask],
-                        installer_config_base_dir: str, artifact_share_base_url: str,
-                        update_strategy: RepoUpdateStrategy,
-                        sync_s3: str, sync_ext: str, rta: str, ifw_tools: str,
-                        build_repositories: bool, sync_repositories: bool,
-                        event_injector: str, export_data: Dict[str, str]) -> None:
+async def handle_update(
+    staging_server: str,
+    staging_server_root: str,
+    license_: str,
+    tasks: List[ReleaseTask],
+    installer_config_base_dir: str,
+    artifact_share_base_url: str,
+    update_strategy: RepoUpdateStrategy,
+    sync_s3: str,
+    sync_ext: str,
+    rta: str,
+    ifw_tools: str,
+    build_repositories: bool,
+    sync_repositories: bool,
+    event_injector: str,
+    export_data: Dict[str, str],
+    dry_run: Optional[DryRunMode] = None,
+) -> None:
     """Build all online repositories, update those to staging area and sync to production."""
     log.info("Starting repository update for %i tasks..", len(tasks))
     if build_repositories:
         # this may take a while depending on how big the repositories are
         async with EventRegister(f"{license_}: repo build", event_injector, export_data):
-            await build_online_repositories(tasks, license_, installer_config_base_dir,
-                                            artifact_share_base_url, ifw_tools,
-                                            build_repositories)
+            await build_online_repositories(
+                tasks,
+                license_,
+                installer_config_base_dir,
+                artifact_share_base_url,
+                ifw_tools,
+                build_repositories,
+                dry_run,
+            )
     if update_strategy.requires_remote_update():
         async with EventRegister(f"{license_}: repo update", event_injector, export_data):
             await update_repositories(tasks, staging_server, update_strategy, rta)
@@ -706,18 +733,42 @@ async def sign_offline_installer(installer_path: str, installer_name: str) -> No
         log.info("No signing available for this host platform: %s", platform.system())
 
 
-async def build_offline_tasks(staging_server: str, staging_server_root: str, tasks: List[ReleaseTask], license_: str,
-                              installer_config_base_dir: str, artifact_share_base_url: str,
-                              ifw_tools: str, installer_build_id: str, update_staging: bool,
-                              enable_oss_snapshots: bool, event_injector: str, export_data: Dict[str, str]) -> None:
+async def build_offline_tasks(
+    staging_server: str, staging_server_root: str, tasks: List[ReleaseTask], license_: str,
+    installer_config_base_dir: str, artifact_share_base_url: str,
+    ifw_tools: str, installer_build_id: str, update_staging: bool,
+    enable_oss_snapshots: bool, event_injector: str, export_data: Dict[str, str],
+    dry_run: Optional[DryRunMode] = None
+) -> None:
     async with EventRegister(f"{license_}: offline", event_injector, export_data):
-        await _build_offline_tasks(staging_server, staging_server_root, tasks, license_, installer_config_base_dir,
-                                   artifact_share_base_url, ifw_tools, installer_build_id, update_staging, enable_oss_snapshots)
+        await _build_offline_tasks(
+            staging_server,
+            staging_server_root,
+            tasks,
+            license_,
+            installer_config_base_dir,
+            artifact_share_base_url,
+            ifw_tools,
+            installer_build_id,
+            update_staging,
+            enable_oss_snapshots,
+            dry_run,
+        )
 
 
-async def _build_offline_tasks(staging_server: str, staging_server_root: str, tasks: List[ReleaseTask], license_: str,
-                               installer_config_base_dir: str, artifact_share_base_url: str,
-                               ifw_tools: str, installer_build_id: str, update_staging: bool, enable_oss_snapshots: bool) -> None:
+async def _build_offline_tasks(
+    staging_server: str,
+    staging_server_root: str,
+    tasks: List[ReleaseTask],
+    license_: str,
+    installer_config_base_dir: str,
+    artifact_share_base_url: str,
+    ifw_tools: str,
+    installer_build_id: str,
+    update_staging: bool,
+    enable_oss_snapshots: bool,
+    dry_run: Optional[DryRunMode] = None,
+) -> None:
     log.info("Offline installer task(s): %i", len(tasks))
 
     assert license_, "The 'license_' must be defined!"
@@ -751,6 +802,7 @@ async def _build_offline_tasks(staging_server: str, staging_server_root: str, ta
             force_version_number_increase=True,
             substitution_list=task.get_installer_string_replacement_list(),
             build_timestamp=job_timestamp,
+            dry_run=dry_run,
         )
         log.info(str(installer_task))
         try:
@@ -837,22 +889,25 @@ def main() -> None:
         if config.has_section("script.default_args"):
             defaults.update(dict(config.items("script.default_args")))
 
-    parser = argparse.ArgumentParser(description="Script to update release repositories",
-                                     usage="To build linux android only repositories from given config file and update staging "
-                                           "and production repositories and sync to S3 distribution:\n\n"
-                                           "release_repo_updater.py --ifw-tools=<7z URL> "
-                                           "--staging-server=username@server.com "
-                                           "--staging-server-root=/base/path/for/online/repositories "
-                                           "--config=<path to top level config .ini file> "
-                                           "--task-filter=linux,x64,common --task-filter=linux,x64,opensource "
-                                           "--artifacts-share-url=<http(s)://some.server.com/base/path/for/artifacts> "
-                                           "--license=opensource "
-                                           "--repo-domain=qtsdkrepository "
-                                           "--build-repositories "
-                                           "--update-staging "
-                                           "--update-production "
-                                           "--rta=<RTA trigger base URL> "
-                                           "--sync-s3=<S3 bucket address> ")
+    parser = argparse.ArgumentParser(
+        description="Script to update release repositories",
+        usage="To build linux android only repositories from given config file and update staging "
+              "and production repositories and sync to S3 distribution:\n\n"
+              "release_repo_updater.py --ifw-tools=<7z URL> "
+              "--staging-server=username@server.com "
+              "--staging-server-root=/base/path/for/online/repositories "
+              "--config=<path to top level config .ini file> "
+              "--task-filter=linux,x64,common --task-filter=linux,x64,opensource "
+              "--artifacts-share-url=<http(s)://some.server.com/base/path/for/artifacts> "
+              "--license=opensource "
+              "--repo-domain=qtsdkrepository "
+              "--build-repositories "
+              "--update-staging "
+              "--update-production "
+              "--rta=<RTA trigger base URL> "
+              "--sync-s3=<S3 bucket address> ",
+        parents=[DryRunMode.get_parser()]  # add parser for --dry-run argument as parent
+    )
     parser.add_argument("--ifw-tools", dest="ifw_tools", type=str, default=os.getenv("IFW_TOOLS"),
                         help="Archive which contains the ifw tools")
 
@@ -906,7 +961,11 @@ def main() -> None:
                              "The --config file must point to export summary file.")
     parser.set_defaults(**defaults)  # these are from provided --config file
     args = parser.parse_args(sys.argv[1:])
-
+    if args.dry_run:
+        assert not args.sync_s3, "'--sync-s3' specified with '--dry-run'"
+        assert not args.sync_ext, "'--sync-ext' specified with '--dry-run'"
+        assert not args.update_staging, "'--update-staging' specified with '--dry-run'"
+        assert not args.update_production, "'--update-production' specified with '--dry-run'"
     assert args.config, "'--config' was not given!"
     assert args.staging_server_root, "'--staging-server-root' was not given!"
 
@@ -945,6 +1004,7 @@ def main() -> None:
                 enable_oss_snapshots=args.enable_oss_snapshots,
                 event_injector=args.event_injector,
                 export_data=export_data,
+                dry_run=DryRunMode[args.dry_run] if args.dry_run else None,
             )
         )
 
@@ -980,6 +1040,7 @@ def main() -> None:
                 sync_repositories=args.sync_s3 or args.sync_ext,
                 event_injector=args.event_injector,
                 export_data=export_data,
+                dry_run=DryRunMode[args.dry_run] if args.dry_run else None,
             )
         )
 
diff --git a/packaging-tools/sdkcomponent.py b/packaging-tools/sdkcomponent.py
index ed86a912f..a57d0b2f5 100644
--- a/packaging-tools/sdkcomponent.py
+++ b/packaging-tools/sdkcomponent.py
@@ -36,6 +36,7 @@ from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 from urllib.parse import urlparse
 
+from bldinstallercommon import uri_exists
 from logging_util import init_logger
 
 log = init_logger(__name__, debug_mode=False)
@@ -109,6 +110,12 @@ class IfwPayloadItem:
                 f"[[{self.package_name}]] Invalid payload configuration - check your configs!"
             )
 
+    def validate_uri(self) -> None:
+        """Validate that the uri location exists either on the file system or online"""
+        log.info("[%s] Checking payload uri: %s", self.package_name, self.archive_uri)
+        if not uri_exists(self.archive_uri):
+            raise IfwSdkError(f"[{self.package_name}] Missing payload {self.archive_uri}")
+
     def _ensure_ifw_archive_name(self) -> str:
         """
         Get the archive name by splitting from its uri if a name doesn't already exist
@@ -253,17 +260,31 @@ class IfwSdkComponent:
         if self.comp_sha1_uri:
             self.comp_sha1_uri = self.archive_resolver.resolve_payload_uri(self.comp_sha1_uri)
 
-    def validate(self) -> None:
+    def validate(self, uri_check: bool = True, ignore_errors: bool = False) -> bool:
         """
         Perform validation on IfwSdkComponent, raise error if component not valid
 
+        Args:
+            uri_check: Whether to check that component's payload URIs are available
+            ignore_errors: Does not raise caught errors
+
         Raises:
             AssertionError: When the component's package name doesn't exist
             IfwSdkError: When component with payload doesn't have target install base configured
         """
-        assert self.ifw_sdk_comp_name, "Undefined package name?"
-        if self.downloadable_archives and not self.target_install_base:
-            raise IfwSdkError(f"[{self.ifw_sdk_comp_name}] is missing 'target_install_base'")
+        try:
+            assert self.ifw_sdk_comp_name, "Undefined package name?"
+            if self.downloadable_archives and not self.target_install_base:
+                raise IfwSdkError(f"[{self.ifw_sdk_comp_name}] is missing 'target_install_base'")
+            if uri_check:
+                for archive in self.downloadable_archives:
+                    archive.validate_uri()
+            return True
+        except IfwSdkError as err:
+            if not ignore_errors:
+                raise
+            log.exception("[%s] Ignored error in component: %s", self.ifw_sdk_comp_name, err)
+        return False
 
     def generate_downloadable_archive_list(self) -> List[List[str]]:
         """