sdkcomponent: Compress to a single archive when using pattern match uri

Change behavior when using pattern match in payload uri:
- Do not support extract operation with pattern matching
- Compress all matching content to a single payload archive
- Include source directory structure in the final archive to avoid name
conflicts
- Require to declare "archive_name" in configs if using patterns
- Adjust create_installer, unit tests

Change-Id: I7b89010d1d872f8b3bff8e1d496147675a9a6df1
Reviewed-by: Jani Heikkinen <jani.heikkinen@qt.io>

3 files changed, 124 insertions, 91 deletions

diff --git a/packaging-tools/create_installer.py b/packaging-tools/create_installer.py
index c80121555..49c817bac 100644
--- a/packaging-tools/create_installer.py
+++ b/packaging-tools/create_installer.py
@@ -44,6 +44,7 @@ from time import gmtime, strftime
 from typing import Any, Dict, Generator, Generic, List, Optional, Tuple, TypeVar
 
 from temppathlib import TemporaryDirectory
+from urlpath import URL  # type: ignore
 
 from bld_utils import download, is_linux, is_macos, is_windows
 from bldinstallercommon import (
@@ -611,40 +612,50 @@ def get_component_data(
         data_dir_dest: A directory location for the final component data
         compress_dir: A directory containing the items to compress to the final archive
     """
-    # Continue if payload item has no data (final uri part missing)
-    download_name = Path(archive.archive_uri).name
-    if not download_name:
-        log.info("[%s] Payload item has no data", archive.package_name)
-        return
-    # Download payload to a temporary directory to avoid naming clashes
-    with TemporaryDirectory() as temp_dir:
-        download_path = temp_dir.path / download_name
-        log.info("[%s] Download: %s", archive.package_name, download_name)
-        download(archive.archive_uri, str(download_path))
-        # For non-archive payload and non-extractable archives, move to install_dir for packing
-        if (
-            download_path.suffix not in archive.supported_arch_formats
-            or archive.disable_extract_archive is True
-        ):
-            shutil.move(str(download_path), install_dir)
-        # For payload already in IFW compatible format, use the raw artifact and continue
-        elif not archive.requires_extraction and download_path.suffix in archive.ifw_arch_formats:
-            # Save to data dir as archive_name
-            log.info(
-                "[%s] Rename raw artifact to final archive name: %s -> %s",
-                archive.package_name, download_path.name, archive.archive_name
-            )
-            if download_path.suffix != Path(archive.archive_name).suffix:
-                log.warning(
-                    "Raw artifact saved with a different suffix: %s -> %s",
-                    download_path.suffix, Path(archive.archive_name).suffix
-                )
-            shutil.move(str(download_path), str(data_dir_dest / archive.archive_name))
+    # Handle pattern match payload URIs for IfwPayloadItem
+    if archive.payload_base_uri:
+        for payload_uri in archive.payload_uris:
+            # Get download path relative to the base URI, keeping the original structure
+            dl_name = str(URL(payload_uri).relative_to(URL(archive.payload_base_uri)))
+            # Download to install dir with the correct paths
+            dl_path = Path(install_dir, dl_name)
+            log.info("[%s] Download: %s", archive.package_name, dl_name)
+            download(payload_uri, str(dl_path))
+    else:
+        # Continue if payload item has no data (final uri part missing)
+        download_name = Path(archive.payload_uris[0]).name
+        if not download_name:
+            log.info("[%s] Payload item has no data", archive.package_name)
             return
-        # Extract payload archive when required to be patched or recompressed to compatible format
-        else:
-            log.info("[%s] Extract: %s", archive.package_name, archive.archive_name)
-            extract_component_data(download_path, install_dir)
+        # Download payload to a temporary directory to avoid naming clashes
+        with TemporaryDirectory() as temp_dir:
+            dl_path = temp_dir.path / download_name
+            log.info("[%s] Download: %s", archive.package_name, download_name)
+            download(archive.payload_uris[0], str(dl_path))
+            # For non-archive payload and non-extractable archives, move to install_dir for packing
+            if (
+                dl_path.suffix not in archive.supported_arch_formats
+                or archive.disable_extract_archive is True
+            ):
+                shutil.move(str(dl_path), install_dir)
+            # For payload already in IFW compatible format, use the raw artifact and continue
+            elif not archive.requires_extraction and dl_path.suffix in archive.ifw_arch_formats:
+                # Save to data dir as archive_name
+                log.info(
+                    "[%s] Rename raw artifact to final archive name: %s -> %s",
+                    archive.package_name, dl_path.name, archive.archive_name
+                )
+                if dl_path.suffix != Path(archive.archive_name).suffix:
+                    log.warning(
+                        "Raw artifact saved with a different suffix: %s -> %s",
+                        dl_path.suffix, Path(archive.archive_name).suffix
+                    )
+                shutil.move(str(dl_path), str(data_dir_dest / archive.archive_name))
+                return
+            # Extract payload when required to be patched or recompressed to compatible format
+            else:
+                log.info("[%s] Extract: %s", archive.package_name, archive.archive_name)
+                extract_component_data(dl_path, install_dir)
     # If patching items are specified, execute them here
     if archive.requires_patching:
         log.info("[%s] Patch: %s", archive.package_name, archive.archive_name)
diff --git a/packaging-tools/sdkcomponent.py b/packaging-tools/sdkcomponent.py
index ca5724a3a..6e9f4decc 100644
--- a/packaging-tools/sdkcomponent.py
+++ b/packaging-tools/sdkcomponent.py
@@ -63,7 +63,7 @@ class IfwPayloadItem:
     """Payload item class for IfwSdkComponent's archives"""
 
     package_name: str
-    archive_uri: str
+    payload_uris: List[str]
     archive_action: Optional[Tuple[Path, str]]
     disable_extract_archive: bool
     package_strip_dirs: int
@@ -74,6 +74,7 @@ class IfwPayloadItem:
     rpath_target: str
     component_sha1: str
     archive_name: str
+    payload_base_uri: str = field(default_factory=str)
     errors: List[str] = field(default_factory=list)
     # List of archive formats supported by Installer Framework:
     ifw_arch_formats: Tuple[str, ...] = (".7z", ".tar", ".gz", ".zip", ".xz", ".bz2")
@@ -94,15 +95,15 @@ class IfwPayloadItem:
             script_path, _ = self.archive_action
             if not script_path.exists() and script_path.is_file():
                 self.errors += [f"Unable to locate custom archive action script: {script_path}"]
-        if not self.archive_uri:
-            self.errors += [f"[{self.package_name}] is missing 'archive_uri'"]
-        if self.package_strip_dirs is None:
-            self.errors += [f"[{self.package_name}] is missing 'package_strip_dirs'"]
-        if not self.archive_uri.endswith(self.supported_arch_formats) and self.requires_patching:
+        if not self.payload_uris:
+            self.errors += [f"[{self.package_name}] payload contains no payload URIs"]
+        if not self.payload_base_uri and not self.payload_uris[0].endswith(
+            self.supported_arch_formats
+        ):
             if self.package_strip_dirs != 0:
-                self.errors += [f"[{self.package_name}] package_strip_dirs!=0 for a non-archive"]
+                self.errors += [f"[{self.package_name}] strip dirs set for a non-archive"]
             if self.package_finalize_items:
-                self.errors += [f"[{self.package_name}] package_finalize_items for a non-archive"]
+                self.errors += [f"[{self.package_name}] finalize items set for a non-archive"]
 
     def validate(self) -> None:
         """
@@ -122,18 +123,28 @@ class IfwPayloadItem:
 
     def validate_uri(self) -> None:
         """Validate that the uri location exists either on the file system or online"""
-        log.info("[%s] Checking payload uri: %s", self.package_name, self.archive_uri)
-        if not uri_exists(self.archive_uri):
-            raise IfwSdkError(f"[{self.package_name}] Missing payload {self.archive_uri}")
+        if self.payload_base_uri:
+            log.info("[%s] Skip checking already resolved uris", self.package_name)
+            return
+        log.info("[%s] Checking payload uri: %s", self.package_name, self.payload_uris[0])
+        if not uri_exists(self.payload_uris[0]):
+            raise IfwSdkError(f"[{self.package_name}] Missing payload {self.payload_uris[0]}")
 
     def _ensure_ifw_archive_name(self) -> str:
         """
         Get the archive name by splitting from its uri if a name doesn't already exist
+        When using pattern match uri, 'archive_name' must be specified in config
 
         Returns:
             Name for the payload item
+        Raises:
+            IfwSdkError: When 'archive_name' is missing and can't be resolved from URI
         """
-        archive_name: str = self.archive_name or Path(self.archive_uri).name
+        archive_name = self.archive_name
+        if not self.payload_base_uri and not self.archive_name:
+            archive_name = Path(self.payload_uris[0]).name
+        if not archive_name:
+            raise IfwSdkError(f"[{self.package_name}] is missing 'archive_name'")
         # make sure final archive format is supported by IFW (default: .7z)
         if not archive_name.endswith(self.ifw_arch_formats):
             archive_name += ".7z"
@@ -181,14 +192,17 @@ class IfwPayloadItem:
             A boolean for whether extracting the payload is needed.
         """
         if self._requires_extraction is None:
-            if self.archive_uri.endswith(self.ifw_arch_formats):
+            if self.payload_base_uri:
+                # Content from pattern match tree does not support extract
+                self._requires_extraction = False
+            elif self.payload_uris[0].endswith(self.ifw_arch_formats):
                 # Extract IFW supported archives if patching required or archive has a sha1 file
                 # Otherwise, use the raw CI artifact
                 self._requires_extraction = bool(self.component_sha1) or self.requires_patching
                 # If archive extraction is disabled, compress as-is (disable_extract_archive=False)
                 if self.disable_extract_archive:
                     self._requires_extraction = False
-            elif self.archive_uri.endswith(self.supported_arch_formats):
+            elif self.payload_uris[0].endswith(self.supported_arch_formats):
                 # Repack supported archives to IFW friendly archive format
                 self._requires_extraction = True
             else:
@@ -199,7 +213,8 @@ class IfwPayloadItem:
     def __str__(self) -> str:
         return f"""
 - Final payload archive name: {self.archive_name}
-  Source payload URI: {self.archive_uri}
+  Source payload URIs: {f'(Base: {self.payload_base_uri})' if self.payload_base_uri else ''}
+    {f'{os.linesep}    '.join(self.payload_uris)}
   Extract archive: {self.requires_extraction}
   Patch payload: {self.requires_patching}""" + (
             f""", config:
@@ -242,7 +257,7 @@ class ArchiveResolver:
         log.info("Crawl: %s", url)
         return await loop.run_in_executor(None, htmllistparse.fetch_listing, url, 30)
 
-    async def resolve_uri_pattern(self, pattern: str, base_url: Optional[URL] = None) -> List[URL]:
+    async def resolve_uri_pattern(self, pattern: str, base_url: Optional[URL] = None) -> Tuple[URL, List[URL]]:
         """
         Return payload URIs from remote tree, fnmatch pattern match for given arguments.
         Patterns will match arbitrary number of '/' allowing recursive search.
@@ -274,11 +289,11 @@ class ArchiveResolver:
         # recursively look for pattern matches inside the matching child directories
         coros = [self.resolve_uri_pattern(pattern, url) for url in child_list]
         results = await asyncio.gather(*coros)
-        for item in results:
+        for _, item in results:
             uri_list.extend(item)
-        return uri_list
+        return base_url, uri_list
 
-    def resolve_payload_uri(self, unresolved_archive_uri: str) -> List[str]:
+    def resolve_payload_uri(self, unresolved_archive_uri: str) -> Tuple[str, List[str]]:
         """
         Resolves the given archive URI and resolves it based on the type of URI given
         Available URI types, in the order of priority:
@@ -291,21 +306,23 @@ class ArchiveResolver:
             unresolved_archive_uri: Original URI to resolve
 
         Returns:
-            A resolved URI location for the payload
+            A base URI for the pattern match payload
+            A list containing resolved URI location(s) for the payload
         """
         # is it a URL containing a fnmatch pattern
         if any(char in unresolved_archive_uri for char in ("*", "[", "]", "?")):
             pattern = self.absolute_url(unresolved_archive_uri)
-            return [str(url) for url in asyncio_run(self.resolve_uri_pattern(pattern))]
+            payload_base_uri, url_list = asyncio_run(self.resolve_uri_pattern(pattern))
+            return str(payload_base_uri), [str(url) for url in url_list]
         # is it a file system path or an absolute URL which can be downloaded
         if os.path.exists(unresolved_archive_uri) or URL(unresolved_archive_uri).netloc:
-            return [unresolved_archive_uri]
+            return "", [unresolved_archive_uri]
         # is it relative to pkg template root dir, under the 'data' directory
         pkg_data_dir = os.path.join(self.pkg_template_folder, "data", unresolved_archive_uri)
         if os.path.exists(pkg_data_dir):
-            return [pkg_data_dir]
+            return "", [pkg_data_dir]
         # ok, we assume this is a URL which can be downloaded
-        return [self.absolute_url(unresolved_archive_uri)]
+        return "", [self.absolute_url(unresolved_archive_uri)]
 
 
 @dataclass
@@ -331,7 +348,7 @@ class IfwSdkComponent:
     def __post_init__(self) -> None:
         """Post init: resolve component sha1 uri if it exists"""
         if self.comp_sha1_uri:
-            match_uris = self.archive_resolver.resolve_payload_uri(self.comp_sha1_uri)
+            _, match_uris = self.archive_resolver.resolve_payload_uri(self.comp_sha1_uri)
             assert len(match_uris) == 1, f"More than one match for component sha: {match_uris}"
             self.comp_sha1_uri = match_uris.pop()
 
@@ -524,7 +541,7 @@ def parse_ifw_sdk_archives(
     for arch_section_name in archive_sections:
         config_subst = ConfigSubst(config, arch_section_name, substitutions)
         unresolved_archive_uri = config_subst.get("archive_uri")
-        resolved_uris = archive_resolver.resolve_payload_uri(unresolved_archive_uri)
+        base_uri, resolved_uris = archive_resolver.resolve_payload_uri(unresolved_archive_uri)
         archive_action_string = config_subst.get("archive_action", "")
         archive_action: Optional[Tuple[Path, str]] = None
         if archive_action_string:
@@ -547,21 +564,21 @@ def parse_ifw_sdk_archives(
             rpath_target = os.sep + rpath_target
         component_sha1_file = config_subst.get("component_sha1_file")
         archive_name = config_subst.get("archive_name")
-        for resolved_archive_uri in resolved_uris:
-            payload = IfwPayloadItem(
-                package_name=arch_section_name,
-                archive_uri=resolved_archive_uri,
-                archive_action=archive_action,
-                disable_extract_archive=disable_extract_archive,
-                package_strip_dirs=package_strip_dirs,
-                package_finalize_items=package_finalize_items,
-                parent_target_install_base=parent_target_install_base,
-                arch_target_install_base=target_install_base,
-                arch_target_install_dir=target_install_dir,
-                rpath_target=rpath_target,
-                component_sha1=component_sha1_file,
-                archive_name=archive_name,
-            )
-            payload.validate()
-            parsed_archives.append(payload)
+        payload = IfwPayloadItem(
+            package_name=arch_section_name,
+            payload_uris=resolved_uris,
+            archive_action=archive_action,
+            disable_extract_archive=disable_extract_archive,
+            package_strip_dirs=package_strip_dirs,
+            package_finalize_items=package_finalize_items,
+            parent_target_install_base=parent_target_install_base,
+            arch_target_install_base=target_install_base,
+            arch_target_install_dir=target_install_dir,
+            rpath_target=rpath_target,
+            component_sha1=component_sha1_file,
+            archive_name=archive_name,
+            payload_base_uri=base_uri,
+        )
+        payload.validate()
+        parsed_archives.append(payload)
     return parsed_archives
diff --git a/packaging-tools/tests/test_sdkcomponent.py b/packaging-tools/tests/test_sdkcomponent.py
index e535d599e..9d71dd6c3 100644
--- a/packaging-tools/tests/test_sdkcomponent.py
+++ b/packaging-tools/tests/test_sdkcomponent.py
@@ -168,17 +168,17 @@ class TestRunner(unittest.TestCase):
                     ]
                 ],
             )
-            self.assertEqual(
-                {a.archive_uri for a in comp.downloadable_archives},
-                {
+            self.assertCountEqual(
+                [a.payload_uris[0] for a in comp.downloadable_archives],
+                [
                     file_share_base_url + "qt/dev/release_content/qtbase/qtbase-RHEL_7_4.7z",
                     file_share_base_url + "qt/dev/release_content/qtsvg/qtsvg-RHEL_7_4.7z",
                     file_share_base_url + "opensource/5.10.0/foo/qtdeclarative-RHEL_7_4.7z",
-                },
+                ],
             )
-            self.assertEqual(
-                {a.archive_name for a in comp.downloadable_archives},
-                {"qtbase-RHEL_7_4.7z", "qtsvg-RHEL_7_4.7z", "qtdeclarative-RHEL_7_4.7z"},
+            self.assertCountEqual(
+                [a.archive_name for a in comp.downloadable_archives],
+                ["qtbase-RHEL_7_4.7z", "qtsvg-RHEL_7_4.7z", "qtdeclarative-RHEL_7_4.7z"],
             )
             for downloadable_archive in comp.downloadable_archives:
                 self.assertFalse(downloadable_archive.errors)
@@ -190,7 +190,7 @@ class TestRunner(unittest.TestCase):
         with self.assertRaises(AssertionError):
             IfwPayloadItem(
                 package_name="",
-                archive_uri="http://foo.com/readme.7z",
+                payload_uris=["http://foo.com/readme.7z"],
                 archive_action=None,
                 disable_extract_archive=False,
                 package_strip_dirs=0,
@@ -206,7 +206,7 @@ class TestRunner(unittest.TestCase):
     def test_ifw_payload_item_valid(self) -> None:
         item = IfwPayloadItem(
             package_name="foobar",
-            archive_uri="http://foo.com/readme.7z",
+            payload_uris=["http://foo.com/readme.7z"],
             archive_action=None,
             disable_extract_archive=False,
             package_strip_dirs=0,
@@ -253,7 +253,7 @@ class TestRunner(unittest.TestCase):
     ) -> None:
         item = IfwPayloadItem(
             package_name="foobar",
-            archive_uri=archive_uri,
+            payload_uris=[archive_uri],
             archive_action=archive_action,
             disable_extract_archive=disable_extract_archive,
             package_strip_dirs=package_strip_dirs,
@@ -278,12 +278,14 @@ class TestRunner(unittest.TestCase):
                 pass
 
             resolver = ArchiveResolver("http://intranet.local.it/artifacts", str(template_folder))
-            self.assertEqual(resolver.resolve_payload_uri("readme.txt").pop(), str(payload_file))
+            _, resolved = resolver.resolve_payload_uri("readme.txt")
+            self.assertEqual(resolved.pop(), str(payload_file))
+            _, resolved = resolver.resolve_payload_uri("qt/qtsvg/qtsvg-RHEL_7_4.7z")
             self.assertEqual(
-                resolver.resolve_payload_uri("qt/qtsvg/qtsvg-RHEL_7_4.7z").pop(),
-                "http://intranet.local.it/artifacts/qt/qtsvg/qtsvg-RHEL_7_4.7z",
+                resolved.pop(), "http://intranet.local.it/artifacts/qt/qtsvg/qtsvg-RHEL_7_4.7z"
             )
-            self.assertEqual(resolver.resolve_payload_uri(__file__).pop(), __file__)
+            _, resolved = resolver.resolve_payload_uri(__file__)
+            self.assertEqual(resolved.pop(), __file__)
 
     @data(  # type: ignore
         (
@@ -343,8 +345,11 @@ class TestRunner(unittest.TestCase):
     )
     @unpack  # type: ignore
     @unittest.mock.patch("htmllistparse.fetch_listing", side_effect=create_listing)  # type: ignore
-    def test_pattern_archive_resolver(self, pattern: str, expected: List[str], _: Any) -> None:
+    def test_pattern_archive_resolver(
+        self, pattern: str, expected_uris: List[str], _: Any
+    ) -> None:
         resolver = ArchiveResolver("", "")
+        expected = (URL("http://fileshare.intra/base/path"), expected_uris)
         self.assertCountEqual(asyncio.run(resolver.resolve_uri_pattern(pattern, None)), expected)
 
     def test_locate_pkg_templ_dir_invalid(self) -> None: