release_task_reader: Improve task parsing logging outputv6.5.0-packaging

Allow parse_config to specify multiple task types e.g. IFW and QBSP.
Then tasks excluded by the filters get printed out only once.

Change-Id: I9b310227c1579425ae1edacdfb3809df2727058c
Reviewed-by: Iikka Eklund <iikka.eklund@qt.io>

6 files changed, 89 insertions, 67 deletions

diff --git a/packaging-tools/release_repo_updater.py b/packaging-tools/release_repo_updater.py
index 4e4dc33dc..54f3e2618 100755
--- a/packaging-tools/release_repo_updater.py
+++ b/packaging-tools/release_repo_updater.py
@@ -1173,7 +1173,7 @@ def handle_offline_jobs(
     # get offline tasks
     tasks = parse_config(
         config_file=args.config,
-        task_type=TaskType.IFW_TASK_TYPE,
+        task_types=[TaskType.IFW_TASK_TYPE],
         task_filters=append_to_task_filters(args.task_filters, "offline"),
     )
     asyncio_run(
@@ -1200,17 +1200,16 @@ def handle_online_repo_jobs(
     export_data: Dict[str, str],
 ) -> None:
     # get online repository tasks per type
-    for task_type in [TaskType.IFW_TASK_TYPE, TaskType.QBSP_TASK_TYPE]:
-        tasks = parse_config(
-            config_file=args.config,
-            task_type=task_type,
-            task_filters=append_to_task_filters(args.task_filters, "repository"),
-        )
-
-        if not tasks:
-            log.info("No tasks found for type: %s", task_type.value)
+    task_types = [TaskType.IFW_TASK_TYPE, TaskType.QBSP_TASK_TYPE]
+    tasks = parse_config(
+        config_file=args.config,
+        task_types=task_types,
+        task_filters=append_to_task_filters(args.task_filters, "repository"),
+    )
+    for task_type, task_list in tasks.items():
+        if not task_list:
+            log.info("No tasks found for type: %s", task_type)
             continue
-
         update_strategy = RepoUpdateStrategy.get_strategy(
             staging_server_root=args.staging_server_root,
             license_=args.license_,
@@ -1228,7 +1227,7 @@ def handle_online_repo_jobs(
         )
 
         bld_args = build_strategy.parse_bld_args(args)
-        build_strategy.execute_build(tasks, bld_args)  # type: ignore
+        build_strategy.execute_build(task_list, bld_args)  # type: ignore
 
 
 def main() -> None:
diff --git a/packaging-tools/release_repo_updater_deb.py b/packaging-tools/release_repo_updater_deb.py
index ec2aebba6..c84ed7322 100644
--- a/packaging-tools/release_repo_updater_deb.py
+++ b/packaging-tools/release_repo_updater_deb.py
@@ -610,7 +610,7 @@ def parse_args() -> argparse.Namespace:
 def run(args: argparse.Namespace) -> None:
     tasks = parse_config(
         args.config,
-        task_type=TaskType.DEB_TASK_TYPE,
+        task_types=[TaskType.DEB_TASK_TYPE],
         task_filters=append_to_task_filters(args.task_filters, "deb"),
     )
     auth = HTTPBasicAuth(username=args.aptly_api_user, password=args.api_pass)
diff --git a/packaging-tools/release_task_reader.py b/packaging-tools/release_task_reader.py
index 53f48ee02..8388e5234 100755
--- a/packaging-tools/release_task_reader.py
+++ b/packaging-tools/release_task_reader.py
@@ -34,6 +34,7 @@ import os
 import re
 import sys
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from configparser import ConfigParser, ExtendedInterpolation, SectionProxy
 from enum import Enum
 from typing import Any, Dict, List, Type, Union
@@ -298,10 +299,12 @@ def append_to_task_filters(task_filters: List[str], task_filter: str) -> List[st
 
 def parse_data(
     settings: ConfigParser,
-    task_type: TaskType,
+    task_types: List[TaskType],
     task_filters: List[str],
-) -> List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]]:
-    tasks: List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]] = []
+) -> Dict[TaskType, List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]]]:
+    tasks: Dict[
+        TaskType, List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]]
+    ] = defaultdict(list)
     sec_filters_list = [get_filter_parts(x) for x in task_filters]
     common_substs = settings.get("common.substitutions", "substitutions", fallback="")
 
@@ -318,13 +321,16 @@ def parse_data(
                     break
         if append_task:
             log.info("Parsing Task: %s", section)
-            task = ReleaseTaskFactory.task_from_spec(task_spec=section,
-                                                     requested_task_type=task_type,
-                                                     name=section,
-                                                     settings=settings[section],
-                                                     common_substitutions=common_substs)
-            if task is not None:
-                tasks.append(task)
+            for task_type in task_types:
+                task = ReleaseTaskFactory.task_from_spec(
+                    task_spec=section,
+                    requested_task_type=task_type,
+                    name=section,
+                    settings=settings[section],
+                    common_substitutions=common_substs
+                )
+                if task is not None:
+                    tasks[task_type].append(task)
         else:
             log.info("Skipping task: [%s] - excluded by filter(s): %s", section, sec_filters_list)
     return tasks
@@ -332,14 +338,14 @@ def parse_data(
 
 def parse_config(
     config_file: str,
-    task_type: TaskType,
+    task_types: List[TaskType],
     task_filters: List[str],
-) -> List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]]:
+) -> Dict[TaskType, List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]]]:
     if not os.path.isfile(config_file):
         raise ReleaseTaskError(f"Not such file: {config_file}")
     settings = ConfigParser(interpolation=ExtendedInterpolation())
     settings.read(config_file)
-    return parse_data(settings, task_type, task_filters)
+    return parse_data(settings, task_types, task_filters)
 
 
 def main() -> None:
@@ -355,7 +361,7 @@ def main() -> None:
     args = parser.parse_args(sys.argv[1:])
 
     assert os.path.isfile(args.config), f"Not a valid file: {args.config}"
-    parse_config(args.config, TaskType.from_value(args.task_type), args.task_filters)
+    parse_config(args.config, [TaskType.from_value(args.task_type)], args.task_filters)
 
 
 if __name__ == "__main__":
diff --git a/packaging-tools/tests/test_debian_repo_release.py b/packaging-tools/tests/test_debian_repo_release.py
index 0c6788c7a..49a32062a 100755
--- a/packaging-tools/tests/test_debian_repo_release.py
+++ b/packaging-tools/tests/test_debian_repo_release.py
@@ -241,12 +241,12 @@ class TestDebianRepoRelease(unittest.TestCase):
         """
         config = ConfigParser()
         config.read_string(sample_config)
-        tasks = parse_data(config, task_type=TaskType.DEB_TASK_TYPE, task_filters=[])
+        tasks = parse_data(config, task_types=[TaskType.DEB_TASK_TYPE], task_filters=[])
 
         client = AptlyApiClient(api_endpoint=f"http://{self.aptly_api}/", http_auth=None)
         create_and_publish_repos(
             client,
-            tasks,  # type: ignore
+            tasks[TaskType.DEB_TASK_TYPE],  # type: ignore
             gpg_key="",
             gpg_passphrase="",
             rta=None,
diff --git a/packaging-tools/tests/test_release_repo_updater.py b/packaging-tools/tests/test_release_repo_updater.py
index bd865cd5e..77bf9c669 100755
--- a/packaging-tools/tests/test_release_repo_updater.py
+++ b/packaging-tools/tests/test_release_repo_updater.py
@@ -222,16 +222,16 @@ class TestReleaseRepoUpdater(unittest.TestCase):
         config.read_string(sample_config)
 
         # parse all tasks i.e. no filters
-        tasks = parse_data(config, task_type=TaskType.IFW_TASK_TYPE, task_filters=[])
+        tasks = parse_data(config, task_types=[TaskType.IFW_TASK_TYPE], task_filters=[])
         await build_online_repositories(
-            tasks=cast(List[IFWReleaseTask], tasks),
+            tasks=cast(List[IFWReleaseTask], tasks[TaskType.IFW_TASK_TYPE]),
             license_="opensource",
             installer_config_base_dir="foo",
             artifact_share_base_url="foo",
             ifw_tools="foo",
             build_repositories=False,
         )
-        task = cast(IFWReleaseTask, tasks.pop())
+        task = cast(IFWReleaseTask, tasks[TaskType.IFW_TASK_TYPE].pop())
         self.assertTrue(task.source_online_repository_path.endswith("foo/bar/path_1/online_repository"))
 
     @asyncio_test
diff --git a/packaging-tools/tests/test_release_task_reader.py b/packaging-tools/tests/test_release_task_reader.py
index 350aebfef..e57819e86 100755
--- a/packaging-tools/tests/test_release_task_reader.py
+++ b/packaging-tools/tests/test_release_task_reader.py
@@ -107,10 +107,10 @@ class TestReleaseTaskReader(unittest.TestCase):
         config = ConfigParser()
         config.read_string(sample_config)
         if is_valid:
-            self.assertIsNotNone(parse_data(config, task_type=task_type, task_filters=filters))
+            self.assertIsNotNone(parse_data(config, task_types=[task_type], task_filters=filters))
         else:
             with self.assertRaises(ReleaseTaskError):
-                parse_data(config, task_type=task_type, task_filters=filters)
+                parse_data(config, task_types=[task_type], task_filters=filters)
 
     @data(  # type: ignore
         ("""
@@ -150,38 +150,48 @@ class TestReleaseTaskReader(unittest.TestCase):
         config = ConfigParser()
         config.read_string(sample_config)
         # parse all tasks i.e. no filters
-        tasks = parse_data(config, task_type=TaskType.IFW_TASK_TYPE, task_filters=[])
-        self.assertTrue(len(tasks) == 4, "Did not parse all tasks from sample config")
+        tasks = parse_data(config, task_types=[TaskType.IFW_TASK_TYPE], task_filters=[])
+        self.assertTrue(
+            len(tasks[TaskType.IFW_TASK_TYPE]) == 4, "Did not parse all tasks from sample config"
+        )
         # parse only "repository" tasks
-        tasks = parse_data(config, task_type=TaskType.IFW_TASK_TYPE, task_filters=["repository"])
-        self.assertTrue(len(tasks) == 1)
-        self.assertEqual(tasks[0].config_file, "foobar-file-repository")
-        self.assertEqual(tasks[0].substitutions, ["arg1", "arg2", "arg3"])
-        self.assertEqual(tasks[0].rta_key_list, ["key1", "key2", "key3", "key4"])
-        self.assertEqual(tasks[0].repo_path, "foo/bar/path")
-        self.assertEqual(sorted(tasks[0].rta_key_list), sorted(["key1", "key2", "key3", "key4"]))
+        tasks = parse_data(
+            config, task_types=[TaskType.IFW_TASK_TYPE], task_filters=["repository"]
+        )
+        self.assertTrue(len(tasks[TaskType.IFW_TASK_TYPE]) == 1)
+        self.assertEqual(
+            tasks[TaskType.IFW_TASK_TYPE][0].config_file, "foobar-file-repository"
+        )
+        self.assertEqual(
+            tasks[TaskType.IFW_TASK_TYPE][0].substitutions, ["arg1", "arg2", "arg3"]
+        )
+        self.assertEqual(
+            tasks[TaskType.IFW_TASK_TYPE][0].rta_key_list, ["key1", "key2", "key3", "key4"]
+        )
+        self.assertEqual(tasks[TaskType.IFW_TASK_TYPE][0].repo_path, "foo/bar/path")
+        self.assertEqual(sorted(tasks[TaskType.IFW_TASK_TYPE][0].rta_key_list), sorted(["key1", "key2", "key3", "key4"]))
 
         # parse only "offline" tasks with multiple filters
         tasks = parse_data(
             config,
-            task_type=TaskType.IFW_TASK_TYPE,
+            task_types=[TaskType.IFW_TASK_TYPE],
             task_filters=["offline,linux,x86_64"],
         )
-        self.assertTrue(len(tasks) == 2)
+        self.assertTrue(len(tasks[TaskType.IFW_TASK_TYPE]) == 2)
         tasks = parse_data(
             config,
-            task_type=TaskType.IFW_TASK_TYPE,
+            task_types=[TaskType.IFW_TASK_TYPE],
             task_filters=["offline,linux,x86_64,foobar"],
         )
-        self.assertTrue(len(tasks) == 1)
+        self.assertTrue(len(tasks[TaskType.IFW_TASK_TYPE]) == 1)
 
         # parse "offline" tasks with multiple filters and "online" tasks
         tasks = parse_data(
             config,
-            task_type=TaskType.IFW_TASK_TYPE,
+            task_types=[TaskType.IFW_TASK_TYPE],
             task_filters=["offline,linux,x86_64", "online,linux,x86_64"],
         )
-        self.assertTrue(len(tasks) == 3)
+        self.assertTrue(len(tasks[TaskType.IFW_TASK_TYPE]) == 3)
 
     @asyncio_test
     async def test_release_task_reader_deb(self) -> None:
@@ -208,20 +218,27 @@ class TestReleaseTaskReader(unittest.TestCase):
         config.read_string(sample_config)
 
         # parse all tasks i.e. no filters
-        tasks = parse_data(config, task_type=TaskType.DEB_TASK_TYPE, task_filters=[])
-        self.assertTrue(len(tasks) == 2, "Did not parse all tasks from sample config")
+        tasks = parse_data(config, task_types=[TaskType.DEB_TASK_TYPE], task_filters=[])
+        self.assertTrue(len(tasks[TaskType.DEB_TASK_TYPE]) == 2, "Did not parse all tasks from sample config")
 
         # parse only "repository" tasks
-        tasks = parse_data(config, task_type=TaskType.DEB_TASK_TYPE, task_filters=["amd64"])
-        self.assertTrue(len(tasks) == 1)
-        self.assertEqual(tasks[0].substitutions, ["foo=bar", "aaa=bbb"])
-        self.assertEqual(tasks[0].repo_path, "test_repo/amd64")
-        self.assertEqual(tasks[0].distribution, "barbar-file-repository-amd64")  # type: ignore
-        self.assertEqual(tasks[0].component, "main")  # type: ignore
-        self.assertEqual(tasks[0].architectures, ["amd64"])  # type: ignore
-        self.assertEqual(tasks[0].content_sources, ["http://bar.com/content1",  # type: ignore
-                                                    "http://bar.com/content2"])
-        self.assertEqual(sorted(tasks[0].rta_key_list), sorted(["key1", "key2"]))
+        tasks = parse_data(config, task_types=[TaskType.DEB_TASK_TYPE], task_filters=["amd64"])
+        self.assertTrue(len(tasks[TaskType.DEB_TASK_TYPE]) == 1)
+        self.assertEqual(tasks[TaskType.DEB_TASK_TYPE][0].substitutions, ["foo=bar", "aaa=bbb"])
+        self.assertEqual(tasks[TaskType.DEB_TASK_TYPE][0].repo_path, "test_repo/amd64")
+        self.assertEqual(
+            tasks[TaskType.DEB_TASK_TYPE][0].distribution,  # type: ignore
+            "barbar-file-repository-amd64"
+        )
+        self.assertEqual(tasks[TaskType.DEB_TASK_TYPE][0].component, "main")  # type: ignore
+        self.assertEqual(tasks[TaskType.DEB_TASK_TYPE][0].architectures, ["amd64"])  # type: ignore
+        self.assertEqual(
+            tasks[TaskType.DEB_TASK_TYPE][0].content_sources,  # type: ignore
+            ["http://bar.com/content1", "http://bar.com/content2"]
+        )
+        self.assertEqual(
+            sorted(tasks[TaskType.DEB_TASK_TYPE][0].rta_key_list), sorted(["key1", "key2"])
+        )
 
     @asyncio_test
     async def test_release_task_reader_qbsp(self) -> None:
@@ -240,13 +257,13 @@ class TestReleaseTaskReader(unittest.TestCase):
         config.read_string(sample_config)
 
         # parse all tasks i.e. no filters
-        tasks = parse_data(config, task_type=TaskType.QBSP_TASK_TYPE, task_filters=[])
-        self.assertTrue(len(tasks) == 2, "Did not parse all tasks from sample config")
+        tasks = parse_data(config, task_types=[TaskType.QBSP_TASK_TYPE], task_filters=[])
+        self.assertTrue(len(tasks[TaskType.QBSP_TASK_TYPE]) == 2, "Did not parse all tasks from sample config")
 
         # parse only "foo" tasks
-        tasks = parse_data(config, task_type=TaskType.QBSP_TASK_TYPE, task_filters=["foo"])
-        self.assertTrue(len(tasks) == 1)
-        task = tasks.pop()
+        tasks = parse_data(config, task_types=[TaskType.QBSP_TASK_TYPE], task_filters=["foo"])
+        self.assertTrue(len(tasks[TaskType.QBSP_TASK_TYPE]) == 1)
+        task = tasks[TaskType.QBSP_TASK_TYPE].pop()
         assert isinstance(task, QBSPReleaseTask)
         self.assertEqual(task.qbsp_file, "https://foo.com/path1/foo.qbsp")
         self.assertEqual(task.repo_path, "test_repo/temp1")
@@ -264,7 +281,7 @@ class TestReleaseTaskReader(unittest.TestCase):
         config = ConfigParser()
         config.read_string(sample_config)
         with self.assertRaises(ReleaseTaskError):
-            parse_data(config, task_type=TaskType.IFW_TASK_TYPE, task_filters=[])
+            parse_data(config, task_types=[TaskType.IFW_TASK_TYPE], task_filters=[])
 
 
 if __name__ == '__main__':