From b5f022b3cd96a807c8200956d6e2617991868b58 Mon Sep 17 00:00:00 2001 From: Patrik Teivonen Date: Wed, 1 Mar 2023 10:30:38 +0200 Subject: release_task_reader: Improve task parsing logging output Allow parse_config to specify multiple task types e.g. IFW and QBSP. Then tasks excluded by the filters get printed out only once. Change-Id: I9b310227c1579425ae1edacdfb3809df2727058c Reviewed-by: Iikka Eklund --- packaging-tools/release_repo_updater.py | 23 +++--- packaging-tools/release_repo_updater_deb.py | 2 +- packaging-tools/release_task_reader.py | 34 +++++---- packaging-tools/tests/test_debian_repo_release.py | 4 +- packaging-tools/tests/test_release_repo_updater.py | 6 +- packaging-tools/tests/test_release_task_reader.py | 87 +++++++++++++--------- 6 files changed, 89 insertions(+), 67 deletions(-) diff --git a/packaging-tools/release_repo_updater.py b/packaging-tools/release_repo_updater.py index 4e4dc33dc..54f3e2618 100755 --- a/packaging-tools/release_repo_updater.py +++ b/packaging-tools/release_repo_updater.py @@ -1173,7 +1173,7 @@ def handle_offline_jobs( # get offline tasks tasks = parse_config( config_file=args.config, - task_type=TaskType.IFW_TASK_TYPE, + task_types=[TaskType.IFW_TASK_TYPE], task_filters=append_to_task_filters(args.task_filters, "offline"), ) asyncio_run( @@ -1200,17 +1200,16 @@ def handle_online_repo_jobs( export_data: Dict[str, str], ) -> None: # get online repository tasks per type - for task_type in [TaskType.IFW_TASK_TYPE, TaskType.QBSP_TASK_TYPE]: - tasks = parse_config( - config_file=args.config, - task_type=task_type, - task_filters=append_to_task_filters(args.task_filters, "repository"), - ) - - if not tasks: - log.info("No tasks found for type: %s", task_type.value) + task_types = [TaskType.IFW_TASK_TYPE, TaskType.QBSP_TASK_TYPE] + tasks = parse_config( + config_file=args.config, + task_types=task_types, + task_filters=append_to_task_filters(args.task_filters, "repository"), + ) + for task_type, task_list in tasks.items(): + if not task_list: + log.info("No tasks found for type: %s", task_type) continue - update_strategy = RepoUpdateStrategy.get_strategy( staging_server_root=args.staging_server_root, license_=args.license_, @@ -1228,7 +1227,7 @@ def handle_online_repo_jobs( ) bld_args = build_strategy.parse_bld_args(args) - build_strategy.execute_build(tasks, bld_args) # type: ignore + build_strategy.execute_build(task_list, bld_args) # type: ignore def main() -> None: diff --git a/packaging-tools/release_repo_updater_deb.py b/packaging-tools/release_repo_updater_deb.py index ec2aebba6..c84ed7322 100644 --- a/packaging-tools/release_repo_updater_deb.py +++ b/packaging-tools/release_repo_updater_deb.py @@ -610,7 +610,7 @@ def parse_args() -> argparse.Namespace: def run(args: argparse.Namespace) -> None: tasks = parse_config( args.config, - task_type=TaskType.DEB_TASK_TYPE, + task_types=[TaskType.DEB_TASK_TYPE], task_filters=append_to_task_filters(args.task_filters, "deb"), ) auth = HTTPBasicAuth(username=args.aptly_api_user, password=args.api_pass) diff --git a/packaging-tools/release_task_reader.py b/packaging-tools/release_task_reader.py index 53f48ee02..8388e5234 100755 --- a/packaging-tools/release_task_reader.py +++ b/packaging-tools/release_task_reader.py @@ -34,6 +34,7 @@ import os import re import sys from abc import ABC, abstractmethod +from collections import defaultdict from configparser import ConfigParser, ExtendedInterpolation, SectionProxy from enum import Enum from typing import Any, Dict, List, Type, Union @@ -298,10 +299,12 @@ def append_to_task_filters(task_filters: List[str], task_filter: str) -> List[st def parse_data( settings: ConfigParser, - task_type: TaskType, + task_types: List[TaskType], task_filters: List[str], -) -> List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]]: - tasks: List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]] = [] +) -> Dict[TaskType, List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]]]: + tasks: Dict[ + TaskType, List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]] + ] = defaultdict(list) sec_filters_list = [get_filter_parts(x) for x in task_filters] common_substs = settings.get("common.substitutions", "substitutions", fallback="") @@ -318,13 +321,16 @@ def parse_data( break if append_task: log.info("Parsing Task: %s", section) - task = ReleaseTaskFactory.task_from_spec(task_spec=section, - requested_task_type=task_type, - name=section, - settings=settings[section], - common_substitutions=common_substs) - if task is not None: - tasks.append(task) + for task_type in task_types: + task = ReleaseTaskFactory.task_from_spec( + task_spec=section, + requested_task_type=task_type, + name=section, + settings=settings[section], + common_substitutions=common_substs + ) + if task is not None: + tasks[task_type].append(task) else: log.info("Skipping task: [%s] - excluded by filter(s): %s", section, sec_filters_list) return tasks @@ -332,14 +338,14 @@ def parse_data( def parse_config( config_file: str, - task_type: TaskType, + task_types: List[TaskType], task_filters: List[str], -) -> List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]]: +) -> Dict[TaskType, List[Union[IFWReleaseTask, QBSPReleaseTask, DebReleaseTask]]]: if not os.path.isfile(config_file): raise ReleaseTaskError(f"Not such file: {config_file}") settings = ConfigParser(interpolation=ExtendedInterpolation()) settings.read(config_file) - return parse_data(settings, task_type, task_filters) + return parse_data(settings, task_types, task_filters) def main() -> None: @@ -355,7 +361,7 @@ def main() -> None: args = parser.parse_args(sys.argv[1:]) assert os.path.isfile(args.config), f"Not a valid file: {args.config}" - parse_config(args.config, TaskType.from_value(args.task_type), args.task_filters) + parse_config(args.config, [TaskType.from_value(args.task_type)], args.task_filters) if __name__ == "__main__": diff --git a/packaging-tools/tests/test_debian_repo_release.py b/packaging-tools/tests/test_debian_repo_release.py index 0c6788c7a..49a32062a 100755 --- a/packaging-tools/tests/test_debian_repo_release.py +++ b/packaging-tools/tests/test_debian_repo_release.py @@ -241,12 +241,12 @@ class TestDebianRepoRelease(unittest.TestCase): """ config = ConfigParser() config.read_string(sample_config) - tasks = parse_data(config, task_type=TaskType.DEB_TASK_TYPE, task_filters=[]) + tasks = parse_data(config, task_types=[TaskType.DEB_TASK_TYPE], task_filters=[]) client = AptlyApiClient(api_endpoint=f"http://{self.aptly_api}/", http_auth=None) create_and_publish_repos( client, - tasks, # type: ignore + tasks[TaskType.DEB_TASK_TYPE], # type: ignore gpg_key="", gpg_passphrase="", rta=None, diff --git a/packaging-tools/tests/test_release_repo_updater.py b/packaging-tools/tests/test_release_repo_updater.py index bd865cd5e..77bf9c669 100755 --- a/packaging-tools/tests/test_release_repo_updater.py +++ b/packaging-tools/tests/test_release_repo_updater.py @@ -222,16 +222,16 @@ class TestReleaseRepoUpdater(unittest.TestCase): config.read_string(sample_config) # parse all tasks i.e. no filters - tasks = parse_data(config, task_type=TaskType.IFW_TASK_TYPE, task_filters=[]) + tasks = parse_data(config, task_types=[TaskType.IFW_TASK_TYPE], task_filters=[]) await build_online_repositories( - tasks=cast(List[IFWReleaseTask], tasks), + tasks=cast(List[IFWReleaseTask], tasks[TaskType.IFW_TASK_TYPE]), license_="opensource", installer_config_base_dir="foo", artifact_share_base_url="foo", ifw_tools="foo", build_repositories=False, ) - task = cast(IFWReleaseTask, tasks.pop()) + task = cast(IFWReleaseTask, tasks[TaskType.IFW_TASK_TYPE].pop()) self.assertTrue(task.source_online_repository_path.endswith("foo/bar/path_1/online_repository")) @asyncio_test diff --git a/packaging-tools/tests/test_release_task_reader.py b/packaging-tools/tests/test_release_task_reader.py index 350aebfef..e57819e86 100755 --- a/packaging-tools/tests/test_release_task_reader.py +++ b/packaging-tools/tests/test_release_task_reader.py @@ -107,10 +107,10 @@ class TestReleaseTaskReader(unittest.TestCase): config = ConfigParser() config.read_string(sample_config) if is_valid: - self.assertIsNotNone(parse_data(config, task_type=task_type, task_filters=filters)) + self.assertIsNotNone(parse_data(config, task_types=[task_type], task_filters=filters)) else: with self.assertRaises(ReleaseTaskError): - parse_data(config, task_type=task_type, task_filters=filters) + parse_data(config, task_types=[task_type], task_filters=filters) @data( # type: ignore (""" @@ -150,38 +150,48 @@ class TestReleaseTaskReader(unittest.TestCase): config = ConfigParser() config.read_string(sample_config) # parse all tasks i.e. no filters - tasks = parse_data(config, task_type=TaskType.IFW_TASK_TYPE, task_filters=[]) - self.assertTrue(len(tasks) == 4, "Did not parse all tasks from sample config") + tasks = parse_data(config, task_types=[TaskType.IFW_TASK_TYPE], task_filters=[]) + self.assertTrue( + len(tasks[TaskType.IFW_TASK_TYPE]) == 4, "Did not parse all tasks from sample config" + ) # parse only "repository" tasks - tasks = parse_data(config, task_type=TaskType.IFW_TASK_TYPE, task_filters=["repository"]) - self.assertTrue(len(tasks) == 1) - self.assertEqual(tasks[0].config_file, "foobar-file-repository") - self.assertEqual(tasks[0].substitutions, ["arg1", "arg2", "arg3"]) - self.assertEqual(tasks[0].rta_key_list, ["key1", "key2", "key3", "key4"]) - self.assertEqual(tasks[0].repo_path, "foo/bar/path") - self.assertEqual(sorted(tasks[0].rta_key_list), sorted(["key1", "key2", "key3", "key4"])) + tasks = parse_data( + config, task_types=[TaskType.IFW_TASK_TYPE], task_filters=["repository"] + ) + self.assertTrue(len(tasks[TaskType.IFW_TASK_TYPE]) == 1) + self.assertEqual( + tasks[TaskType.IFW_TASK_TYPE][0].config_file, "foobar-file-repository" + ) + self.assertEqual( + tasks[TaskType.IFW_TASK_TYPE][0].substitutions, ["arg1", "arg2", "arg3"] + ) + self.assertEqual( + tasks[TaskType.IFW_TASK_TYPE][0].rta_key_list, ["key1", "key2", "key3", "key4"] + ) + self.assertEqual(tasks[TaskType.IFW_TASK_TYPE][0].repo_path, "foo/bar/path") + self.assertEqual(sorted(tasks[TaskType.IFW_TASK_TYPE][0].rta_key_list), sorted(["key1", "key2", "key3", "key4"])) # parse only "offline" tasks with multiple filters tasks = parse_data( config, - task_type=TaskType.IFW_TASK_TYPE, + task_types=[TaskType.IFW_TASK_TYPE], task_filters=["offline,linux,x86_64"], ) - self.assertTrue(len(tasks) == 2) + self.assertTrue(len(tasks[TaskType.IFW_TASK_TYPE]) == 2) tasks = parse_data( config, - task_type=TaskType.IFW_TASK_TYPE, + task_types=[TaskType.IFW_TASK_TYPE], task_filters=["offline,linux,x86_64,foobar"], ) - self.assertTrue(len(tasks) == 1) + self.assertTrue(len(tasks[TaskType.IFW_TASK_TYPE]) == 1) # parse "offline" tasks with multiple filters and "online" tasks tasks = parse_data( config, - task_type=TaskType.IFW_TASK_TYPE, + task_types=[TaskType.IFW_TASK_TYPE], task_filters=["offline,linux,x86_64", "online,linux,x86_64"], ) - self.assertTrue(len(tasks) == 3) + self.assertTrue(len(tasks[TaskType.IFW_TASK_TYPE]) == 3) @asyncio_test async def test_release_task_reader_deb(self) -> None: @@ -208,20 +218,27 @@ class TestReleaseTaskReader(unittest.TestCase): config.read_string(sample_config) # parse all tasks i.e. no filters - tasks = parse_data(config, task_type=TaskType.DEB_TASK_TYPE, task_filters=[]) - self.assertTrue(len(tasks) == 2, "Did not parse all tasks from sample config") + tasks = parse_data(config, task_types=[TaskType.DEB_TASK_TYPE], task_filters=[]) + self.assertTrue(len(tasks[TaskType.DEB_TASK_TYPE]) == 2, "Did not parse all tasks from sample config") # parse only "repository" tasks - tasks = parse_data(config, task_type=TaskType.DEB_TASK_TYPE, task_filters=["amd64"]) - self.assertTrue(len(tasks) == 1) - self.assertEqual(tasks[0].substitutions, ["foo=bar", "aaa=bbb"]) - self.assertEqual(tasks[0].repo_path, "test_repo/amd64") - self.assertEqual(tasks[0].distribution, "barbar-file-repository-amd64") # type: ignore - self.assertEqual(tasks[0].component, "main") # type: ignore - self.assertEqual(tasks[0].architectures, ["amd64"]) # type: ignore - self.assertEqual(tasks[0].content_sources, ["http://bar.com/content1", # type: ignore - "http://bar.com/content2"]) - self.assertEqual(sorted(tasks[0].rta_key_list), sorted(["key1", "key2"])) + tasks = parse_data(config, task_types=[TaskType.DEB_TASK_TYPE], task_filters=["amd64"]) + self.assertTrue(len(tasks[TaskType.DEB_TASK_TYPE]) == 1) + self.assertEqual(tasks[TaskType.DEB_TASK_TYPE][0].substitutions, ["foo=bar", "aaa=bbb"]) + self.assertEqual(tasks[TaskType.DEB_TASK_TYPE][0].repo_path, "test_repo/amd64") + self.assertEqual( + tasks[TaskType.DEB_TASK_TYPE][0].distribution, # type: ignore + "barbar-file-repository-amd64" + ) + self.assertEqual(tasks[TaskType.DEB_TASK_TYPE][0].component, "main") # type: ignore + self.assertEqual(tasks[TaskType.DEB_TASK_TYPE][0].architectures, ["amd64"]) # type: ignore + self.assertEqual( + tasks[TaskType.DEB_TASK_TYPE][0].content_sources, # type: ignore + ["http://bar.com/content1", "http://bar.com/content2"] + ) + self.assertEqual( + sorted(tasks[TaskType.DEB_TASK_TYPE][0].rta_key_list), sorted(["key1", "key2"]) + ) @asyncio_test async def test_release_task_reader_qbsp(self) -> None: @@ -240,13 +257,13 @@ class TestReleaseTaskReader(unittest.TestCase): config.read_string(sample_config) # parse all tasks i.e. no filters - tasks = parse_data(config, task_type=TaskType.QBSP_TASK_TYPE, task_filters=[]) - self.assertTrue(len(tasks) == 2, "Did not parse all tasks from sample config") + tasks = parse_data(config, task_types=[TaskType.QBSP_TASK_TYPE], task_filters=[]) + self.assertTrue(len(tasks[TaskType.QBSP_TASK_TYPE]) == 2, "Did not parse all tasks from sample config") # parse only "foo" tasks - tasks = parse_data(config, task_type=TaskType.QBSP_TASK_TYPE, task_filters=["foo"]) - self.assertTrue(len(tasks) == 1) - task = tasks.pop() + tasks = parse_data(config, task_types=[TaskType.QBSP_TASK_TYPE], task_filters=["foo"]) + self.assertTrue(len(tasks[TaskType.QBSP_TASK_TYPE]) == 1) + task = tasks[TaskType.QBSP_TASK_TYPE].pop() assert isinstance(task, QBSPReleaseTask) self.assertEqual(task.qbsp_file, "https://foo.com/path1/foo.qbsp") self.assertEqual(task.repo_path, "test_repo/temp1") @@ -264,7 +281,7 @@ class TestReleaseTaskReader(unittest.TestCase): config = ConfigParser() config.read_string(sample_config) with self.assertRaises(ReleaseTaskError): - parse_data(config, task_type=TaskType.IFW_TASK_TYPE, task_filters=[]) + parse_data(config, task_types=[TaskType.IFW_TASK_TYPE], task_filters=[]) if __name__ == '__main__': -- cgit v1.2.3