Add a machine-readable state printer for bot monitoring

This enables the bot to be monitored by a monitoring system,
providing the full state of the bot in a machine-readable format.

This patch also contains a number of previously in-production
hotfixes.

Task-number: QTQAINFRA-5489
Change-Id: I7647935d79c1f7d184f60bfac1bba1e0ab2526b7
Reviewed-by: Daniel Smith <daniel.smith@qt.io>

9 files changed, 105 insertions, 48 deletions

diff --git a/util/dependency_updater/config.yaml.template b/util/dependency_updater/config.yaml.template
index 1db7019..fd94f78 100644
--- a/util/dependency_updater/config.yaml.template
+++ b/util/dependency_updater/config.yaml.template
@@ -2,6 +2,6 @@ GERRIT_HOST: codereview.qt-project.org
 GERRIT_STATE_PATH: playground/tqtc-personal-projects
 GERRIT_USERNAME: ''
 GERRIT_PASSWORD: ''
-INTERNAL_COIN_HOST: ''
+COIN_INTERNAL_HOST: ''
 MS_TEAMS_NOTIFY_URL: ''
 REPOS: []
diff --git a/util/dependency_updater/main.py b/util/dependency_updater/main.py
index 67910cd..fae3f0a 100644
--- a/util/dependency_updater/main.py
+++ b/util/dependency_updater/main.py
@@ -4,10 +4,19 @@
 import argparse
 import os
 import sys
-
 import yaml
+import json
+
+from tools import Namespace, Proposal, config as Config, state, toolbox, dependency_resolver, repo as Repo
+
 
-from tools import Namespace, config as Config, state, toolbox, dependency_resolver, repo as Repo
+class ProposalEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Proposal):
+            return obj.__dict__
+        elif isinstance(obj, set):
+            return list(obj)
+        return json.JSONEncoder.default(self, obj)
 
 
 def parse_args(print_help: bool = False) -> Namespace:
@@ -99,6 +108,12 @@ def clear():
         os.system('clear')
 
 
+def printJsonDump(config):
+    print("=+=+=+ JSON DUMP +=+=+=")
+    print(json.dumps([repo.__dict__ for repo in config.state_data.values()], cls=ProposalEncoder))
+    print("=+=+=+ END JSON DUMP +=+=+=")
+
+
 def main():
     # Initial setup
     config = Config._load_config("config.yaml", parse_args())
@@ -106,9 +121,11 @@ def main():
     config.state_repo = state.check_create_local_repo(config)
     if config.args.prune_and_keep:
         print(config.args.prune_and_keep)
+        printJsonDump(config)
         state.clear_state(config, config.args.prune_and_keep)
         exit()
     if config.args.reset:
+        printJsonDump(config)
         state.clear_state(config)
         exit()
     if config.args.update_default_repos:
@@ -147,6 +164,9 @@ def main():
             changes_since_last_run = True
             print(f"{repo.id} moved from {progress.name} to {repo.progress.name}.")
 
+    # Try to set the round topic, but qtbase may not be ready yet.
+    toolbox.set_round_topic(config)
+
     # Check to see if we should abort as finished-failed
     if config.state_data.get("pause_on_finish_fail"):
         # If none of the retry conditions are met, print the help and exit.
@@ -158,6 +178,7 @@ def main():
                 " --rewind, or --retry_failed, or manually integrate any of the"
                 " failed changes below.")
             print(toolbox.state_printer(config))
+            printJsonDump(config)
             parse_args(print_help=True)
             exit()
         # Continue the round and try again.
@@ -204,9 +225,11 @@ def main():
                   f" {config.state_data[config.rewind_module.id].dep_list}")
         else:
             config.state_data[config.rewind_module.id].proposal.change_id = ""
-            new_sha = toolbox.get_head(config, config.state_data[config.rewind_module.id], True)
+            new_sha, message = toolbox.get_head(config, config.state_data[config.rewind_module.id],
+                                                True)
             print(f"\nRewinding round to {config.rewind_module.id} @ {new_sha}\n")
             config.state_data[config.rewind_module.id].original_ref = new_sha
+            config.state_data[config.rewind_module.id].original_message = message
             config.state_data[config.rewind_module.id].proposal.merged_ref = new_sha
             config.state_data[config.rewind_module.id].progress = Repo.PROGRESS.DONE_NO_UPDATE
             config.teams_connector.send_teams_webhook_basic(
@@ -257,6 +280,9 @@ def main():
             config.teams_connector.send_teams_webhook_module_failed(repo,
                                                                     test_failures=toolbox.parse_failed_integration_log(
                                                                         config, repo))
+        # bump the progress of the repo that has restaged.
+        repo.progress, repo.proposal.merged_ref, repo.proposal.gerrit_status = \
+            toolbox.get_check_progress(config, repo)
 
     # Check and see if we're ready to push a supermodule update if all the blocking repos
     # Have finished updating successfully.
@@ -278,6 +304,9 @@ def main():
     # Create new dependencies.yaml proposals for all PROGRESS.READY modules.
     config.state_data = dependency_resolver.recursive_prepare_updates(config)
 
+    # Try to set the round topic again. Qtbase should have been prepared now.
+    toolbox.set_round_topic(config)
+
     for repo in [r for r in config.state_data.values() if r.progress == Repo.PROGRESS.READY]:
         print(f"Proposed update to {repo.id}:")
         print("-----------------------------")
@@ -313,6 +342,8 @@ def main():
     else:
         print("No updates pushed this round. Nothing else to do this run.")
 
+    printJsonDump(config)
+
     # Determine how to exit
     clear_state = False
     if not any(r.progress < Repo.PROGRESS.DONE for r in config.state_data.values()):
diff --git a/util/dependency_updater/tools/__init__.py b/util/dependency_updater/tools/__init__.py
index d5fb2a8..7f18fa6 100644
--- a/util/dependency_updater/tools/__init__.py
+++ b/util/dependency_updater/tools/__init__.py
@@ -1,5 +1,6 @@
-__all__ = ['config', 'namespace', 'teams_connector']
+__all__ = ['config', 'proposal', 'namespace', 'teams_connector']
 
 from .config import Config
 from .namespace import Namespace
+from .proposal import Proposal
 from .teams_connector import TeamsConnector
diff --git a/util/dependency_updater/tools/config.py b/util/dependency_updater/tools/config.py
index fe3ea11..db73509 100644
--- a/util/dependency_updater/tools/config.py
+++ b/util/dependency_updater/tools/config.py
@@ -25,7 +25,7 @@ class Config(Namespace):
     GERRIT_STATE_PATH: str
     GERRIT_USERNAME: str
     GERRIT_PASSWORD: str
-    INTERNAL_COIN_HOST: str
+    COIN_INTERNAL_HOST: str
     MS_TEAMS_NOTIFY_URL: str
     state_repo: Repo
     state_data: dict[str, Repo] = {}
diff --git a/util/dependency_updater/tools/dependency_resolver.py b/util/dependency_updater/tools/dependency_resolver.py
index ff38e9b..1a025d6 100644
--- a/util/dependency_updater/tools/dependency_resolver.py
+++ b/util/dependency_updater/tools/dependency_resolver.py
@@ -67,7 +67,7 @@ def retrieve_or_generate_proposal(config: Config, repo) -> Proposal:
         for dep in repo.deps_yaml.get("dependencies"):
             prefix, dep_name = toolbox.strip_prefix(dep)
             full_name = [n for n in repo.dep_list if dep_name in n].pop()
-            proposal["dependencies"][dep]["ref"] = toolbox.get_head(config, full_name)
+            proposal["dependencies"][dep]["ref"] = toolbox.get_head(config, full_name)[0]
     if proposal == repo.deps_yaml:
         print(f"{repo.id} dependencies are already up-to-date")
     else:
@@ -210,7 +210,10 @@ def determine_ready(config: Config, repo: Repo) -> tuple[PROGRESS, bool, Union[l
     if repo.progress < PROGRESS.IN_PROGRESS \
             or repo.progress == PROGRESS.DONE_FAILED_DEPENDENCY:
         for dependency in repo.dep_list:
-            dep_repo = config.state_data[dependency]
+            try:
+                dep_repo = config.state_data[dependency]
+            except Exception as e:
+                print(f"{repo.id} requested nonexistent dependency: {dependency}")
             if dependency in config.state_data.keys():
                 # Recurse and update the progress in the case that we're trying
                 # to rewind with many previously failed dependencies.
diff --git a/util/dependency_updater/tools/proposal.py b/util/dependency_updater/tools/proposal.py
index 64ff0c3..63ece97 100644
--- a/util/dependency_updater/tools/proposal.py
+++ b/util/dependency_updater/tools/proposal.py
@@ -1,19 +1,16 @@
 # Copyright (C) 2020 The Qt Company Ltd.
 # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
-class Proposal:
-    proposed_yaml: dict
-    change_id: str
-    change_number: str
-    gerrit_status: str = ""
-    merged_ref: str = ""
-    inconsistent_set: dict
+from types import SimpleNamespace
 
+class Proposal(SimpleNamespace):
     def __init__(self, proposed_yaml: dict = None,
                  change_id: str = None, change_number: str = None, inconsistent_set: dict = None):
         self.proposed_yaml = proposed_yaml
         self.change_id = change_id
         self.change_number = change_number
+        self.gerrit_status = ""
+        self.merged_ref = ""
         self.inconsistent_set = inconsistent_set
 
     def __setattr__(self, key, value):
diff --git a/util/dependency_updater/tools/repo.py b/util/dependency_updater/tools/repo.py
index 1e6363c..8c04ef6 100644
--- a/util/dependency_updater/tools/repo.py
+++ b/util/dependency_updater/tools/repo.py
@@ -25,6 +25,9 @@ class PROGRESS(IntEnum):
     DONE_FAILED_DEPENDENCY = 11
     IGNORE_IS_META = 12
 
+    def __repr__(self):
+        return f"{self.name}"
+
 
 class Repo(Namespace):
     """Base information about a repository/submodule"""
@@ -32,10 +35,12 @@ class Repo(Namespace):
     prefix: str = ""  # Bare prefix such as qt/ or qt/tqtc-
     name: str = ""  # Bare name such as qtbase
     original_ref: str = ""  # Ref to associate with this repo. This value should never be changed.
+    original_message: str = ""  # Commit message subject of the parent. This value should never be changed.
     branch: str = ""  # Branch where dependencies.yaml was found. May differ from the specified branch.
     deps_yaml: yaml = dict()
     dep_list: list[str]
     proposal: Proposal = Proposal()
+    topic: str = ""
     to_stage: list[str]
     progress: PROGRESS = PROGRESS.UNSPECIFIED
     failed_dependencies: list[str]
@@ -56,6 +61,7 @@ class Repo(Namespace):
         self.prefix = prefix
         self.name = id.removeprefix(prefix)
         self.proposal = proposal or Proposal()
+        self.topic = ""
         if to_stage is not None:
             self.to_stage = to_stage
         if proposal and proposal.change_id not in self.to_stage:
diff --git a/util/dependency_updater/tools/teams_connector.py b/util/dependency_updater/tools/teams_connector.py
index 08239b8..1c6e0af 100644
--- a/util/dependency_updater/tools/teams_connector.py
+++ b/util/dependency_updater/tools/teams_connector.py
@@ -3,6 +3,7 @@
 
 import pymsteams as msteams
 import yaml
+import json
 from gerrit.changes import change as GerritChange
 from .repo import Repo, PROGRESS
 from typing import Union
diff --git a/util/dependency_updater/tools/toolbox.py b/util/dependency_updater/tools/toolbox.py
index b674eac..aec3ae8 100644
--- a/util/dependency_updater/tools/toolbox.py
+++ b/util/dependency_updater/tools/toolbox.py
@@ -80,6 +80,15 @@ def gerrit_link_maker(config: Config, change: Union[GerritChange.GerritChange, R
     return f"({mini_sha}) {subject[:70]}{'...' if len(subject) > 70 else ''}", url
 
 
+def set_round_topic(config: Config):
+    """Set the round topic"""
+    if config.state_data.get(config.args.repo_prefix+'qtbase').proposal.merged_ref:
+        print("Dependency Round Topic:",
+              f"dependency_round_{config.state_data.get(config.args.repo_prefix+'qtbase').proposal.merged_ref[:10]}")
+        for repo in config.state_data.values():
+            repo.topic = f"dependency_round_{config.state_data.get(config.args.repo_prefix+'qtbase').proposal.merged_ref[:10]}"
+
+
 def get_repos(config: Config, repos_override: list[str] = None, non_blocking_override: Union[list[str], None] = []) -> dict[str, Repo]:
     """Create a dict of initialized Repo objects. If repos_override is not specified,
     repos from the application's config/arguments are initialized alongside qt5 submodules
@@ -117,7 +126,7 @@ def get_repos(config: Config, repos_override: list[str] = None, non_blocking_ove
         else:
             # Initialize the new repo
             repo.deps_yaml, repo.branch = get_dependencies_yaml(config, repo)
-            repo.original_ref = get_head(config, repo)
+            repo.original_ref, repo.original_message = get_head(config, repo)
             retdict[repo.id] = repo
     if not config.args.update_default_repos and not config.args.use_head:
         for repo in retdict.keys():
@@ -243,26 +252,27 @@ def get_head(config: Config, repo: Union[Repo, str], pull_head: bool = False) ->
     saved ref from state if the repo progress is >= PROGRESS.DONE.
     Override state refs and pull remote branch HEAD with pull_head=True"""
     gerrit = config.datasources.gerrit_client
+    real_branch = repo.branch if type(repo) is Repo else config.args.branch
     if type(repo) == str:
         repo = search_for_repo(config, repo)
     if (not pull_head and repo.id in config.state_data.keys()
             and config.state_data[repo.id].progress >= PROGRESS.DONE):
         if config.state_data[repo.id].proposal.merged_ref:
-            return config.state_data[repo.id].proposal.merged_ref
+            return config.state_data[repo.id].proposal.merged_ref, ""
         else:
-            return config.state_data[repo.id].original_ref
+            return config.state_data[repo.id].original_ref, config.state_data[repo.id].original_message
     if repo.id in config.qt5_default.keys() and not config.args.use_head:
         r = gerrit.projects.get(config.args.repo_prefix + 'qt5').branches.get(
-            'refs/heads/' + config.args.branch).get_file_content(repo.name)
-        return bytes.decode(base64.b64decode(r), "utf-8")
+            'refs/heads/' + real_branch).get_file_content(repo.name)
+        return bytes.decode(base64.b64decode(r), "utf-8"), ""
     else:
-        branches = [config.args.branch, "dev", "master"]
+        branches = [real_branch, "dev", "master"]
         branch_head = None
         for branch in branches:
             try:
                 branch_head = gerrit.projects.get(repo.id).branches.get(f"refs/heads/{branch}")
-                if branch != config.args.branch and not config.suppress_warn:
-                    print(f"INFO: Using {branch} instead of {config.args.branch} "
+                if branch != real_branch and not config.suppress_warn:
+                    print(f"INFO: Using {branch} instead of {real_branch} "
                           f"as the reference for {repo}")
                 break
             except GerritExceptions.UnknownBranch:
@@ -270,9 +280,11 @@ def get_head(config: Config, repo: Union[Repo, str], pull_head: bool = False) ->
         if not branch_head:
             if not config.suppress_warn:
                 print(f"Exhausted branch options for {repo}! Tried {branches}")
-                return ""
+                return "", ""
         else:
-            return branch_head.revision
+            # Fetch the commit details for the branch head
+            branch_head = gerrit.projects.get(repo.id).get_commit(branch_head.revision)
+            return branch_head.commit, branch_head.subject
 
 
 def get_top_integration_sha(config, repo: Repo) -> str:
@@ -298,22 +310,26 @@ def get_top_integration_sha(config, repo: Repo) -> str:
                     integration_id = url.split("/")[-1]  # Get just the integration ID
                     break
             break
+    sha = ""
+    print(f"Looking for integration sha for {repo.proposal.change_id} from {repo.id}")
     if integration_id:
-        r = requests.get(f"https://testresults.qt.io/coin/api/integration/{repo.id}/tasks/{integration_id}")
+        r = requests.get(f"https://testresults.qt.io/coin/api/taskDetail?id={integration_id}")
         if r.status_code == 200:
-            sha = json.loads(r.text)[4]["1"]["rec"]["6"]["str"]
-            print(f"Found integration sha {sha} from Integration ID: {integration_id}")
-            return sha
-        else:
-            # Fallback to internal COIN if available. The task probably hadn't replicated to
-            # testresults yet.
-            try:
-                r = requests.get(f"http://{config.INTERNAL_COIN_HOST}/coin/api/integration/"
-                                 f"{repo.id}/tasks/{integration_id}")
-            except requests.exceptions.ConnectionError:
-                pass
-            if r.status_code == 200:
-                sha = json.loads(r.text)[4]["1"]["rec"]["6"]["str"]
+            data = json.loads(r.text)
+            tasks = data.get("tasks")
+            if tasks and len(tasks) > 0:
+                sha = data["tasks"].pop()["final_sha"]
+                print(f"Found integration sha {sha} from Integration ID: {integration_id}")
+                return sha
+
+        # Fallback to internal COIN if available. The task probably hadn't replicated to
+        # testresults yet.
+        r = requests.get(f"{config.COIN_INTERNAL_HOST}/coin/api/taskDetail?id={integration_id}")  # Update to use config variable
+        if r.status_code == 200:
+            data = json.loads(r.text)
+            tasks = data.get("tasks")
+            if tasks and len(tasks) > 0:
+                sha = data["tasks"].pop()["final_sha"]
                 print(f"Found integration sha {sha} from Integration ID: {integration_id}")
                 return sha
         print(f"ERROR: Failed to retrieve integration sha from testresults/coin for integration ID"
@@ -478,7 +494,7 @@ def get_dependencies_yaml(config, repo: Repo, fetch_head: bool = False) -> tuple
         if repo.id in config.state_data.keys():
             print(f"Using state data for {repo.id}")
             return config.state_data[repo.id].deps_yaml, config.state_data[repo.id].branch
-        qt5_repo_sha = get_head(config, repo)
+        qt5_repo_sha = get_head(config, repo)[0]
         try:
             r = gerrit.projects.get(repo.id).get_commit(qt5_repo_sha).get_file_content(
                 'dependencies.yaml')
@@ -723,7 +739,8 @@ def push_submodule_update(config: Config, repo: Repo, retry: bool = False) -> Pr
 
     current_head_deps, _ = get_dependencies_yaml(config, repo, fetch_head=True)
     if current_head_deps == repo.proposal.proposed_yaml:
-        repo.proposal.merged_ref = get_head(config, repo, pull_head=True)
+        repo.proposal.merged_ref, repo.proposal.original_message = get_head(config, repo,
+                                                                            pull_head=True)
         repo.proposal.change_id = ""
         repo.proposal.change_number = ""
         print(f"Branch head for {repo.id} is already up-to-date! Not pushing an update!")
@@ -758,7 +775,7 @@ def push_submodule_update(config: Config, repo: Repo, retry: bool = False) -> Pr
         print(f"Rebased change {change.change_id}")
     except GerritExceptions.ConflictError:
         if (not change.get_revision("current").get_commit().parents[0]["commit"]
-                == get_head(config, repo, True)):
+                == get_head(config, repo, True)[0]):
             print("WARN: Failed to rebase change due to conflicts."
                   " Abandoning and recreating the change.")
             # Failed to rebase because of conflicts
@@ -901,7 +918,7 @@ def push_supermodule_update(config: Config, retry: bool = False) -> Repo:
         print(f"Rebased change {change.change_id}")
     except GerritExceptions.ConflictError:
         if (not change.get_revision("current").get_commit().parents[0]["commit"]
-                == get_head(config, qt5_repo, True)):
+                == get_head(config, qt5_repo, True)[0]):
             print("WARN: Failed to rebase change due to conflicts."
                   " Abandoning and recreating the change.")
             # Failed to rebase because of conflicts
@@ -1034,7 +1051,7 @@ def push_yocto_update(config: Config, retry: bool = False) -> Repo:
                     pinned_submodule_sha = search_pinned_submodule(config, module_repo,
                                                                    submodule_repo)
                 if not pinned_submodule_sha:
-                    print(f"Couldn't find a submodule named {submodule_repo.id}"
+                    print(f"Couldn't find a submodule named {repo_name_maybe_submodule}"
                           f' in {module_repo.id}. Trying raw submodule name: "{submodule_name}"')
                     pinned_submodule_sha = search_pinned_submodule(config, module_repo,
                                                                    submodule_name)
@@ -1050,7 +1067,7 @@ def push_yocto_update(config: Config, retry: bool = False) -> Repo:
             module_name = repo_name_maybe_submodule
         module_repo = search_for_repo(config, module_name)
         if not module_repo.original_ref:
-            module_repo.original_ref = get_head(config, module_repo)
+            module_repo.original_ref, module_repo.original_message = get_head(config, module_repo)
         if pinned_submodule_sha:
             file_lines[i] = line.replace(sha, f'"{pinned_submodule_sha}"')
         else:
@@ -1099,7 +1116,7 @@ def push_yocto_update(config: Config, retry: bool = False) -> Repo:
         change.get_revision("current").rebase({"base": ""})
         print(f"Rebased change {change.change_id}")
     except GerritExceptions.ConflictError:
-        if not change.get_revision("current").get_commit().parents[0]["commit"] == get_head(config, yocto_repo, True):
+        if not change.get_revision("current").get_commit().parents[0]["commit"] == get_head(config, yocto_repo, True)[0]:
             print("WARN: Failed to rebase change due to conflicts."
                   " Abandoning and recreating the change.")
             # Failed to rebase because of conflicts
@@ -1166,7 +1183,8 @@ def acquire_change_edit(config: Config,
             "project": repo.id,
             "subject": subject,
             "branch": repo.branch,
-            "status": "NEW"
+            "status": "NEW",
+            "topic": repo.topic
         })
         print(f"Created new change for {repo.id}: {change.change_id}")
         repo.proposal.change_id = change.change_id
@@ -1230,7 +1248,7 @@ def reset_module_properties(config: Config, repo: Repo) -> Repo:
     repo.stage_count = 0
     repo.retry_count = 0
     repo.to_stage = list()
-    repo.original_ref = get_head(config, repo, True)
+    repo.original_ref, repo.original_message = get_head(config, repo, True)
     return repo