Change state storage mechanism

Store the module update batch state in
refs/personal/<user>/state/<branch> in gerrit instead of in a local
file. That allows running the bot manually and interact well with an
automated run.

Change-Id: I10cf43e1c77bc5568c0cdb99c8fabd6fc4bd2f64
Reviewed-by: Frederik Gladhorn <frederik.gladhorn@qt.io>

4 files changed, 142 insertions, 8 deletions

diff --git a/src/qtmoduleupdater/README.md b/src/qtmoduleupdater/README.md
index 33b60d92..04d7051d 100644
--- a/src/qtmoduleupdater/README.md
+++ b/src/qtmoduleupdater/README.md
@@ -10,7 +10,7 @@ This tool automates the pushing of updates through the graph of dependencies and
 
 ## Algorithm
 
-The process of updating dependencies starts by collecting a list of all repositories and determining the root of the graph. That's typically qtbase. From there on, updates to all repositories are posted that only depend on the root. All other repositories remain in a "todo" list. The root is remembered in a "done" list and all repositories that we are currently trying to bring up-to-date are in a "pending" list. Once this process is started, the program saves its state in a ```.json``` file and terminates.
+The process of updating dependencies starts by collecting a list of all repositories and determining the root of the graph. That's typically qtbase. From there on, updates to all repositories are posted that only depend on the root. All other repositories remain in a "todo" list. The root is remembered in a "done" list and all repositories that we are currently trying to bring up-to-date are in a "pending" list. Once this process is started, the program saves its state in personal branch under refs/personal/qt_submodule_updater_bot/state/<branch> and terminates.
 
 The next time the Qt Module Updater is started, it resumes the state and begins checking the state of all pending updates. If an update succeeded, then the corresponding repository is added to the "done" list and we can prepare updates for repositories that have now their dependencies satisfied by picking them from the "todo" list. If the update failed, the repository is dropped from the batch of updates and all other repositories that directly or indirectly depend on the failed one are also removed. After every such iteration of processing pending updates and pushing new ones to Gerrit, the process terminates and saves its state.
 
diff --git a/src/qtmoduleupdater/gerrit.go b/src/qtmoduleupdater/gerrit.go
index 346ac1fa..d37deeb5 100644
--- a/src/qtmoduleupdater/gerrit.go
+++ b/src/qtmoduleupdater/gerrit.go
@@ -224,7 +224,7 @@ func (instance *gerritInstance) pushChange(repoPath string, branch string, commi
 		pushURL.User = url.User(instance.pushUserName)
 	}
 
-	return repo.Push(pushURL, commitID, "refs/for/"+branch)
+	return repo.Push(pushURL, nil, commitID, "refs/for/"+branch)
 }
 
 func (instance *gerritInstance) reviewAndStageChange(repoPath string, branch string, commitID OID, summary string) error {
diff --git a/src/qtmoduleupdater/moduleupdatebatch.go b/src/qtmoduleupdater/moduleupdatebatch.go
index e779847a..e65043b2 100644
--- a/src/qtmoduleupdater/moduleupdatebatch.go
+++ b/src/qtmoduleupdater/moduleupdatebatch.go
@@ -28,9 +28,11 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"log"
+	"net/url"
 	"os"
 	"strings"
 )
@@ -63,7 +65,7 @@ func newModuleUpdateBatch(product string, branch string, productRef string) (*Mo
 	}
 	var err error
 
-	err = batch.loadState()
+	err = batch.loadStateFromCommit()
 	if os.IsNotExist(err) {
 		err = batch.loadTodoList()
 		if err != nil {
@@ -189,6 +191,136 @@ func (batch *ModuleUpdateBatch) stateFileName() string {
 	return fmt.Sprintf("state_%s_%s.json", sanitizeBranchOrRepo(batch.Product), sanitizeBranchOrRepo(batch.Branch))
 }
 
+func (batch *ModuleUpdateBatch) saveStateAsCommit(gerrit *gerritInstance) error {
+	productRepo, err := OpenRepository(batch.Product)
+	if err != nil {
+		return fmt.Errorf("Error opening product repo: %s", err)
+	}
+
+	index, err := productRepo.NewIndex()
+	if err != nil {
+		return fmt.Errorf("Error creating temporary index for saving batch state: %s", err)
+	}
+	defer index.Free()
+
+	jsonBuffer := &bytes.Buffer{}
+	encoder := json.NewEncoder(jsonBuffer)
+	encoder.SetIndent("", "    ")
+	if err = encoder.Encode(batch); err != nil {
+		return fmt.Errorf("Error serializing module update batch state to json: %s", err)
+	}
+
+	indexEntry := &IndexEntry{
+		Permissions: "100644",
+		Path:        "state.json",
+	}
+
+	if err = index.HashObject(indexEntry, jsonBuffer.Bytes()); err != nil {
+		return fmt.Errorf("Error adding json serialized module update batch state to git database: %s", err)
+	}
+
+	if err = index.Add(indexEntry); err != nil {
+		return fmt.Errorf("Error adding json serialized module update batch state to git index: %s", err)
+	}
+
+	tree, err := index.WriteTree()
+	if err != nil {
+		return fmt.Errorf("Error writing tree object with module update batch state: %s", err)
+	}
+
+	commit, err := productRepo.CommitTree(tree, "Module update batch state")
+	if err != nil {
+		return fmt.Errorf("Error writing commit for module update batch state: %s", err)
+	}
+
+	log.Println("Module state saved as commit", commit)
+
+	pushURL, err := RepoURL(batch.Product)
+	if err != nil {
+		return fmt.Errorf("Error determining %s repo URL: %s", batch.Product, err)
+	}
+
+	if gerrit.pushUserName != "" {
+		pushURL.User = url.User(gerrit.pushUserName)
+	}
+
+	targetRef := "refs/personal/" + pushURL.User.Username() + "/state/" + batch.Branch
+
+	log.Printf("Saving batch state to %s\n", targetRef)
+
+	return productRepo.Push(pushURL, []string{"-f"}, commit, targetRef)
+}
+
+func (batch *ModuleUpdateBatch) loadStateFromCommit() error {
+	productRepo, err := OpenRepository(batch.Product)
+	if err != nil {
+		return fmt.Errorf("Error opening product repo: %s", err)
+	}
+
+	repoURL, err := RepoURL(batch.Product)
+	if err != nil {
+		return fmt.Errorf("Error determining %s repo URL: %s", batch.Product, err)
+	}
+
+	log.Printf("Fetching state.json from personal branch")
+
+	// ### url
+	stateCommit, err := productRepo.Fetch(repoURL, "refs/personal/qt_submodule_update_bot/state/"+batch.Branch)
+	if err != nil {
+		return os.ErrNotExist
+	}
+
+	index, err := productRepo.NewIndex()
+	if err != nil {
+		return fmt.Errorf("Error creating git index when trying to load batch state: %s", err)
+	}
+	defer index.Free()
+
+	if err = index.ReadTree(stateCommit); err != nil {
+		return fmt.Errorf("Error reading tree of state commit %s: %s", stateCommit, err)
+	}
+
+	indexEntry, err := lookupPathIndexEntry(index, "state.json")
+	if err != nil {
+		return fmt.Errorf("Error looking up state.json from index in state commit %s: %s", stateCommit, err)
+	}
+
+	stateJSON, err := productRepo.LookupBlob(indexEntry.ID)
+	if err != nil {
+		return fmt.Errorf("Error reading state json from git db: %s", err)
+	}
+
+	decoder := json.NewDecoder(bytes.NewBuffer(stateJSON))
+	err = decoder.Decode(batch)
+	if err != nil {
+		return fmt.Errorf("Error decoding JSON state file: %s", err)
+	}
+
+	return nil
+}
+
+func (batch *ModuleUpdateBatch) clearStateCommit(gerrit *gerritInstance) error {
+	productRepo, err := OpenRepository(batch.Product)
+	if err != nil {
+		return fmt.Errorf("Error opening product repo: %s", err)
+	}
+
+	pushURL, err := RepoURL(batch.Product)
+	if err != nil {
+		return fmt.Errorf("Error determining %s repo URL: %s", batch.Product, err)
+	}
+
+	if gerrit.pushUserName != "" {
+		pushURL.User = url.User(gerrit.pushUserName)
+	}
+
+	targetRef := "refs/personal/" + pushURL.User.Username() + "/state/" + batch.Branch
+
+	log.Printf("Clearing batch state at %s\n", targetRef)
+
+	return productRepo.Push(pushURL, nil, "", targetRef)
+}
+
 func (batch *ModuleUpdateBatch) saveState() error {
 	fileName := batch.stateFileName()
 	outputFile, err := os.Create(fileName)
@@ -273,7 +405,7 @@ func (batch *ModuleUpdateBatch) runOneIteration(gerrit *gerritInstance) error {
 	batch.printSummary()
 
 	if !batch.isDone() {
-		err := batch.saveState()
+		err := batch.saveStateAsCommit(gerrit)
 		if err != nil {
 			return err
 		}
@@ -285,7 +417,7 @@ func (batch *ModuleUpdateBatch) runOneIteration(gerrit *gerritInstance) error {
 			}
 		}
 
-		batch.clearState()
+		batch.clearStateCommit(gerrit)
 	}
 
 	return nil
diff --git a/src/qtmoduleupdater/repo.go b/src/qtmoduleupdater/repo.go
index 7e1eb8a3..a347c83c 100644
--- a/src/qtmoduleupdater/repo.go
+++ b/src/qtmoduleupdater/repo.go
@@ -252,10 +252,12 @@ func (repo Repository) Fetch(url *url.URL, refSpec string) (sha1 OID, err error)
 	return ref, nil
 }
 
-// Push is a wrapper around the git push commit.
-func (repo Repository) Push(url *url.URL, commit OID, targetRef string) error {
+// Push is a wrapper around the git push commit, similar to the Push() function
+// but allowing additional options to be passed to the git push invocation.
+func (repo Repository) Push(url *url.URL, options []string, commit OID, targetRef string) error {
 	refSpec := fmt.Sprintf("%s:%s", commit, targetRef)
-	_, err := repo.gitCommand("push", url.String(), refSpec).Run()
+	options = append(options, url.String(), refSpec)
+	_, err := repo.gitCommand("push", options...).Run()
 	return err
 }