Initial import of Qt Module Updater tool

This tool serves the purpose of updating the dependencies.yaml files in
repositories.

Change-Id: I9b428566a4a29eda3ceecc68accfa04bd98d66cb
Reviewed-by: Aapo Keskimolo <aapo.keskimolo@qt.io>

15 files changed, 2202 insertions, 0 deletions

diff --git a/src/qtmoduleupdater/.gitignore b/src/qtmoduleupdater/.gitignore
new file mode 100644
index 00000000..6a9c77fe
--- /dev/null
+++ b/src/qtmoduleupdater/.gitignore
@@ -0,0 +1,4 @@
+git-repos
+.vscode
+qtmoduleupdater
+state_*.json
diff --git a/src/qtmoduleupdater/README.md b/src/qtmoduleupdater/README.md
new file mode 100644
index 00000000..33b60d92
--- /dev/null
+++ b/src/qtmoduleupdater/README.md
@@ -0,0 +1,32 @@
+# Qt Module Updater
+
+This is a tool that serves the purpose of automating the process of keeping pinned dependencies between Qt git repositories up-to-date.
+
+Qt modules in git repositories depend on modules from other repositories and therefore each repository encodes its dependencies to other repositories using a configuration file called ```dependencies.yaml```. It lists the required and optional dependencies as well as the commit sha1s that are known to work.
+
+All repositories with their dependencies form a graph, with qtbase typically at the root. A newer version of qtbase shall result in a change to ```dependencies.yaml``` in qtsvg. Once approved by the CI system, a change to qtdeclarative is needed to pull in the newer version of qtsvg and implicitly qtbase.
+
+This tool automates the pushing of updates through the graph of dependencies and once all modules of qt5.git are complete, an update of submodule sha1s to qt5.git will be posted.
+
+## Algorithm
+
+The process of updating dependencies starts by collecting a list of all repositories and determining the root of the graph. That's typically qtbase. From there on, updates to all repositories are posted that only depend on the root. All other repositories remain in a "todo" list. The root is remembered in a "done" list and all repositories that we are currently trying to bring up-to-date are in a "pending" list. Once this process is started, the program saves its state in a ```.json``` file and terminates.
+
+The next time the Qt Module Updater is started, it resumes the state and begins checking the state of all pending updates. If an update succeeded, then the corresponding repository is added to the "done" list and we can prepare updates for repositories that have now their dependencies satisfied by picking them from the "todo" list. If the update failed, the repository is dropped from the batch of updates and all other repositories that directly or indirectly depend on the failed one are also removed. After every such iteration of processing pending updates and pushing new ones to Gerrit, the process terminates and saves its state.
+
+When the todo list is empty and there are no more pending updates, the batch update is complete. If during that update there were no failures, the Qt Module Updater will also push a change to qt5.git with an update to all submodule sha1s of the new consistent set of modules.
+
+## Usage
+
+The Qt Module Updater is written in Golang and requires at least version 1.13. To build the program, simply clone the repository and run
+
+    go build
+
+When running the program, git repositories will be cloned from Qt's Gerrit instance and stored as bare clones in the ```git-repos``` sub-directory.
+
+Every invocation requires passing a ```-branch=``` that specifies the Qt version branch to use as reference. By default, repositories from ```qt/qt5``` are picked up, but it is possible to override this with the ```-product=``` parameter.
+
+When run in a production environment, it is desirable to pass the ```-stage-as-bot``` parameter, to ensure that changes are pushed as the special Qt Submodule Update bot.
+
+For manual testing, it is also possible to use the ```-manual-stage``` parameter to merely push changes to Gerrit but not automatically stage them.
+
diff --git a/src/qtmoduleupdater/TODO.md b/src/qtmoduleupdater/TODO.md
new file mode 100644
index 00000000..40f07e8f
--- /dev/null
+++ b/src/qtmoduleupdater/TODO.md
@@ -0,0 +1,4 @@
+# A brief list of pending things that need to be implemented
+
+* Be more lenient towards failures during the attempt to update dependencies.yml in a module:
+    * We could keep "failed" updates in the pending list, but we have to be careful about terminating the overall algorithm. Might be as simple as considering remaining pending changes as all failed if the todo list is empty otherwise.
diff --git a/src/qtmoduleupdater/dependenciesupdateresultenum_string.go b/src/qtmoduleupdater/dependenciesupdateresultenum_string.go
new file mode 100644
index 00000000..db80b97c
--- /dev/null
+++ b/src/qtmoduleupdater/dependenciesupdateresultenum_string.go
@@ -0,0 +1,25 @@
+// Code generated by "stringer -type=DependenciesUpdateResultEnum"; DO NOT EDIT.
+
+package main
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[DependenciesUpdateDependencyMissing-0]
+	_ = x[DependenciesUpdateContentUpToDate-1]
+	_ = x[DependenciesUpdateUpdateScheduled-2]
+}
+
+const _DependenciesUpdateResultEnum_name = "DependenciesUpdateDependencyMissingDependenciesUpdateContentUpToDateDependenciesUpdateUpdateScheduled"
+
+var _DependenciesUpdateResultEnum_index = [...]uint8{0, 35, 68, 101}
+
+func (i DependenciesUpdateResultEnum) String() string {
+	if i < 0 || i >= DependenciesUpdateResultEnum(len(_DependenciesUpdateResultEnum_index)-1) {
+		return "DependenciesUpdateResultEnum(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _DependenciesUpdateResultEnum_name[_DependenciesUpdateResultEnum_index[i]:_DependenciesUpdateResultEnum_index[i+1]]
+}
diff --git a/src/qtmoduleupdater/gerrit.go b/src/qtmoduleupdater/gerrit.go
new file mode 100644
index 00000000..d3ef67f5
--- /dev/null
+++ b/src/qtmoduleupdater/gerrit.go
@@ -0,0 +1,260 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the repo tools module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net"
+	"net/url"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// GerritPatchSet corresponds to the patch set JSON object returned by Gerrit's JSON API.
+type GerritPatchSet struct {
+	Number         int             `json:"number"`
+	Revision       string          `json:"revision"`
+	Parents        []string        `json:"parents"`
+	Uploader       json.RawMessage `json:"uploader"`
+	CreatedOn      uint            `json:"createdOn"`
+	Author         json.RawMessage `json:"author"`
+	SizeInsertions int             `json:"sizeInsertions"`
+	SizeDeletions  int             `json:"sizeDeletions"`
+}
+
+// GerritChangeOrStats corresponds to the JSON data returned by Gerrit's Query JSON API.
+type GerritChangeOrStats struct {
+	Type                string `json:"type"`
+	RowCount            int    `json:"rowCount"`
+	RunTimeMilliseconds int    `json:"runTimeMilliseconds"`
+
+	Project     string           `json:"project"`
+	Branch      string           `json:"branch"`
+	ID          string           `json:"id"`
+	Number      int              `json:"number"`
+	Subject     string           `json:"subject"`
+	Owner       json.RawMessage  `json:"owner"`
+	URL         string           `json:"url"`
+	CreatedOn   uint             `json:"createdOn"`
+	LastUpdated uint             `json:"lastUpdated"`
+	SortKey     string           `json:"sortKey"`
+	Open        bool             `json:"open"`
+	Status      string           `json:"status"`
+	PatchSets   []GerritPatchSet `json:"patchSets"`
+}
+
+func gerritSSHCommand(gerritURL url.URL, arguments ...string) (*exec.Cmd, error) {
+	user := os.Getenv("GIT_SSH_USER")
+	if user != "" {
+		gerritURL.User = url.User(user)
+	}
+
+	host, port, err := net.SplitHostPort(gerritURL.Host)
+	if err != nil {
+		return nil, fmt.Errorf("Error splitting host and port from gerrit URL: %s", err)
+	}
+
+	userAtHost := host
+	if gerritURL.User != nil {
+		userAtHost = gerritURL.User.Username() + "@" + host
+	}
+
+	newArgs := []string{"-oBatchMode=yes", userAtHost, "-p", port}
+	newArgs = append(newArgs, arguments...)
+	ssh := os.Getenv("GIT_SSH")
+	if ssh == "" {
+		ssh = "ssh"
+	}
+	log.Printf("Running gerrit ssh command: 'ssh %v'\n", newArgs)
+	return exec.Command(ssh, newArgs...), nil
+}
+
+func getGerritChangeStatus(project string, branch string, changeID string) (status string, err error) {
+	gerritURL, err := RepoURL(project)
+	if err != nil {
+		return "", fmt.Errorf("Error parsing gerrit URL: %s", err)
+	}
+	queryString := fmt.Sprintf(`project:%s branch:%s %s`, project, branch, changeID)
+	cmd, err := gerritSSHCommand(*gerritURL, "gerrit", "query", "--patch-sets", "--format JSON", queryString)
+	if err != nil {
+		return "", err
+	}
+	output, err := cmd.Output()
+	if err != nil {
+		return "", fmt.Errorf("Error running gerrit query command: %s", err)
+	}
+
+	var id string
+
+	for _, line := range strings.Split(string(output), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+
+		var field GerritChangeOrStats
+		err = json.Unmarshal([]byte(line), &field)
+		if err != nil {
+			return "", fmt.Errorf("Error reading gerrit json response: %s:%s", err, string(output))
+		}
+		if field.Type == "stats" {
+			if field.RowCount != 1 {
+				return "", fmt.Errorf("unexpected row count %v when querying for existing gerrit change", field.RowCount)
+			}
+			continue
+		}
+
+		if field.Project != project {
+			return "", fmt.Errorf("unexpectedly found change for a different project. Received %s, expected %s for %s", field.Project, project, changeID)
+		}
+		if id != "" {
+			return "", fmt.Errorf("unexpectedly found multiple changes for change ID %s", changeID)
+		}
+		id = field.ID
+		status = field.Status
+	}
+	return status, nil
+}
+
+func getExistingChange(project string, branch string) (gerritChangeID string, changeNumber int, patchSetNr int, err error) {
+	gerritURL, err := RepoURL(project)
+	if err != nil {
+		return "", 0, 0, fmt.Errorf("Error parsing gerrit URL: %s", err)
+	}
+	queryString := fmt.Sprintf(`project:%s branch:%s status:open owner:self message:{Update dependencies on \'%s\' in %s}`, project, branch, branch, project)
+	cmd, err := gerritSSHCommand(*gerritURL, "gerrit", "query", "--patch-sets", "--format JSON", queryString)
+	if err != nil {
+		return "", 0, 0, err
+	}
+	output, err := cmd.Output()
+	if err != nil {
+		return "", 0, 0, fmt.Errorf("Error running gerrit query command: %s", err)
+	}
+
+	var id string
+
+	for _, line := range strings.Split(string(output), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+
+		var field GerritChangeOrStats
+		err = json.Unmarshal([]byte(line), &field)
+		if err != nil {
+			return "", 0, 0, fmt.Errorf("Error reading gerrit json response: %s:%s", err, string(output))
+		}
+		if field.Type == "stats" {
+			if field.RowCount == 0 {
+				return "", 0, 0, nil
+			}
+			if field.RowCount != 1 {
+				return "", 0, 0, fmt.Errorf("unexpected row count %v when querying for existing gerrit changes", field.RowCount)
+			}
+			continue
+		}
+
+		if field.Project == project {
+			if id != "" {
+				return "", 0, 0, fmt.Errorf("unexpectedly found multiple changes for submodule updates: Id %s and %s", id, field.ID)
+			}
+			id = field.ID
+			changeNumber = field.Number
+			patchSetNr = 0
+			for _, patchSet := range field.PatchSets {
+				if patchSet.Number > patchSetNr {
+					patchSetNr = patchSet.Number
+				}
+			}
+			continue
+		}
+	}
+	return id, changeNumber, patchSetNr, nil
+}
+
+func escapeGerritMessage(message string) string {
+	replacer := strings.NewReplacer(`\`, `\\`, `"`, `\"`, `'`, `\'`)
+	return `"` + replacer.Replace(message) + `"`
+}
+
+func pushAndStageChange(repoPath string, branch string, commitID OID, summary string, pushUserName string, manualStage bool) error {
+	repo, err := OpenRepository(repoPath)
+	if err != nil {
+		return err
+	}
+
+	pushURL, err := RepoURL(repoPath)
+	if err != nil {
+		return err
+	}
+	if pushUserName != "" {
+		pushURL.User = url.User(pushUserName)
+	}
+
+	err = repo.Push(pushURL, commitID, "refs/for/"+branch)
+	if err != nil {
+		return err
+	}
+
+	reviewArgs := []string{"gerrit", "review", string(commitID)}
+
+	if summary != "" {
+		reviewArgs = append(reviewArgs, "-m", escapeGerritMessage(summary))
+	}
+	// Pass in sanity review, since the sanity bot runs only after a delay and thus the commit will get refused.
+	reviewArgs = append(reviewArgs, "--code-review", "2", "--sanity-review", "1")
+
+	updateCommand, err := gerritSSHCommand(*pushURL, reviewArgs...)
+	if err != nil {
+		return err
+	}
+	updateCommand.Stdout = os.Stdout
+	updateCommand.Stderr = os.Stderr
+	if err = updateCommand.Run(); err != nil {
+		return err
+	}
+
+	if manualStage {
+		return nil
+	}
+
+	stageArgs := []string{"gerrit-plugin-qt-workflow", "stage", string(commitID)}
+	updateCommand, err = gerritSSHCommand(*pushURL, stageArgs...)
+	if err != nil {
+		return err
+	}
+	updateCommand.Stdout = os.Stdout
+	updateCommand.Stderr = os.Stderr
+	if err = updateCommand.Run(); err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/src/qtmoduleupdater/go.mod b/src/qtmoduleupdater/go.mod
new file mode 100644
index 00000000..49d21999
--- /dev/null
+++ b/src/qtmoduleupdater/go.mod
@@ -0,0 +1,9 @@
+module qtmoduleupdater
+
+go 1.13
+
+require (
+	github.com/stretchr/testify v1.4.0
+	github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec
+	gopkg.in/yaml.v2 v2.2.2
+)
diff --git a/src/qtmoduleupdater/go.sum b/src/qtmoduleupdater/go.sum
new file mode 100644
index 00000000..e3ba6cca
--- /dev/null
+++ b/src/qtmoduleupdater/go.sum
@@ -0,0 +1,13 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks=
+github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/src/qtmoduleupdater/main.go b/src/qtmoduleupdater/main.go
new file mode 100644
index 00000000..9e5a6574
--- /dev/null
+++ b/src/qtmoduleupdater/main.go
@@ -0,0 +1,173 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the repo tools module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+)
+
+func setupEnvironmentForSubmoduleUpdateBot() (cleanupFunction func(), username string, err error) {
+	cleanupFunction = func() {}
+
+	submoduleUpdateBotKeyPath := "submodule_update_bot_key_rsa"
+	if _, err = os.Stat(submoduleUpdateBotKeyPath); os.IsNotExist(err) {
+		err = fmt.Errorf("cannot locate submodule update bot SSH key file. Please copy it from the coin secrets repo into the current directory")
+		return
+	}
+
+	var sshWrapperScript *os.File
+
+	cleanupFunction = func() {
+		if sshWrapperScript != nil {
+			os.Remove(sshWrapperScript.Name())
+		}
+	}
+
+	sshWrapperScript, err = ioutil.TempFile("", "")
+	if err != nil {
+		err = fmt.Errorf("Error creating temporary SSH wrapper script: %s", err)
+		return
+	}
+	if err = sshWrapperScript.Chmod(0700); err != nil {
+		sshWrapperScript.Close()
+		err = fmt.Errorf("Error making temporary SSH wrapper script executable: %s", err)
+		return
+	}
+	sshWrapperScript.Close()
+
+	scriptSource := fmt.Sprintf("#!/bin/sh\nexec ssh -i %s \"$@\"", submoduleUpdateBotKeyPath)
+	if err = ioutil.WriteFile(sshWrapperScript.Name(), []byte(scriptSource), 0700); err != nil {
+		err = fmt.Errorf("Error writing to temporary SSH wrapper script: %s", err)
+		return
+	}
+	os.Setenv("GIT_SSH", sshWrapperScript.Name())
+	os.Setenv("GIT_SSH_USER", "qt_submodule_update_bot")
+
+	os.Setenv("GIT_AUTHOR_NAME", "Qt Submodule Update Bot")
+	os.Setenv("GIT_COMMITTER_NAME", "Qt Submodule Update Bot")
+	os.Setenv("GIT_AUTHOR_EMAIL", "qt_submodule_update_bot@qt-project.org")
+	os.Setenv("GIT_COMMITTER_EMAIL", "qt_submodule_update_bot@qt-project.org")
+
+	username = "qt_submodule_update_bot"
+	return
+}
+
+func appMain() error {
+	var product string
+	flag.StringVar(&product, "product", "qt/qt5" /*default*/, "Product repository to use as reference and push completed updates to")
+	stageAsBot := false
+	flag.BoolVar(&stageAsBot, "stage-as-bot", false /*default*/, "Push changes to Gerrit using the submodule update bot account")
+	var branch string
+	flag.StringVar(&branch, "branch", "", "Branch to update")
+	var fetchRef string
+	flag.StringVar(&fetchRef, "fetch-ref", "", "Git ref in qt5 to use as basis for a new round of updates")
+	manualStage := false
+	flag.BoolVar(&manualStage, "manual-stage", false /*default*/, "Do not stage changes automatically")
+	summaryOnly := false
+	flag.BoolVar(&summaryOnly, "summarize", false /*default*/, "")
+	verbose := false
+	flag.BoolVar(&verbose, "verbose", false /*default*/, "Enable verbose logging output")
+	flag.Parse()
+
+	if !verbose {
+		oldWriter := log.Writer()
+		defer log.SetOutput(oldWriter)
+		log.SetOutput(ioutil.Discard)
+	}
+
+	if branch == "" {
+		return fmt.Errorf("missing branch. Please specify -branch=<name of branch>")
+	}
+
+	var pushUserName string
+	if stageAsBot {
+		var cleaner func()
+		var err error
+		cleaner, pushUserName, err = setupEnvironmentForSubmoduleUpdateBot()
+		if err != nil {
+			return fmt.Errorf("error preparing environment to work as submodule-update user: %s", err)
+		}
+		defer cleaner()
+	}
+
+	batch := &ModuleUpdateBatch{
+		Product: product,
+		Branch:  branch,
+	}
+	var err error
+
+	err = batch.loadState()
+	if os.IsNotExist(err) {
+		err = batch.loadTodoList(fetchRef)
+		if err != nil {
+			return err
+		}
+	}
+
+	if summaryOnly {
+		batch.PrintSummary()
+		return nil
+	}
+
+	batch.checkPendingModules()
+
+	if err := batch.scheduleUpdates(pushUserName, manualStage); err != nil {
+		return err
+	}
+
+	batch.PrintSummary()
+
+	if !batch.isDone() {
+		err = batch.saveState()
+		if err != nil {
+			return err
+		}
+	} else {
+		os.Remove("state.json")
+
+		if batch.FailedModuleCount == 0 {
+			fmt.Println("Preparing qt5 update")
+			if err = prepareQt5Update(product, batch.Branch, batch.Done, pushUserName, manualStage); err != nil {
+				return fmt.Errorf("error preparing qt5 update: %s", err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func main() {
+	err := appMain()
+	if err != nil {
+		log.Fatalf("Error: %s\n", err)
+	}
+}
diff --git a/src/qtmoduleupdater/module.go b/src/qtmoduleupdater/module.go
new file mode 100644
index 00000000..7fa22beb
--- /dev/null
+++ b/src/qtmoduleupdater/module.go
@@ -0,0 +1,452 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the repo tools module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+// Module represents a git repository with a dependencies.yaml file
+// that needs updating.
+type Module struct {
+	RepoPath             string   // relative path in Gerrit, such as qt/qtsvg
+	RequiredDependencies []string // Dependencies as per dependencies.yaml
+	OptionalDependencies []string // Dependencies as per dependencies.yaml
+	Branch               string
+	Tip                  OID
+}
+
+// YAMLModule for unmarshaling module information from dependencies file.
+type YAMLModule struct {
+	Ref      string `yaml:"ref"`
+	Required bool   `yaml:"required"`
+}
+
+// YAMLDependenciesMap is a map from string to YAMLModule but always
+// produces a sorted YAML map when serializing
+type YAMLDependenciesMap map[string]*YAMLModule
+
+// YAMLDependencies for unmarshaling module information from dependencies file.
+type YAMLDependencies struct {
+	Dependencies YAMLDependenciesMap `yaml:"dependencies"`
+}
+
+// MarshalYAML implements the marshalling of the dependencies while
+// making sure the entries are sorted.
+func (depMap *YAMLDependenciesMap) MarshalYAML() (interface{}, error) {
+	var sortedKeys []string
+	for key := range *depMap {
+		sortedKeys = append(sortedKeys, key)
+	}
+
+	sort.Strings(sortedKeys)
+
+	var result yaml.MapSlice
+
+	for _, key := range sortedKeys {
+		entry := (*depMap)[key]
+		result = append(result, yaml.MapItem{
+			Key:   key,
+			Value: entry,
+		})
+	}
+
+	return result, nil
+}
+
+//go:generate stringer -type=DependenciesUpdateResultEnum
+
+// DependenciesUpdateResultEnum describes the different states after attempting to update the dependencies.yaml for a module.
+type DependenciesUpdateResultEnum int
+
+const (
+	// DependenciesUpdateDependencyMissing indicates that a dependency is not available yet.
+	DependenciesUpdateDependencyMissing DependenciesUpdateResultEnum = iota
+	// DependenciesUpdateContentUpToDate indicates that no further updates to dependencies.yaml are required.
+	DependenciesUpdateContentUpToDate
+	// DependenciesUpdateUpdateScheduled indicates that an update to dependencies.yaml was necessary and has been pushed to Gerrit.
+	DependenciesUpdateUpdateScheduled
+)
+
+type pathNotExistError struct {
+	path string
+}
+
+func (p *pathNotExistError) Error() string {
+	return fmt.Sprintf("Could not locate %s in git tree", p.path)
+}
+
+func readDependenciesYAML(repoPath string, repo Repository, commit OID) (dependencies *YAMLDependencies, err error) {
+	path := "dependencies.yaml"
+
+	tree, err := repo.ListTree(commit)
+	if err != nil {
+		return nil, fmt.Errorf("could not list tree for commit %s: %s", commit, err)
+	}
+
+	entry, ok := tree.Entries[path]
+	if !ok {
+		return nil, &pathNotExistError{path}
+	}
+
+	if entry.Type != ObjectBlob {
+		return nil, fmt.Errorf("%s is not a file/blob", path)
+	}
+
+	blob, err := repo.LookupBlob(entry.ID)
+	if err != nil {
+		return nil, fmt.Errorf("Error looking up %s blob: %s", path, err)
+	}
+
+	yamlData := &YAMLDependencies{}
+	err = yaml.Unmarshal(blob, yamlData)
+	if err != nil {
+		return nil, fmt.Errorf("Error unmarshaling dependencies.yaml: %s", err)
+	}
+
+	if yamlData.Dependencies != nil {
+		for name, dependency := range yamlData.Dependencies {
+			if strings.HasPrefix(name, "/") {
+				continue
+			}
+			absoluteName := filepath.Clean(filepath.Join(repoPath, name))
+			delete(yamlData.Dependencies, name)
+			yamlData.Dependencies[absoluteName] = dependency
+		}
+	}
+
+	return yamlData, nil
+}
+
+// NewModule constructs a new module type for a dependencies.yaml update
+// from a given qt5 submodule structure.
+func NewModule(moduleName string, branch string, qt5Modules map[string]*submodule) (*Module, error) {
+	var repoPath string
+	if !strings.Contains(moduleName, "/") {
+		repoPath = "qt/" + moduleName
+	} else {
+		repoPath = moduleName
+	}
+
+	repo, err := OpenRepository(repoPath)
+	if err != nil {
+		return nil, fmt.Errorf("Error opening submodule %s: %s", moduleName, err)
+	}
+
+	if subModule, ok := qt5Modules[moduleName]; ok {
+		branch = subModule.branch
+	}
+	headRef := "refs/heads/" + branch
+
+	repoURL, err := RepoURL(repoPath)
+	if err != nil {
+		return nil, fmt.Errorf("could not find fetch url for %s: %s", moduleName, err)
+	}
+
+	moduleTipCommit, err := repo.Fetch(repoURL, headRef)
+	if err != nil {
+		return nil, fmt.Errorf("could not fetch repo tip %s of %s: %s", headRef, moduleName, err)
+	}
+
+	yamlDependencies, _ := readDependenciesYAML(repoPath, repo, moduleTipCommit)
+	if yamlDependencies == nil {
+		yamlDependencies = &YAMLDependencies{}
+		yamlDependencies.Dependencies = make(map[string]*YAMLModule)
+
+		subModule, ok := qt5Modules[moduleName]
+		if !ok {
+			return nil, fmt.Errorf("could not find %s in .gitmodules in qt5.git", moduleName)
+		}
+
+		populateDependencies := func(required bool, dependencies []string) {
+			for _, dependency := range dependencies {
+				_, knownModule := qt5Modules[dependency]
+				if !required && !knownModule {
+					continue
+				}
+
+				var yamlModule YAMLModule
+				yamlModule.Required = required
+				yamlModule.Ref = string(subModule.headCommit)
+
+				yamlDependencies.Dependencies[dependency] = &yamlModule
+			}
+		}
+
+		populateDependencies(true, subModule.requiredDependencies)
+		populateDependencies(false, subModule.optionalDependencies)
+	}
+
+	result := &Module{}
+	result.RepoPath = repoPath
+	result.Branch = branch
+	result.Tip = moduleTipCommit
+
+	for dependency, yamlModule := range yamlDependencies.Dependencies {
+		if yamlModule.Required {
+			result.RequiredDependencies = append(result.RequiredDependencies, dependency)
+		} else {
+			result.OptionalDependencies = append(result.OptionalDependencies, dependency)
+		}
+	}
+
+	return result, nil
+}
+
+func (module *Module) hasDependency(dependency string) bool {
+	for _, dep := range module.RequiredDependencies {
+		if dep == dependency {
+			return true
+		}
+	}
+	for _, dep := range module.OptionalDependencies {
+		if dep == dependency {
+			return true
+		}
+	}
+	return false
+}
+
+func (module *Module) refreshTip() error {
+	repo, err := OpenRepository(module.RepoPath)
+	if err != nil {
+		return fmt.Errorf("Error opening submodule %s: %s", module.RepoPath, err)
+	}
+
+	headRef := "refs/heads/" + module.Branch
+
+	repoURL, err := RepoURL(module.RepoPath)
+	if err != nil {
+		return fmt.Errorf("could not find fetch url for %s: %s", module.RepoPath, err)
+	}
+
+	moduleTipCommit, err := repo.Fetch(repoURL, headRef)
+	if err != nil {
+		return fmt.Errorf("could not fetch repo tip %s of %s: %s", headRef, module.RepoPath, err)
+	}
+
+	module.Tip = moduleTipCommit
+	return nil
+}
+
+func (module *Module) maybePrepareUpdatedDependenciesYaml(availableModules map[string]*Module) (yaml *YAMLDependencies, err error) {
+	var proposedUpdate YAMLDependencies
+	proposedUpdate.Dependencies = make(map[string]*YAMLModule)
+
+	updateDeps := func(required bool, deps []string) (allDependenciesAvailable bool, err error) {
+		for _, dep := range deps {
+			depModule, ok := availableModules[dep]
+			if !ok {
+				return false, nil
+			}
+
+			yamlModule := &YAMLModule{}
+			yamlModule.Required = required
+			yamlModule.Ref = string(depModule.Tip)
+
+			path, err := filepath.Rel(module.RepoPath, depModule.RepoPath)
+			if err != nil {
+				path = module.RepoPath
+			}
+			proposedUpdate.Dependencies[path] = yamlModule
+		}
+		return true, nil
+	}
+
+	if allDepsOk, err := updateDeps( /*required*/ true, module.RequiredDependencies); err != nil || !allDepsOk {
+		return nil, err
+	}
+	if allDepsOk, err := updateDeps( /*required*/ false, module.OptionalDependencies); err != nil || !allDepsOk {
+		return nil, err
+	}
+
+	return &proposedUpdate, nil
+}
+
+func lookupPathIndexEntry(index *Index, path string) (*IndexEntry, error) {
+	for i := 0; i < index.EntryCount(); i++ {
+		entry, err := index.EntryByIndex(i)
+		if err != nil {
+			return nil, err
+		}
+		if entry.Path == path {
+			return entry, nil
+		}
+	}
+	return nil, fmt.Errorf("could not locate path %s in index", path)
+}
+
+func (module *Module) generateChangeLogOfDependencies(oldDependencies *YAMLDependencies, newDependencies *YAMLDependencies) string {
+	if oldDependencies == nil || newDependencies == nil || oldDependencies.Dependencies == nil || newDependencies.Dependencies == nil {
+		log.Printf("Empty set of dependencies for change log update for %s\n", module.RepoPath)
+		return ""
+	}
+
+	var changeLog []string
+
+	for dependencyName, dependency := range newDependencies.Dependencies {
+		dependencyRepoPath := filepath.Clean(filepath.Join(module.RepoPath, dependencyName))
+		oldDependency, ok := oldDependencies.Dependencies[dependencyRepoPath]
+		if !ok {
+			log.Printf("Could not find module %s in the old dependencies table %v\n", dependencyRepoPath, oldDependencies.Dependencies)
+			continue
+		}
+		oldSha1 := oldDependency.Ref
+		newSha1 := dependency.Ref
+
+		depRepo, err := OpenRepository(dependencyRepoPath)
+		if err != nil {
+			log.Printf("Could not open dependency repo %s for changelog analysis: %s", dependencyRepoPath, err)
+			continue
+		}
+
+		changes, err := depRepo.LogOutput(`--pretty=format:  %m %s`, "--first-parent", string(oldSha1)+".."+string(newSha1))
+		if err != nil {
+			log.Printf("Oddly git log failed: %s\n", err)
+			continue
+		}
+		changeLog = append(changeLog, fmt.Sprintf("%s %s..%s:", dependencyRepoPath, oldSha1, newSha1))
+		changeLog = append(changeLog, changes...)
+		changeLog = append(changeLog, "")
+	}
+
+	summary := strings.Join(changeLog, "\n  ")
+	// Limit due to maximum command line size when talking to gerrit via ssh :(
+	if len(summary) > 65000 {
+		summary = summary[:65000]
+	}
+	return summary
+}
+
+type dependenciesUpdateResult struct {
+	result   DependenciesUpdateResultEnum
+	changeID string
+	commitID OID
+	summary  string
+}
+
+func (module *Module) updateDependenciesForModule(availableModules map[string]*Module) (result dependenciesUpdateResult, err error) {
+	yamlObject, err := module.maybePrepareUpdatedDependenciesYaml(availableModules)
+	if err != nil {
+		return dependenciesUpdateResult{}, err
+	}
+	if yamlObject == nil {
+		return dependenciesUpdateResult{result: DependenciesUpdateDependencyMissing}, nil
+	}
+
+	repo, err := OpenRepository(module.RepoPath)
+	if err != nil {
+		return dependenciesUpdateResult{}, fmt.Errorf("Error opening repo to retrieve tip: %s", err)
+	}
+
+	index, err := repo.NewIndex()
+	if err != nil {
+		return dependenciesUpdateResult{}, fmt.Errorf("Error creating temporary git index: %s", err)
+	}
+	defer index.Free()
+
+	err = index.ReadTree(module.Tip)
+	if err != nil {
+		return dependenciesUpdateResult{}, fmt.Errorf("Error populating temporary index from tree: %s", err)
+	}
+
+	existingEntry, _ := lookupPathIndexEntry(index, "dependencies.yaml")
+
+	updatedIndexEntryForFile := &IndexEntry{
+		Permissions: "100644",
+		Path:        "dependencies.yaml",
+	}
+
+	yamlBuffer := &bytes.Buffer{}
+	yamlEncoder := yaml.NewEncoder(yamlBuffer)
+	yamlEncoder.Encode(*yamlObject)
+	yamlEncoder.Close()
+
+	if err := index.HashObject(updatedIndexEntryForFile, yamlBuffer.Bytes()); err != nil {
+		return dependenciesUpdateResult{}, err
+	}
+
+	var summary string
+
+	if existingEntry != nil {
+		if updatedIndexEntryForFile.ID == existingEntry.ID {
+			return dependenciesUpdateResult{result: DependenciesUpdateContentUpToDate}, nil
+		}
+
+		log.Printf("Found existing dependencies file in %s, trying to read it to compare\n", module.RepoPath)
+		oldDependenciesFile, err := readDependenciesYAML(module.RepoPath, repo, module.Tip)
+		if err != nil {
+			log.Printf("Could not decode existing yaml dependencies file for change log generation purposes: %s\n", err)
+		} else {
+			summary = module.generateChangeLogOfDependencies(oldDependenciesFile, yamlObject)
+		}
+	}
+
+	if err := index.Add(updatedIndexEntryForFile); err != nil {
+		return dependenciesUpdateResult{}, err
+	}
+
+	newTree, err := index.WriteTree()
+	if err != nil {
+		return dependenciesUpdateResult{}, err
+	}
+
+	changeID, _, _, err := getExistingChange(module.RepoPath, module.Branch)
+	if err != nil {
+		return dependenciesUpdateResult{}, fmt.Errorf("failure to check for existing change id for module %s: %s", module.RepoPath, err)
+	}
+
+	if changeID == "" {
+		changeID = fmt.Sprintf("I%s", newTree)
+	}
+
+	message := fmt.Sprintf("Update dependencies on '%s' in %s\n\nChange-Id: %s\n", module.Branch, module.RepoPath, changeID)
+
+	parentCommit := module.Tip
+
+	commitOid, err := repo.CommitTree(newTree, message, parentCommit)
+	if err != nil {
+		return dependenciesUpdateResult{}, fmt.Errorf("Error creating git commit for dependencies update in module %s: %s", module.RepoPath, err)
+	}
+
+	log.Printf("New update commit created for %s: %s", module.RepoPath, commitOid)
+
+	return dependenciesUpdateResult{
+		result:   DependenciesUpdateUpdateScheduled,
+		changeID: changeID,
+		commitID: commitOid,
+		summary:  summary,
+	}, nil
+}
diff --git a/src/qtmoduleupdater/module_test.go b/src/qtmoduleupdater/module_test.go
new file mode 100644
index 00000000..38d341f6
--- /dev/null
+++ b/src/qtmoduleupdater/module_test.go
@@ -0,0 +1,135 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the repo tools module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package main
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+func TestModuleYamlMarshalling(t *testing.T) {
+	var module YAMLDependencies
+	module.Dependencies = make(map[string]*YAMLModule)
+
+	module.Dependencies["a"] = &YAMLModule{
+		Ref:      "refs/heads/foo",
+		Required: false,
+	}
+	module.Dependencies["b"] = &YAMLModule{
+		Ref:      "refs/heads/bar",
+		Required: true,
+	}
+
+	output := &bytes.Buffer{}
+	encoder := yaml.NewEncoder(output)
+	encoder.Encode(&module)
+	encoder.Close()
+
+	yamlStr := output.String()
+
+	assert.Equal(t, `dependencies:
+  a:
+    ref: refs/heads/foo
+    required: false
+  b:
+    ref: refs/heads/bar
+    required: true
+`, yamlStr, "Yaml output should be as expected")
+}
+
+func TestProposedUpdateFailsForModulesThatDependOnMoreThanQtBase(t *testing.T) {
+	// Make sure that this always points to the latest LTS branch. If it fails, update it.
+	ref := "refs/heads/5.12"
+	qt5Modules, err := getQt5ProductModules("qt/qt5", ref, "")
+
+	assert.Nil(t, err, "Retrieving qt5 modules expected to work")
+
+	todoMap, availableModules, err := loadTodoAndDoneModuleMapFromSubModules(ref, qt5Modules)
+	assert.Nil(t, err, "No error expected creating module map")
+
+	_, ok := availableModules["qt/qtbase"]
+	assert.True(t, ok, "qt/qtbase must be present in the module map")
+
+	qtSvg, ok := todoMap["qt/qtsvg"]
+	assert.True(t, ok, "qtsvg must be present in the module map")
+	yamlObject, err := qtSvg.maybePrepareUpdatedDependenciesYaml(availableModules)
+	assert.NotNil(t, yamlObject, "Yaml object must be defined for qtsvg")
+
+	yamlStr := &bytes.Buffer{}
+	encoder := yaml.NewEncoder(yamlStr)
+	encoder.Encode(*yamlObject)
+	encoder.Close()
+
+	assert.Nil(t, err, "It should be possible to create a new dependencies.yaml file for qtsvg")
+	assert.NotEqual(t, "", yamlStr, "Yaml string must not be empty for qtsvg")
+
+	qtDeclarative, ok := todoMap["qt/qtdeclarative"]
+	assert.True(t, ok, "qtdeclarative must be present in the module map")
+	yamlObject, err = qtDeclarative.maybePrepareUpdatedDependenciesYaml(availableModules)
+
+	assert.Nil(t, err, "It should be possible to create a new dependencies.yaml file for qtdeclarative")
+	assert.Nil(t, yamlObject, "Yaml string be empty for qtdeclarative because dependencies are not available yet")
+}
+
+func TestRemovalOfNonExistentOptionalDependencies(t *testing.T) {
+	// Make sure that this always points to the latest LTS branch. If it fails, update it.
+	ref := "refs/heads/5.12"
+	qt5Modules, err := getQt5ProductModules("qt/qt5", ref, "")
+	assert.Nil(t, err, "Retrieving qt5 modules expected to work")
+
+	_, haveSvg := qt5Modules["qt/qtsvg"]
+	assert.True(t, haveSvg, "qtsvg needs to be in qt5.git")
+	delete(qt5Modules, "qt/qtsvg")
+
+	qtDeclarative := qt5Modules["qt/qtdeclarative"]
+	assert.NotNil(t, qtDeclarative, "need qtdeclarative")
+
+	assert.Contains(t, qtDeclarative.optionalDependencies, "qt/qtsvg")
+	qtDeclarativeModule, err := NewModule("qt/qtdeclarative", ref, qt5Modules)
+	assert.Nil(t, err, "There shall not be any error creating the module")
+
+	assert.NotNil(t, qtDeclarativeModule, "qtdeclarative module shall exist")
+
+	assert.Contains(t, qtDeclarativeModule.RequiredDependencies, "qt/qtbase")
+	assert.NotContains(t, qtDeclarativeModule.OptionalDependencies, "qt/qtsvg")
+}
+
+func TestQueryChangeStatus(t *testing.T) {
+	status, err := getGerritChangeStatus("qt/qtbase", "dev", "Ie6f0e2e3bb198a95dd40e7416adc8ffb29f3b2ba")
+	assert.Nil(t, err, "Querying should not produce an error")
+	assert.Equal(t, "MERGED", status)
+
+	status, err = getGerritChangeStatus("qt/qtbase", "dev", "I6e4349f4d72de307a579f59bb689fd0638690403")
+	assert.Nil(t, err, "Querying should not produce an error")
+	assert.Equal(t, "ABANDONED", status)
+
+}
diff --git a/src/qtmoduleupdater/moduleupdatebatch.go b/src/qtmoduleupdater/moduleupdatebatch.go
new file mode 100644
index 00000000..b79949cb
--- /dev/null
+++ b/src/qtmoduleupdater/moduleupdatebatch.go
@@ -0,0 +1,227 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the repo tools module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+)
+
+// PendingUpdate describes that a module needs an updated dependencies.yaml and we are waiting for the change
+// to succeed/fail
+type PendingUpdate struct {
+	Module   *Module
+	ChangeID string
+}
+
+// ModuleUpdateBatch is used to serialize and de-serialize the module updating state, used for debugging.
+type ModuleUpdateBatch struct {
+	Product           string
+	Branch            string
+	Todo              map[string]*Module
+	Done              map[string]*Module
+	Pending           []*PendingUpdate
+	FailedModuleCount int
+}
+
+func (batch *ModuleUpdateBatch) scheduleUpdates(pushUserName string, manualStage bool) error {
+	for _, moduleToUpdate := range batch.Todo {
+		update, err := moduleToUpdate.updateDependenciesForModule(batch.Done)
+		if err != nil {
+			return fmt.Errorf("fatal error proposing module update: %s", err)
+		}
+		log.Printf("Attempting update for module %s resulted in %v\n", moduleToUpdate.RepoPath, update.result)
+		if update.result == DependenciesUpdateContentUpToDate {
+			batch.Done[moduleToUpdate.RepoPath] = moduleToUpdate
+			delete(batch.Todo, moduleToUpdate.RepoPath)
+		} else if update.result == DependenciesUpdateDependencyMissing {
+			// Nothing to be done, we are waiting for indirect dependencies
+		} else if update.result == DependenciesUpdateUpdateScheduled {
+			// push and stage
+			if err = pushAndStageChange(moduleToUpdate.RepoPath, moduleToUpdate.Branch, update.commitID, update.summary, pushUserName, manualStage); err != nil {
+				return fmt.Errorf("error pushing change upate: %s", err)
+			}
+			batch.Pending = append(batch.Pending, &PendingUpdate{moduleToUpdate, update.changeID})
+			delete(batch.Todo, moduleToUpdate.RepoPath)
+		} else {
+			return fmt.Errorf("invalid state returned by updateDependenciesForModule for %s", moduleToUpdate.RepoPath)
+		}
+	}
+
+	return nil
+}
+
+func removeAllDirectAndIndirectDependencies(allModules *map[string]*Module, moduleToRemove string) {
+	for moduleName, module := range *allModules {
+		if module.hasDependency(moduleToRemove) {
+			delete(*allModules, moduleName)
+			removeAllDirectAndIndirectDependencies(allModules, module.RepoPath)
+		}
+	}
+}
+
+func (batch *ModuleUpdateBatch) checkPendingModules() {
+	var newPending []*PendingUpdate
+	for _, pendingUpdate := range batch.Pending {
+		module := pendingUpdate.Module
+		status, err := getGerritChangeStatus(module.RepoPath, module.Branch, pendingUpdate.ChangeID)
+		if err != nil || status == "STAGED" || status == "INTEGRATING" || status == "STAGING" {
+			// no change yet
+			newPending = append(newPending, pendingUpdate)
+			continue
+		} else if status == "MERGED" {
+			module.refreshTip()
+			batch.Done[module.RepoPath] = module
+		} else {
+			// Open or abandoned, not sure -- either way an error integrating the update
+			removeAllDirectAndIndirectDependencies(&batch.Todo, module.RepoPath)
+			batch.FailedModuleCount++
+		}
+	}
+	batch.Pending = newPending
+}
+
+func loadTodoAndDoneModuleMapFromSubModules(branch string, submodules map[string]*submodule) (todo map[string]*Module, done map[string]*Module, err error) {
+	todoModules := make(map[string]*Module)
+	doneModules := make(map[string]*Module)
+
+	for name, submodule := range submodules {
+		// Erase modules that don't follow the qt5 branching scheme and don't need
+		// dependencies.yaml
+		if submodule.branch == "master" {
+			continue
+		}
+
+		module, err := NewModule(name, branch, submodules)
+		if err != nil {
+			return nil, nil, fmt.Errorf("could not create internal module structure: %s", err)
+		}
+
+		if submodule.repoType == "inherited" || name == "qt/qtbase" {
+			doneModules[module.RepoPath] = module
+		} else {
+			todoModules[module.RepoPath] = module
+		}
+	}
+
+	return todoModules, doneModules, nil
+}
+
+func (batch *ModuleUpdateBatch) loadTodoList(qt5FetchRef string) error {
+	qt5Modules, err := getQt5ProductModules(batch.Product, batch.Branch, qt5FetchRef)
+	if err != nil {
+		return fmt.Errorf("Error listing qt5 product modules: %s", err)
+	}
+
+	batch.Todo, batch.Done, err = loadTodoAndDoneModuleMapFromSubModules(batch.Branch, qt5Modules)
+	return err
+}
+
+func sanitizeBranchOrRepo(s string) string {
+	s = strings.ToLower(s)
+	s = strings.ReplaceAll(s, "/", "_")
+	s = strings.ReplaceAll(s, "-", "_")
+	return s
+}
+
+func (batch *ModuleUpdateBatch) stateFileName() string {
+	return fmt.Sprintf("state_%s_%s.json", sanitizeBranchOrRepo(batch.Product), sanitizeBranchOrRepo(batch.Branch))
+}
+
+func (batch *ModuleUpdateBatch) saveState() error {
+	fileName := batch.stateFileName()
+	outputFile, err := os.Create(fileName)
+	if err != nil {
+		return fmt.Errorf("failed to create state file %s: %s", fileName, err)
+	}
+	defer outputFile.Close()
+
+	encoder := json.NewEncoder(outputFile)
+	encoder.SetIndent("", "    ")
+	return encoder.Encode(batch)
+}
+
+func (batch *ModuleUpdateBatch) loadState() error {
+	fileName := batch.stateFileName()
+	inputFile, err := os.Open(fileName)
+	if err != nil {
+		return err
+	}
+	defer inputFile.Close()
+
+	decoder := json.NewDecoder(inputFile)
+	err = decoder.Decode(batch)
+	if err != nil {
+		return fmt.Errorf("Error decoding JSON state file: %s", err)
+	}
+	return nil
+}
+
+func (batch *ModuleUpdateBatch) isDone() bool {
+	return len(batch.Todo) == 0 && len(batch.Pending) == 0
+}
+
+func (batch *ModuleUpdateBatch) PrintSummary() {
+	fmt.Fprintf(os.Stdout, "Summary of git repository dependency update for target branch %s based off of %s\n", batch.Branch, batch.Product)
+
+	if batch.isDone() {
+		if batch.FailedModuleCount > 0 {
+			fmt.Fprintf(os.Stdout, "    %v modules failed to be updated. Check Gerrit for the %s branch\n", batch.FailedModuleCount, batch.Branch)
+		} else {
+			fmt.Fprintf(os.Stdout, "    No updates are necessary for any modules - everything is up-to-date\n")
+		}
+		return
+	}
+
+	if len(batch.Done) > 0 {
+		fmt.Fprintf(os.Stdout, "The following modules have been brought up-to-date:\n")
+
+		for name := range batch.Done {
+			fmt.Println("    " + name)
+		}
+	}
+
+	if len(batch.Pending) > 0 {
+		fmt.Fprintf(os.Stdout, "The following modules are current in-progress:\n")
+
+		for _, pending := range batch.Pending {
+			fmt.Println("    " + pending.Module.RepoPath)
+		}
+	}
+
+	fmt.Fprintf(os.Stdout, "The following modules are outdated and are either waiting for one of their dependencies or are ready for an update:\n")
+	for name := range batch.Todo {
+		fmt.Println("    " + name)
+	}
+
+	fmt.Println()
+	fmt.Println()
+}
diff --git a/src/qtmoduleupdater/qt5.go b/src/qtmoduleupdater/qt5.go
new file mode 100644
index 00000000..33efb052
--- /dev/null
+++ b/src/qtmoduleupdater/qt5.go
@@ -0,0 +1,250 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the repo tools module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"net/url"
+	"strings"
+
+	"github.com/vaughan0/go-ini"
+)
+
+type submodule struct {
+	url                  string
+	branch               string
+	repoType             string
+	requiredDependencies []string
+	optionalDependencies []string
+	headCommit           OID
+}
+
+func listSubmodules(repo Repository, repoURL *url.URL, commit OID) (modules map[string]*submodule, err error) {
+	tree, err := repo.ListTree(commit)
+	if err != nil {
+		return
+	}
+
+	gitModulesEntry, ok := tree.Entries[".gitmodules"]
+	if !ok {
+		err = fmt.Errorf("could not locate .gitmodules in git tree")
+		return
+	}
+	if gitModulesEntry.Type != ObjectBlob {
+		err = fmt.Errorf(".gitmodules is not a file/blob")
+		return
+	}
+	blob, err := repo.LookupBlob(gitModulesEntry.ID)
+	if err != nil {
+		err = fmt.Errorf("Error looking up .gitmodules blob: %s", err)
+		return
+	}
+
+	baseURL := *repoURL
+	baseURL.Path = baseURL.Path + "/repo.git"
+	baseURL.Scheme = "ssh"
+	host, _, err := net.SplitHostPort(baseURL.Host)
+	if err != nil {
+		err = fmt.Errorf("Error splitting host and port from base url %v", baseURL)
+		return
+	}
+	baseURL.Host = host
+
+	gitModules, err := ini.Load(bytes.NewBuffer(blob))
+	for key, values := range gitModules {
+		subModule := strings.TrimPrefix(key, `submodule "`)
+		if subModule == key {
+			continue
+		}
+		subModule = strings.TrimSuffix(subModule, `"`)
+
+		if status, ok := values["status"]; ok {
+			if status == "ignore" {
+				continue
+			}
+		} else if initRepo, ok := values["initrepo"]; ok {
+			if initRepo != "true" {
+				continue
+			}
+		}
+
+		module := &submodule{}
+
+		urlString, ok := values["url"]
+		if !ok {
+			err = fmt.Errorf("could not find submodule URL for submodule %s", subModule)
+			return
+		}
+
+		var subModuleURL *url.URL
+		subModuleURL, err = url.Parse(urlString)
+		if err != nil {
+			err = fmt.Errorf("Error parsing submodule url %s: %s", values["url"], err)
+			return
+		}
+
+		if modules == nil {
+			modules = make(map[string]*submodule)
+		}
+
+		module.url = baseURL.ResolveReference(subModuleURL).String()
+		module.branch = values["branch"]
+
+		if repoType, ok := values["repoType"]; ok {
+			module.repoType = repoType
+		}
+
+		if requiredDependenciesAsString, ok := values["depends"]; ok {
+
+			for _, dep := range strings.Split(requiredDependenciesAsString, " ") {
+				module.requiredDependencies = append(module.requiredDependencies, "qt/"+dep)
+			}
+		}
+
+		if optionalDependenciesAsString, ok := values["recommends"]; ok {
+			for _, dep := range strings.Split(optionalDependenciesAsString, " ") {
+				module.optionalDependencies = append(module.optionalDependencies, "qt/"+dep)
+			}
+		}
+
+		if tree.Entries[subModule].Type != ObjectCommit {
+			return nil, fmt.Errorf("submodule entry for %s does not point to a commit", subModule)
+		}
+
+		module.headCommit = tree.Entries[subModule].ID
+
+		modules["qt/"+subModule] = module
+	}
+	return
+}
+
+func getQt5ProductModules(productProject string, branchOrRef string, productFetchRef string) (modules map[string]*submodule, err error) {
+	if productFetchRef == "" {
+		productFetchRef = branchOrRef
+	}
+	if !strings.HasPrefix(productFetchRef, "refs/") {
+		productFetchRef = "refs/heads/" + productFetchRef
+	}
+
+	productRepoURL, err := RepoURL(productProject)
+	if err != nil {
+		return nil, fmt.Errorf("Error determining %s repo URL: %s", productProject, err)
+	}
+
+	productRepo, err := OpenRepository(productProject)
+	if err != nil {
+		return nil, fmt.Errorf("Error opening product repo: %s", err)
+	}
+
+	productHead, err := productRepo.Fetch(productRepoURL, productFetchRef)
+	if err != nil {
+		return nil, fmt.Errorf("Error fetching product repo: %s", err)
+	}
+
+	return listSubmodules(productRepo, productRepoURL, productHead)
+}
+
+func prepareQt5Update(product string, branch string, updatedModules map[string]*Module, pushUserName string, manualStage bool) error {
+	productRepoURL, err := RepoURL(product)
+	if err != nil {
+		return fmt.Errorf("Error determining %s repo URL: %s", product, err)
+	}
+
+	productRepo, err := OpenRepository(product)
+	if err != nil {
+		return fmt.Errorf("Error opening product repo: %s", err)
+	}
+
+	productHead, err := productRepo.Fetch(productRepoURL, "refs/heads/"+branch)
+	if err != nil {
+		return err
+	}
+
+	index, err := productRepo.NewIndex()
+	if err != nil {
+		return err
+	}
+
+	if err = index.ReadTree(productHead); err != nil {
+		return err
+	}
+
+	qt5Modules, err := listSubmodules(productRepo, productRepoURL, productHead)
+	if err != nil {
+		return fmt.Errorf("error retrieving list of submodules: %s", err)
+	}
+
+	for name, qt5Module := range qt5Modules {
+		updatedModule, ok := updatedModules[name]
+		if !ok {
+			if qt5Module.branch != branch {
+				continue
+			}
+			return fmt.Errorf("could not locate qt5 module %s in map of updated modules", name)
+		}
+
+		unprefixedPath := strings.TrimPrefix(name, "qt/")
+
+		updatedEntry := &IndexEntry{
+			Permissions: "160000",
+			Path:        unprefixedPath,
+			ID:          OID(updatedModule.Tip),
+		}
+
+		if err = index.Add(updatedEntry); err != nil {
+			return fmt.Errorf("could not update submodule index entry for %s: %s", unprefixedPath, err)
+		}
+	}
+
+	newTree, err := index.WriteTree()
+	if err != nil {
+		return fmt.Errorf("could not write index with updated submodule sha1s: %s", err)
+	}
+
+	changeID, _, _, err := getExistingChange(product, branch)
+	if err != nil {
+		return fmt.Errorf("error looking for an existing change while updating submodules: %s", err)
+	}
+
+	if changeID == "" {
+		changeID = fmt.Sprintf("I%s", newTree)
+	}
+
+	message := fmt.Sprintf("Update submodules on '%s' in %s\n\nChange-Id: %s\n", branch, product, changeID)
+
+	commitOid, err := productRepo.CommitTree(newTree, message, productHead)
+	if err != nil {
+		return fmt.Errorf("could not create new commit for submodule update: %s", err)
+	}
+
+	fmt.Printf("Created new commit for submodule update: %s\n", commitOid)
+
+	return pushAndStageChange(product, branch, commitOid, "Updating all submodules with a new consistent set", pushUserName, manualStage)
+}
diff --git a/src/qtmoduleupdater/qt5_test.go b/src/qtmoduleupdater/qt5_test.go
new file mode 100644
index 00000000..d75b4b03
--- /dev/null
+++ b/src/qtmoduleupdater/qt5_test.go
@@ -0,0 +1,46 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the repo tools module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGitModules(t *testing.T) {
+	ref := "refs/heads/5.12"
+	subModules, err := getQt5ProductModules("qt/qt5", ref, "")
+	assert.Nil(t, err, "No errors expected retrieving the submodules")
+
+	qqc, ok := subModules["qt/qtquickcontrols"]
+	assert.True(t, ok, "Could not find qtquickcontrols in submodules")
+
+	assert.Equal(t, []string{"qt/qtdeclarative"}, qqc.requiredDependencies)
+	assert.Equal(t, []string{"qt/qtgraphicaleffects"}, qqc.optionalDependencies)
+}
diff --git a/src/qtmoduleupdater/repo.go b/src/qtmoduleupdater/repo.go
new file mode 100644
index 00000000..39589520
--- /dev/null
+++ b/src/qtmoduleupdater/repo.go
@@ -0,0 +1,423 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the repo tools module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+type commandWithCapturedOutput struct {
+	cmd    *exec.Cmd
+	stdout bytes.Buffer
+	stderr bytes.Buffer
+}
+
+func newCommandWithCapturedOutput(cmd *exec.Cmd) *commandWithCapturedOutput {
+	result := &commandWithCapturedOutput{}
+	result.cmd = cmd
+	result.cmd.Stdout = &result.stdout
+	result.cmd.Stderr = &result.stderr
+	return result
+}
+
+func (c *commandWithCapturedOutput) Run() (string, error) {
+	err := c.cmd.Run()
+	if err != nil {
+		return "", fmt.Errorf("Error running %s: %s\nStdout: %s\nStderr: %s", strings.Join(c.cmd.Args, " "), err, c.stdout.String(), c.stderr.String())
+	}
+	return c.stdout.String(), err
+}
+
+func (c *commandWithCapturedOutput) RunWithSpaceTrimmed() (string, error) {
+	output, err := c.Run()
+	return strings.TrimSpace(output), err
+}
+
+// Repository is a type that wraps various git operations on the given repository on the local disk.
+type Repository string
+
+// OID is a git object identifier, in the form of a SHA1 check-sum.
+type OID string
+
+// RepoURL returns a clone/push/fetch URL for the given project.
+func RepoURL(project string) (*url.URL, error) {
+	gerritConfig := struct {
+		URL  string
+		Port string
+	}{
+		"codereview.qt-project.org",
+		"29418",
+	}
+	repo := &url.URL{}
+	repo.Host = gerritConfig.URL + ":" + gerritConfig.Port
+	repo.Path = "/" + project
+	repo.Scheme = "ssh"
+	return repo, nil
+}
+
+// OpenRepository is used to create a new repository wrapper for the specified project.
+// If the repository doesn't exist yet, it will be cloned.
+func OpenRepository(project string) (Repository, error) {
+	reposLocation := "git-repos"
+	repoPath := filepath.Join(reposLocation, project)
+	if _, err := os.Stat(repoPath); os.IsNotExist(err) {
+		url, err := RepoURL(project)
+		if err != nil {
+			return "", err
+		}
+		log.Printf("Cloning missing repository %s from %s to %s\n", project, url, repoPath)
+		cloneCmd := []string{"clone", "--bare"}
+		for _, ref := range strings.Split(os.Getenv("QT_CI_REPO_REFERENCES"), ":") {
+			path := ref + "/.git/modules/" + strings.Split(project, "/")[1]
+			_, err := os.Stat(path)
+			if err == nil {
+				cloneCmd = append(cloneCmd, []string{"--reference", path}...)
+				break
+			}
+		}
+		cloneCmd = append(cloneCmd, []string{url.String(), repoPath}...)
+		cmd := exec.Command("git", cloneCmd...)
+		if err = cmd.Run(); err != nil {
+			return "", err
+		}
+	}
+	return Repository(repoPath), nil
+}
+
+func (repo Repository) gitCommand(command string, parameters ...string) *commandWithCapturedOutput {
+	parameters = append([]string{"--git-dir=" + string(repo), command}, parameters...)
+	return newCommandWithCapturedOutput(exec.Command("git", parameters...))
+}
+
+// LookupReference resolves the provided git reference by means of calling rev-parse.
+func (repo Repository) LookupReference(ref string) (OID, error) {
+	rev, err := repo.gitCommand("rev-parse", ref).RunWithSpaceTrimmed()
+	return OID(rev), err
+}
+
+// ObjectType denotes the different types of objects stored in a git repository.
+type ObjectType int
+
+const (
+	// ObjectBlob refers to a pure data object
+	ObjectBlob = iota
+	// ObjectCommit refers to a git commit
+	ObjectCommit
+	// ObjectTree refers to a tree if blobs or trees
+	ObjectTree
+)
+
+// TreeEntry describes the entry of a directory listing in git
+type TreeEntry struct {
+	Permissions string
+	Type        ObjectType
+	ID          OID
+}
+
+// Tree is data structure representing the output of the git ls-tree command.
+type Tree struct {
+	Repo    Repository
+	ID      OID
+	Entries map[string]TreeEntry
+}
+
+func (repo Repository) decodeLsTreeOutput(commit OID, output string) (*Tree, error) {
+	result := &Tree{
+		Repo:    repo,
+		ID:      commit,
+		Entries: make(map[string]TreeEntry),
+	}
+	for _, line := range bytes.Split([]byte(output), []byte{0}) {
+		if len(line) == 0 {
+			continue
+		}
+		var entry TreeEntry
+		modeIndex := bytes.IndexByte(line, ' ')
+		if modeIndex == -1 {
+			return nil, fmt.Errorf("missing space after permission field while parsing git ls-tree output")
+		}
+		entry.Permissions = string(line[:modeIndex])
+
+		line = line[modeIndex+1:]
+
+		typeIndex := bytes.IndexByte(line, ' ')
+		if typeIndex == -1 {
+			return nil, fmt.Errorf("missing space after type field while parsing git ls-tree output")
+		}
+		typeName := line[:typeIndex]
+		switch string(typeName) {
+		case "tree":
+			entry.Type = ObjectTree
+		case "blob":
+			entry.Type = ObjectBlob
+		case "commit":
+			entry.Type = ObjectCommit
+		default:
+			return nil, fmt.Errorf("unexpected entry type %s while parsing git ls-tree output", typeName)
+		}
+
+		line = line[typeIndex+1:]
+
+		objectIndex := bytes.IndexByte(line, '\t')
+		if objectIndex == -1 {
+			return nil, fmt.Errorf("missing space after entry field while parsing git ls-tree output")
+		}
+		entry.ID = OID(string(line[:objectIndex]))
+
+		name := string(line[objectIndex+1:])
+
+		result.Entries[name] = entry
+	}
+	return result, nil
+}
+
+// ListTree retrieves a (non-recursive) directory listing of the specified commit.
+func (repo Repository) ListTree(commit OID) (*Tree, error) {
+	output, err := repo.gitCommand("ls-tree", "-z", string(commit)).Run()
+	if err != nil {
+		return nil, err
+	}
+	return repo.decodeLsTreeOutput(commit, output)
+}
+
+// ListTreeWithPath retrieves a (non-recursive) directory listing of the specified commit with the specified path.
+func (repo Repository) ListTreeWithPath(commit OID, subPath string) (*Tree, error) {
+	output, err := repo.gitCommand("ls-tree", "-z", string(commit), subPath).Run()
+	if err != nil {
+		return nil, err
+	}
+	return repo.decodeLsTreeOutput(commit, output)
+}
+
+// Fetch retrieves the specified refSpec from the given url. The result is fetched into FETCH_HEAD
+// and the value of FETCH_HEAD is returned.
+func (repo Repository) Fetch(url *url.URL, refSpec string) (sha1 OID, err error) {
+	if fetch := os.Getenv("NO_FETCH"); len(fetch) == 0 {
+		cmd := repo.gitCommand("fetch", url.String(), refSpec)
+		if _, err = cmd.Run(); err != nil {
+			return "", fmt.Errorf("Error running fetch command: %s", err)
+		}
+	}
+	ref, err := repo.LookupReference("FETCH_HEAD")
+	if err != nil {
+		return "", fmt.Errorf("Error looking up FETCH_HEAD after fetch: %s", err)
+	}
+	return ref, nil
+}
+
+// Push is a wrapper around the git push commit.
+func (repo Repository) Push(url *url.URL, commit OID, targetRef string) error {
+	refSpec := fmt.Sprintf("%s:%s", commit, targetRef)
+	_, err := repo.gitCommand("push", url.String(), refSpec).Run()
+	return err
+}
+
+// LookupBlob returns the byte content of the specified blob object.
+func (repo Repository) LookupBlob(object OID) ([]byte, error) {
+	output, err := repo.gitCommand("cat-file", "blob", string(object)).Run()
+	if err != nil {
+		return nil, err
+	}
+	return []byte(output), nil
+}
+
+// IndexEntry represents an entry in the virtual git index directory structure.
+type IndexEntry struct {
+	Permissions string
+	Path        string
+	ID          OID
+}
+
+// Index is a wrapper around git operations that allow operating on a temporary index.
+type Index struct {
+	file          *os.File
+	repo          Repository
+	cachedEntries []IndexEntry
+}
+
+// NewIndex creates a new git index based on a temporary file. Unless you'd like to
+// start with an empty tree, you may want to populate the index with ReadTree.
+// A newly created index should be freed with Free() at the end of the usage, in order
+// to remove the temporary file.
+func (repo Repository) NewIndex() (result *Index, err error) {
+	result = &Index{}
+	result.repo = repo
+	result.file, err = ioutil.TempFile("", "")
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+// CommitTree creates a new commit object from the specified tree, along with the specified message and parent commits.
+// The new commit id is returned.
+func (repo Repository) CommitTree(tree OID, message string, parents ...OID) (OID, error) {
+	allParams := make([]string, 0, 1+2*len(parents))
+	allParams = append(allParams, string(tree))
+	for _, parent := range parents {
+		allParams = append(allParams, "-p")
+		allParams = append(allParams, string(parent))
+	}
+	cmd := repo.gitCommand("commit-tree", allParams...)
+	cmd.cmd.Stdin = bytes.NewBufferString(message)
+	commit, err := cmd.RunWithSpaceTrimmed()
+	return OID(commit), err
+}
+
+// LogOutput is a wrapper around the git log command.
+func (repo Repository) LogOutput(options ...string) ([]string, error) {
+	logOutput, err := repo.gitCommand("log", options...).Run()
+	if err != nil {
+		return nil, err
+	}
+	var log []string
+	scanner := bufio.NewScanner(bytes.NewBufferString(logOutput))
+	for scanner.Scan() {
+		log = append(log, scanner.Text())
+	}
+	return log, nil
+}
+
+// Free is responsible for deleting the temporary index file.
+func (idx *Index) Free() {
+	os.Remove(idx.file.Name())
+	idx.file = nil
+}
+
+func (idx *Index) gitCommandWithIndex(command string, parameters ...string) *commandWithCapturedOutput {
+	cmd := idx.repo.gitCommand(command, parameters...)
+	cmd.cmd.Env = os.Environ()
+	cmd.cmd.Env = append(cmd.cmd.Env, "GIT_INDEX_FILE="+idx.file.Name())
+	return cmd
+}
+
+func (idx *Index) updateCachedEntries() error {
+	idx.cachedEntries = make([]IndexEntry, 0)
+
+	output, err := idx.gitCommandWithIndex("ls-files", "-z", "--stage").Run()
+	if err != nil {
+		return err
+	}
+	for _, line := range bytes.Split([]byte(output), []byte{0}) {
+		if len(line) == 0 {
+			continue
+		}
+		var entry IndexEntry
+		modeIndex := bytes.IndexByte(line, ' ')
+		if modeIndex == -1 {
+			return fmt.Errorf("missing space after permission field while parsing git ls-files output")
+		}
+		entry.Permissions = string(line[:modeIndex])
+		line = line[modeIndex+1:]
+
+		objectIndex := bytes.IndexByte(line, ' ')
+		if objectIndex == -1 {
+			return fmt.Errorf("missing space after entry field while parsing git ls-files output")
+		}
+		entry.ID = OID(string(line[:objectIndex]))
+		line = line[objectIndex+1:]
+
+		stageIndex := bytes.IndexByte(line, '\t')
+		if stageIndex == -1 {
+			return fmt.Errorf("missing space after stage field while parsing git ls-files output")
+		}
+
+		entry.Path = string(line[stageIndex+1:])
+		idx.cachedEntries = append(idx.cachedEntries, entry)
+	}
+	return nil
+}
+
+// EntryCount returns the number of directory/file entries in the index.
+func (idx *Index) EntryCount() int {
+	return len(idx.cachedEntries)
+}
+
+// EntryByIndex returns the i-th entry in the directory index.
+func (idx *Index) EntryByIndex(i int) (*IndexEntry, error) {
+	if i < 0 || i >= len(idx.cachedEntries) {
+		return nil, fmt.Errorf("Index %v out of range in index (0 - %v)", i, len(idx.cachedEntries))
+	}
+
+	return &idx.cachedEntries[i], nil
+}
+
+// ReadTree populates the index from the specified tree object. This is implemented by calling git read-tree.
+func (idx *Index) ReadTree(tree OID) error {
+	_, err := idx.gitCommandWithIndex("read-tree", "--index-output="+idx.file.Name(), string(tree)).Run()
+	if err != nil {
+		return err
+	}
+	return idx.updateCachedEntries()
+}
+
+// Add adds a new entry to the index or updates an existing one if already present.
+func (idx *Index) Add(entry *IndexEntry) error {
+	_, err := idx.gitCommandWithIndex("update-index", "--add", "--cacheinfo", fmt.Sprintf("%s,%s,%s", entry.Permissions, entry.ID, entry.Path)).Run()
+	if err != nil {
+		return err
+	}
+	return idx.updateCachedEntries()
+}
+
+// HashObject writes content b as git object to the database and updates the entry.
+func (idx *Index) HashObject(entry *IndexEntry, b []byte) error {
+	tempfile, err := ioutil.TempFile("", "")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tempfile.Name())
+
+	if _, err := tempfile.Write(b); err != nil {
+		return err
+	}
+
+	newSha1, err := idx.gitCommandWithIndex("hash-object", "-w", tempfile.Name()).RunWithSpaceTrimmed()
+	if err != nil {
+		return err
+	}
+
+	entry.ID = OID(newSha1)
+
+	return nil
+}
+
+// WriteTree writes the index to the git database as a tree object and returns the tree object id.
+func (idx *Index) WriteTree() (OID, error) {
+	output, err := idx.gitCommandWithIndex("write-tree").RunWithSpaceTrimmed()
+	return OID(output), err
+}
diff --git a/src/qtmoduleupdater/repo_test.go b/src/qtmoduleupdater/repo_test.go
new file mode 100644
index 00000000..4b478ffc
--- /dev/null
+++ b/src/qtmoduleupdater/repo_test.go
@@ -0,0 +1,149 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the repo tools module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestRepo(t *testing.T) {
+	repo, err := OpenRepository("qt/qtbase")
+	if err != nil {
+		t.Fatalf("Unexpected error opening qtbase repo: %s", err)
+	}
+	if !strings.HasSuffix(string(repo), "qt/qtbase") {
+		t.Fatalf("Unexpected repo path %s", repo)
+	}
+
+	ref, err := repo.LookupReference("v5.5.0")
+	if err != nil {
+		t.Fatalf("Unexpected error looking up reference: %s", err)
+	}
+	if string(ref) != "2fde9f59eeab68ede92324e7613daf8be3eaf498" {
+		t.Fatalf("Incorrect sha1 for v5.5.0 tag")
+	}
+
+	tree, err := repo.ListTree(ref)
+	if err != nil {
+		t.Fatalf("Unexpected error listing tree: %s", err)
+	}
+
+	if tree.ID != ref {
+		t.Fatalf("Incorrect tree entry for %s", ref)
+	}
+
+	qtbaseProEntry, ok := tree.Entries["qtbase.pro"]
+	if !ok {
+		t.Fatalf("Missing qtbase.pro entry in tree listing")
+	}
+
+	if qtbaseProEntry.ID != "24d0f5287ba26ee0e53e34c8860c6c7baf7b0268" {
+		t.Fatalf("Unexpected sha1 for qtbase.pro: %s", qtbaseProEntry.ID)
+	}
+
+	if qtbaseProEntry.Type != ObjectBlob {
+		t.Fatalf("Unexpected entry type for qtbase.pro")
+	}
+
+	if qtbaseProEntry.Permissions != "100644" {
+		t.Fatalf("Incorrect permissions for qtbase.pro")
+	}
+
+	qmakeConf, ok := tree.Entries[".qmake.conf"]
+	if !ok {
+		t.Fatalf("Missing .qmake.conf in tree listing")
+	}
+
+	content, err := repo.LookupBlob(qmakeConf.ID)
+	if err != nil {
+		t.Fatalf("Unexpected error looking up .qmake.conf blob: %s", err)
+	}
+
+	expectedContent := `load(qt_build_config)
+CONFIG += qt_example_installs
+CONFIG += warning_clean
+
+QT_SOURCE_TREE = $$PWD
+QT_BUILD_TREE = $$shadowed($$PWD)
+
+# In qtbase, all modules follow qglobal.h
+MODULE_VERSION = $$QT_VERSION
+`
+	if string(content) != expectedContent {
+		t.Fatalf("Unexpected blob content for .qmake.conf: %s", string(content))
+	}
+}
+
+func TestIndex(t *testing.T) {
+	repo, err := OpenRepository("qt/qtbase")
+	if err != nil {
+		t.Fatalf("Unexpected error opening qtbase repo: %s", err)
+	}
+
+	ref, err := repo.LookupReference("v5.5.0")
+	if err != nil {
+		t.Fatalf("Unexpected error looking up v5.5.0 tag")
+	}
+
+	index, err := repo.NewIndex()
+	if err != nil {
+		t.Fatalf("Could not get index.")
+	}
+	defer index.Free()
+
+	err = index.ReadTree(ref)
+	if err != nil {
+		t.Fatalf("Error reading index tree: %s", err)
+	}
+
+	if index.EntryCount() != 21452 {
+		t.Fatalf("Unexpected index entry count %v", index.EntryCount())
+	}
+}
+
+func TestLog(t *testing.T) {
+	repo, err := OpenRepository("qt/qtbase")
+	if err != nil {
+		t.Fatalf("Unexpected error opening qtbase repo: %s", err)
+	}
+
+	output, err := repo.LogOutput(`--pretty=format:  %m %s`, "--first-parent", "v5.0.0~2..v5.0.0")
+	if err != nil {
+		t.Fatalf("Unexpected error calling git log: %s", err)
+	}
+	if len(output) != 2 {
+		t.Fatalf("Unexpected length of git log output array: %v", len(output))
+	}
+	if output[0] != "  > Fix font sizes when X11 has a forced dpi setting" {
+		t.Fatalf("Unexpected first line of log output: %s", output[0])
+	}
+	if output[1] != "  > Fix direct compilation of qtypeinfo.h and others" {
+		t.Fatalf("Unexpected second line of log output: %s", output[0])
+	}
+}