Initial import of golang bindings for reading and interpreting QTestLib XML output

This also comes with code for running qtestlib binaries and collecting the output.

Change-Id: If8cc5db654721a40ab04bbf2e4ba7a93c71ce8df
Reviewed-by: Robin Burchell <robin.burchell@viroteck.net>

4 files changed, 802 insertions, 0 deletions

diff --git a/src/goqtestlib/result.go b/src/goqtestlib/result.go
new file mode 100644
index 00000000..4aa74d98
--- /dev/null
+++ b/src/goqtestlib/result.go
@@ -0,0 +1,211 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package testlib
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+)
+
+// Environment provides information about the environment used to
+// run the unit tests, such as the Qt version used.
+type Environment struct {
+	QtVersion    string
+	QtBuild      string
+	QTestVersion string
+}
+
+// IncidentHeader provides the fields that are common for Incident
+// and Message, basically the file name and line numbers of
+// where the message was produced or the incident happened.
+type IncidentHeader struct {
+	Type string `xml:"type,attr"`
+	File string `xml:"file,attr"`
+	Line int    `xml:"line,attr"`
+}
+
+// Duration measures the time it took to run the test.
+type Duration struct {
+	Msecs float64 `xml:"msecs,attr"`
+}
+
+// Incident usually refers to a failing or a passing test function.
+type Incident struct {
+	IncidentHeader
+	DataTag string `xml:",omitempty"`
+}
+
+// Message represents an arbitrary message produced by QTestLib, usually qDebug() output.
+type Message struct {
+	IncidentHeader
+	Description string
+}
+
+// BenchmarkResult represents the results produced when running benchmarks with QTestLib.
+type BenchmarkResult struct {
+	Metric     string  `xml:"metric,attr"`
+	Tag        string  `xml:"tag,attr"`
+	Value      float64 `xml:"value,attr"`
+	Iterations int     `xml:"iterations,attr"`
+}
+
+// TestFunction represents the results of running a single test function.
+type TestFunction struct {
+	Name             string            `xml:"name,attr"`
+	Incidents        []Incident        `xml:"Incident"`
+	Messages         []Message         `xml:"Message"`
+	BenchmarkResults []BenchmarkResult `xml:"BenchmarkResult"`
+	Duration         Duration          `xml:"Duration"`
+}
+
+// FailingIncidents returns a list of incidents that represent tests failures
+// while the test function was running. For example in table driven tests
+// that is one entry per failed test row.
+func (t *TestFunction) FailingIncidents() []string {
+	var failures []string
+	for _, incident := range t.Incidents {
+		if incident.Type == "fail" {
+			name := t.Name
+			/* No support for tags at the moment due to quoting issues on Windows
+			if incident.DataTag != "" {
+				name += ":"
+				name += incident.DataTag
+			}
+			*/
+			failures = append(failures, name)
+			break
+		}
+	}
+	return failures
+}
+
+// ParsedTestResult represents the parsed output of running tests with QTestLib.
+type ParsedTestResult struct {
+	XMLName   xml.Name       `xml:"TestCase"`
+	Name      string         `xml:"name,attr"`
+	Env       Environment    `xml:"Environment"`
+	Functions []TestFunction `xml:"TestFunction"`
+	Duration  Duration       `xml:"Duration"`
+}
+
+// FailingFunctions returns list of test functions that failed during an earlier run.
+func (p *ParsedTestResult) FailingFunctions() []string {
+	var failing []string
+	for _, f := range p.Functions {
+		failing = append(failing, f.FailingIncidents()...)
+	}
+	return failing
+}
+
+// TestResult provides access to the results of executing a single unit test, such as tst_qiodevice in Qt.
+type TestResult struct {
+	TestCaseName     string // local to the module, for example tests/auto/corelib/io/qiodevice
+	PathToResultsXML string // path where the testlib xml output is stored
+}
+
+// Parse attempts to read the XML file the PathToResultsXML field points to.
+func (r *TestResult) Parse() (*ParsedTestResult, error) {
+	xmlContents, err := ioutil.ReadFile(r.PathToResultsXML)
+	if err != nil {
+		return nil, fmt.Errorf("Error reading results xml file %s: %s", r.PathToResultsXML, err)
+	}
+
+	testCase := &ParsedTestResult{}
+	if err = xml.Unmarshal(xmlContents, &testCase); err != nil {
+		return nil, fmt.Errorf("Error unmarshalling testlib xml output: %s", err)
+	}
+
+	return testCase, err
+}
+
+// TestResultCollection is a collection of test results after running tests on a module of source code.
+type TestResultCollection []TestResult
+
+// Archive produces a .tar.gz archive of the collection and writes it into the given destination writer.
+func (collection *TestResultCollection) Archive(destination io.Writer) (err error) {
+	compressor := gzip.NewWriter(destination)
+
+	archiver := tar.NewWriter(compressor)
+
+	log.Printf("Collecting %v test results ...\n", len(*collection))
+
+	for _, result := range *collection {
+		stat, err := os.Stat(result.PathToResultsXML)
+		if err != nil {
+			return fmt.Errorf("Error call stat() on %s: %s", result.PathToResultsXML, err)
+		}
+		header, err := tar.FileInfoHeader(stat, "")
+		if err != nil {
+			return fmt.Errorf("Error creating tar file header for %s: %s", result.PathToResultsXML, err)
+		}
+		header.Name = result.TestCaseName + ".xml"
+
+		if err := archiver.WriteHeader(header); err != nil {
+			return fmt.Errorf("Error writing tar file header for %s: %s", result.PathToResultsXML, err)
+		}
+
+		file, err := os.Open(result.PathToResultsXML)
+		if err != nil {
+			return fmt.Errorf("Error opening results file %s: %s", result.PathToResultsXML, err)
+		}
+
+		if _, err := io.Copy(archiver, file); err != nil {
+			file.Close()
+			return fmt.Errorf("Error writing results file %s into archive: %s", result.PathToResultsXML, err)
+		}
+
+		file.Close()
+	}
+
+	if err := archiver.Close(); err != nil {
+		return err
+	}
+
+	if err := compressor.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/src/goqtestlib/result_test.go b/src/goqtestlib/result_test.go
new file mode 100644
index 00000000..5dafa96e
--- /dev/null
+++ b/src/goqtestlib/result_test.go
@@ -0,0 +1,191 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package testlib
+
+import (
+	"encoding/xml"
+	"testing"
+)
+
+func TestXMLSchema(t *testing.T) {
+	rawXML := `<?xml version="1.0" encoding="UTF-8"?>
+<TestCase name="tst_QIODevice">
+<Environment>
+    <QtVersion>5.6.0</QtVersion>
+    <QtBuild>Qt 5.6.0 (x86_64&#x002D;little_endian&#x002D;lp64 shared (dynamic) debug build; by GCC 4.9.2)</QtBuild>
+    <QTestVersion>5.6.0</QTestVersion>
+</Environment>
+<TestFunction name="initTestCase">
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0.274962"/>
+</TestFunction>
+<TestFunction name="getSetCheck">
+<Message type="qwarn" file="" line="0">
+    <Description><![CDATA[QIODevice::seek (QTcpSocket): Cannot call seek on a sequential device]]></Description>
+    </Message>
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0.115120"/>
+</TestFunction>
+<TestFunction name="readLine2">
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[1024 - 4]]></DataTag>
+</Incident>
+<Incident type="pass" file="" line="0">
+    <DataTag><![CDATA[1024 - 3]]></DataTag>
+</Incident>
+<BenchmarkResult metric="InstructionReads" tag="" value="19838" iterations="1" />
+</TestFunction>
+<Duration msecs="760.801970"/>
+</TestCase>`
+
+	actual := &ParsedTestResult{}
+	err := xml.Unmarshal([]byte(rawXML), &actual)
+	if err != nil {
+		t.Errorf("Error decoding XML: %s", err)
+		t.FailNow()
+	}
+
+	if actual.Name != "tst_QIODevice" {
+		t.Errorf("Invalid name attribute decoding. Got %s", actual.Name)
+	}
+
+	if actual.Env.QtVersion != "5.6.0" {
+		t.Errorf("Error decoding Qt version from environment. Got %s", actual.Env.QtVersion)
+	}
+
+	if actual.Duration.Msecs != 760.801970 {
+		t.Errorf("Error decoding Test Duration. Got %f", actual.Duration.Msecs)
+	}
+
+	if len(actual.Functions) != 3 {
+		t.Errorf("Incorrect number of parsed test functions. Got %v", len(actual.Functions))
+		t.FailNow()
+	}
+
+	function := actual.Functions[0]
+
+	if function.Name != "initTestCase" {
+		t.Errorf("Incorrectly parsed test function name for first test function. Got %v", function.Name)
+	}
+
+	if len(function.Incidents) != 1 {
+		t.Errorf("Incorrectly parsed incidents for first test function. Parsed %v incidents", len(function.Incidents))
+		t.FailNow()
+	}
+
+	if function.Duration.Msecs != 0.274962 {
+		t.Errorf("Wrong duration %v", function.Duration.Msecs)
+	}
+
+	incident := function.Incidents[0]
+
+	if incident.Type != "pass" {
+		t.Errorf("Incorrectly parsed type for incident. Got %v", incident.Type)
+	}
+
+	if incident.File != "" {
+		t.Errorf("Incorrectly parsed file name for incident. Got %v", incident.File)
+	}
+
+	if incident.Line != 0 {
+		t.Errorf("Incorrectly parsed line number for incident. Got %v", incident.Line)
+	}
+
+	function = actual.Functions[1]
+
+	if len(function.Messages) != 1 {
+		t.Errorf("Incorrectly parsed number of messages incident. Got %v", len(function.Messages))
+	}
+
+	if function.Duration.Msecs != 0.115120 {
+		t.Errorf("Wrong duration %v", function.Duration.Msecs)
+	}
+
+	message := function.Messages[0]
+
+	if message.Type != "qwarn" {
+		t.Errorf("Incorrectly parsed message type. Got %v", len(message.Type))
+	}
+
+	if message.Description != "QIODevice::seek (QTcpSocket): Cannot call seek on a sequential device" {
+		t.Errorf("Incorrectly parsed message description. Got %v", message.Description)
+	}
+
+	function = actual.Functions[2]
+
+	if len(function.Incidents) != 2 {
+		t.Errorf("Incorrectly parsed incidents for third test function. Parsed %v incidents", len(function.Incidents))
+		t.FailNow()
+	}
+
+	if function.Duration.Msecs != 0.0 {
+		t.Errorf("Wrong duration %v", function.Duration.Msecs)
+	}
+
+	incident = function.Incidents[0]
+	if incident.Type != "pass" {
+		t.Errorf("Incorrectly parsed incident type. Got %v", incident.Type)
+	}
+
+	if incident.DataTag != "1024 - 4" {
+		t.Errorf("Incorrectly parsed incident data tag. Got %v", incident.Type)
+	}
+
+	incident = function.Incidents[1]
+	if incident.DataTag != "1024 - 3" {
+		t.Errorf("Incorrectly parsed incident data tag. Got %v", incident.Type)
+	}
+
+	if len(function.BenchmarkResults) != 1 {
+		t.Fatalf("Incorrectly parsed number of benchmark results. Expected 1 got %v", len(function.BenchmarkResults))
+	}
+
+	result := function.BenchmarkResults[0]
+	if result.Metric != "InstructionReads" {
+		t.Errorf("Incorrectly parsed benchmark metric %s", result.Metric)
+	}
+
+	if result.Value != 19838 {
+		t.Errorf("Incorrectly parsed benchmark value %v", result.Value)
+	}
+
+	if result.Iterations != 1 {
+		t.Errorf("Incorrectly parsed benchmark iteration count %v", result.Iterations)
+	}
+}
diff --git a/src/goqtestlib/runner.go b/src/goqtestlib/runner.go
new file mode 100644
index 00000000..eb67c667
--- /dev/null
+++ b/src/goqtestlib/runner.go
@@ -0,0 +1,119 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package testlib
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+const (
+	// NoRepetitionsOnFailure is a convenience value for indicating that even a single failing test function is considered an overall failure.
+	NoRepetitionsOnFailure int = 0
+)
+
+// RunFunction is used by GenerateTestResult to run unit tests in the current environment. In
+// the agent this usually means running "make check".
+type RunFunction func() error
+
+// GenerateTestResult sets up the environment for QTestLib style testing and calls the runner function for
+// to produce the test results. The repetitionsOnFailure allows for a failing test function to fail once
+// and have that failure to be ignored under the condition that consequent repeated running of the same
+// test function does not produce any failures.
+func GenerateTestResult(name string, resultsDirectory string, repetitionsOnFailure int, runner RunFunction) (*TestResult, error) {
+	resultsDir := filepath.Join(resultsDirectory, filepath.Dir(name))
+	os.MkdirAll(resultsDir, 0755)
+	resultsFile, err := ioutil.TempFile(resultsDirectory, name)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating temporary file to collected test output: %s", err)
+	}
+	testResult := &TestResult{
+		TestCaseName:     name,
+		PathToResultsXML: resultsFile.Name(),
+	}
+	resultsFile.Close()
+
+	os.Setenv("QT_CI_RESULTS_PATH", testResult.PathToResultsXML)
+
+	os.Setenv("TESTARGS", fmt.Sprintf("-o %s,xml -o -,txt", testResult.PathToResultsXML))
+	defer os.Setenv("TESTARGS", "")
+
+	err = runner()
+
+	os.Setenv("QT_CI_RESULTS_PATH", "")
+
+	_, ok := err.(*exec.ExitError)
+	if err != nil && !ok {
+		return nil, err
+	}
+
+	fileInfo, statErr := os.Stat(testResult.PathToResultsXML)
+	if os.IsNotExist(statErr) || fileInfo.Size() == 0 {
+		return nil, err
+	}
+
+	if err != nil {
+		parsedResult, err := testResult.Parse()
+		if err != nil {
+			return nil, err
+		}
+
+		failingFunctions := parsedResult.FailingFunctions()
+		if len(failingFunctions) > 0 {
+			if repetitionsOnFailure == 0 {
+				return nil, errors.New("Tests failed")
+			}
+
+			for _, testFunction := range failingFunctions {
+				for i := 0; i < repetitionsOnFailure; i++ {
+					os.Setenv("TESTARGS", testFunction)
+					if err = runner(); err != nil {
+						return nil, err
+					}
+				}
+			}
+		}
+	}
+
+	return testResult, nil
+}
diff --git a/src/goqtestlib/runner_test.go b/src/goqtestlib/runner_test.go
new file mode 100644
index 00000000..1cd7281e
--- /dev/null
+++ b/src/goqtestlib/runner_test.go
@@ -0,0 +1,281 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package testlib
+
+import (
+	"encoding/xml"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+type Output struct {
+	file   string
+	format string
+}
+
+type outputOption struct {
+	outputs []Output
+}
+
+func (o *outputOption) Set(value string) error {
+	s := strings.Split(value, ",")
+	o.outputs = append(o.outputs, Output{file: s[0], format: s[1]})
+	return nil
+}
+
+func (o *outputOption) String() string {
+	return fmt.Sprintf("%v", o.outputs)
+}
+
+func (o *outputOption) XMLOutputFileName() string {
+	for _, o := range o.outputs {
+		if o.file != "-" && o.format == "xml" {
+			return o.file
+		}
+	}
+	return ""
+}
+
+func testResultsEqual(left *ParsedTestResult, right *ParsedTestResult) bool {
+	leftBytes, _ := xml.Marshal(left)
+	rightBytes, _ := xml.Marshal(right)
+	return string(leftBytes) == string(rightBytes)
+}
+
+func makeTestRunner(t *testing.T, outputToProduce ...*ParsedTestResult) RunFunction {
+	return func() error {
+		option := outputOption{}
+		flagSet := flag.NewFlagSet("testlibflagset", flag.PanicOnError)
+		flagSet.Var(&option, "o", "output specifier")
+		if err := flagSet.Parse(strings.Split(os.Getenv("TESTARGS"), " ")); err != nil {
+			return err
+		}
+
+		output := outputToProduce[0]
+		outputToProduce = outputToProduce[1:]
+
+		if outputFile := option.XMLOutputFileName(); outputFile != "" {
+			bytes, err := xml.MarshalIndent(output, "    ", "    ")
+			if err != nil {
+				return err
+			}
+			if err := ioutil.WriteFile(outputFile, bytes, 0644); err != nil {
+				return err
+			}
+		}
+
+		if len(output.FailingFunctions()) > 0 {
+			return &exec.ExitError{}
+		}
+
+		return nil
+	}
+}
+
+const (
+	initialXMLOutputToProduce = `<?xml version="1.0" encoding="UTF-8"?>
+<TestCase name="tst_QIODevice">
+<Environment>
+    <QtVersion>5.6.0</QtVersion>
+    <QtBuild>Qt 5.6.0 (x86_64&#x002D;little_endian&#x002D;lp64 shared (dynamic) debug build; by GCC 4.9.2)</QtBuild>
+    <QTestVersion>5.6.0</QTestVersion>
+</Environment>
+<TestFunction name="initTestCase">
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0.274962"/>
+</TestFunction>
+</TestCase>
+`
+)
+
+func TestGenerateTestResult(t *testing.T) {
+	passingOutput := &ParsedTestResult{}
+	xml.Unmarshal([]byte(initialXMLOutputToProduce), passingOutput)
+
+	testRunner := makeTestRunner(t, passingOutput)
+
+	result, err := GenerateTestResult("testname", os.TempDir(), NoRepetitionsOnFailure, testRunner)
+	if err != nil {
+		t.Errorf("Error generating test result: %s", err)
+		t.FailNow()
+	}
+
+	if result == nil {
+		t.Errorf("Unexpected missing test result")
+		t.FailNow()
+	}
+
+	if result.TestCaseName != "testname" {
+		t.Errorf("Unexpected test case name. Got %s", result.TestCaseName)
+	}
+
+	actualParsedResult, err := result.Parse()
+	if err != nil {
+		t.Errorf("Could not read/parse results xml file at %s: %s", result.PathToResultsXML, err)
+		t.FailNow()
+	}
+
+	if !testResultsEqual(actualParsedResult, passingOutput) {
+		t.Errorf("Unexpected actual output. Got %v", actualParsedResult)
+		t.FailNow()
+	}
+}
+
+func TestFailingOnceTestResult(t *testing.T) {
+	failingOutput := &ParsedTestResult{}
+	xml.Unmarshal([]byte(initialXMLOutputToProduce), failingOutput)
+	failingOutput.Functions[0].Incidents[0].Type = "fail"
+
+	passingOutput := &ParsedTestResult{}
+	xml.Unmarshal([]byte(initialXMLOutputToProduce), passingOutput)
+	passingOutput.Functions[0].Incidents[0].Type = "pass"
+
+	testRunner := makeTestRunner(t, failingOutput, passingOutput, passingOutput)
+
+	result, err := GenerateTestResult("testname", os.TempDir(), 2 /*repetitions*/, testRunner)
+	if err != nil {
+		t.Errorf("Error generating test result: %s", err)
+		t.FailNow()
+	}
+
+	if result == nil {
+		t.Errorf("Unexpected missing test result")
+		t.FailNow()
+	}
+
+	if result.TestCaseName != "testname" {
+		t.Errorf("Unexpected test case name. Got %s", result.TestCaseName)
+	}
+
+	actualParsedResult, err := result.Parse()
+	if err != nil {
+		t.Errorf("Could not read/parse results xml file at %s: %s", result.PathToResultsXML, err)
+		t.FailNow()
+	}
+
+	if !testResultsEqual(actualParsedResult, failingOutput) {
+		t.Errorf("Unexpected actual  output. Got %v", actualParsedResult)
+		t.FailNow()
+	}
+}
+
+func TestFailingOnceWithTagTestResult(t *testing.T) {
+	failingOutput := &ParsedTestResult{}
+	xml.Unmarshal([]byte(initialXMLOutputToProduce), failingOutput)
+
+	passingIncident1 := Incident{}
+	passingIncident1.Type = "pass"
+	passingIncident1.DataTag = "tag1"
+
+	failingIncident2 := Incident{}
+	failingIncident2.Type = "fail"
+	failingIncident2.DataTag = "tag2"
+
+	passingIncident3 := Incident{}
+	passingIncident3.Type = "pass"
+	passingIncident3.DataTag = "tag3"
+
+	failingOutput.Functions[0].Incidents = append([]Incident(nil), passingIncident1, failingIncident2, passingIncident3)
+
+	if len(failingOutput.Functions[0].FailingIncidents()) != 1 {
+		t.Errorf("Incorrect number of failing incidents. Got %v", failingOutput.Functions[0].FailingIncidents())
+		t.FailNow()
+	}
+	if failingOutput.Functions[0].FailingIncidents()[0] != "initTestCase" {
+		t.Errorf("Incorrect failing tagged incidents. Got %v", failingOutput.Functions[0].FailingIncidents()[0])
+		t.FailNow()
+	}
+
+	passingOutput := &ParsedTestResult{}
+	xml.Unmarshal([]byte(initialXMLOutputToProduce), passingOutput)
+
+	passingOutput.Functions[0].Incidents = append([]Incident(nil), failingOutput.Functions[0].Incidents...)
+	passingOutput.Functions[0].Incidents[1].Type = "pass"
+
+	testRunner := makeTestRunner(t, failingOutput, passingOutput, passingOutput)
+
+	result, err := GenerateTestResult("testname", os.TempDir(), 2 /*repetitions*/, testRunner)
+	if err != nil {
+		t.Errorf("Error generating test result: %s", err)
+		t.FailNow()
+	}
+
+	if result == nil {
+		t.Errorf("Unexpected missing test result")
+		t.FailNow()
+	}
+
+	if result.TestCaseName != "testname" {
+		t.Errorf("Unexpected test case name. Got %s", result.TestCaseName)
+	}
+
+	actualParsedResult, err := result.Parse()
+	if err != nil {
+		t.Errorf("Could not read/parse results xml file at %s: %s", result.PathToResultsXML, err)
+		t.FailNow()
+	}
+
+	if !testResultsEqual(actualParsedResult, failingOutput) {
+		t.Errorf("Unexpected actual  output. Got %v", actualParsedResult)
+		t.FailNow()
+	}
+}
+
+func TestFailingNonTestLibTest(t *testing.T) {
+	runner := func() error {
+		// simulate "make check" failing, but we did not write a results .xml file
+		return &exec.ExitError{}
+	}
+
+	result, err := GenerateTestResult("testname", os.TempDir(), NoRepetitionsOnFailure, runner)
+	if result != nil {
+		t.Errorf("Unexpected test result. Expected nil got %v", result)
+		t.FailNow()
+	}
+
+	if err == nil {
+		t.Errorf("Unexpected test success. Expected error")
+		t.FailNow()
+	}
+}