Added benchmark runner

Change-Id: I2d99dc094ed65ad73a47dae48a35ca709788c061
Reviewed-by: Robin Burchell <robin.burchell@viroteck.net>

6 files changed, 286 insertions, 10 deletions

diff --git a/src/benchmarkrunner/README b/src/benchmarkrunner/README
new file mode 100644
index 00000000..0279fdd5
--- /dev/null
+++ b/src/benchmarkrunner/README
@@ -0,0 +1,12 @@
+
+This little tool allows for running a series of benchmarks and collecting their results in one .zip archive,
+for use with qtestcompare.
+
+Usage:
+
+    (1) Change to the directory from where you'd like to run benchmarks recursively (such as tests/benchmarks)
+    (2) Run benchmarkrunner - this will in turn result in a call to "make benchmark" to recursively run the benchmark programs.
+
+
+You can pass --output-file /path/to/archive.zip to store the results in a particular location. If you'd like to pass options
+to qtestlib - such as "-callgrind" - then you can pass those after the parameters, i.e. `benchmarkrunner --output-file /somewhere.zip -- -callgrind`
diff --git a/src/benchmarkrunner/main.go b/src/benchmarkrunner/main.go
new file mode 100644
index 00000000..40e6f5b1
--- /dev/null
+++ b/src/benchmarkrunner/main.go
@@ -0,0 +1,150 @@
+package main
+
+import (
+	"archive/zip"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"code.qt.io/qt/qtqa.git/src/goqtestlib"
+	"github.com/kardianos/osext"
+)
+
+func recordResults(resultsDirectory string, workingDir string) error {
+	os.Unsetenv("TESTRUNNER")
+
+	makeCommand := exec.Command("make", "benchmark")
+	makeCommand.Stdout = os.Stdout
+	makeCommand.Stderr = os.Stderr
+	runner := func(extraArgs []string) error {
+		arguments := strings.Join(extraArgs, " ")
+		benchmarkArgs := os.Getenv("QT_BENCHMARK_ARGS")
+		if benchmarkArgs != "" {
+			arguments = arguments + " " + benchmarkArgs
+		}
+
+		defer goqtestlib.SetEnvironmentVariableAndRestoreOnExit("TESTARGS", arguments)()
+		return makeCommand.Run()
+	}
+
+	name, err := filepath.Rel(os.Getenv("QT_BENCHMARK_BASE_DIRECTORY"), workingDir)
+	if err != nil {
+		return fmt.Errorf("Could not determine relative directory: %s", err)
+	}
+	if name == "." {
+		name = "testcase"
+	}
+
+	repetitions := 0
+	result, err := goqtestlib.GenerateTestResult(name, resultsDirectory, repetitions, runner)
+	if err != nil {
+		return err
+	}
+
+	return os.Rename(result.PathToResultsXML, filepath.Join(resultsDirectory, name+".xml"))
+}
+
+func archiveResults(resultsDir string, outputFile io.Writer) error {
+	archiver := zip.NewWriter(outputFile)
+	defer archiver.Close()
+
+	return filepath.Walk(resultsDir, func(path string, info os.FileInfo, err error) error {
+		if info == nil || info.IsDir() {
+			return nil
+		}
+
+		relativePath, err := filepath.Rel(resultsDir, path)
+		if err != nil {
+			return err
+		}
+
+		sourceFile, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer sourceFile.Close()
+
+		header, err := zip.FileInfoHeader(info)
+		if err != nil {
+			return err
+		}
+		header.Name = relativePath
+
+		file, err := archiver.CreateHeader(header)
+		if err != nil {
+			return err
+		}
+
+		n, err := io.Copy(file, sourceFile)
+		if n != info.Size() {
+			return fmt.Errorf("Incorrect number of bytes written to archive for %s: Wrote %v expected %v", path, n, info.Size())
+		}
+		return err
+	})
+}
+
+func collectResults(workingDir string) error {
+	self, err := osext.Executable()
+	if err != nil {
+		return fmt.Errorf("Unable to determine current executable name: %s", err)
+	}
+
+	resultsDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(resultsDir)
+
+	os.Setenv("TESTRUNNER", self)
+	os.Setenv("QT_BENCHMARK_RECORDING_DIRECTORY", resultsDir)
+	os.Setenv("QT_BENCHMARK_BASE_DIRECTORY", workingDir)
+	os.Setenv("QT_HASH_SEED", "0")
+
+	var outputFileName string
+	flag.StringVar(&outputFileName, "output-file", "results.zip", "Write collected benchmark results into specified file")
+	flag.Parse()
+
+	os.Setenv("QT_BENCHMARK_ARGS", strings.Join(flag.Args(), " "))
+
+	makeCommand := exec.Command("make", "benchmark")
+	makeCommand.Stdout = os.Stdout
+	makeCommand.Stderr = os.Stderr
+
+	if err := makeCommand.Run(); err != nil {
+		return fmt.Errorf("Error running make benchmark: %s", err)
+	}
+
+	outputFile, err := os.Create(outputFileName)
+	if err != nil {
+		return fmt.Errorf("Error creating output file: %s", err)
+	}
+	defer outputFile.Close()
+
+	return archiveResults(resultsDir, outputFile)
+}
+
+func appMain() error {
+	workingDir, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("Error determining current working directory: %s", err)
+	}
+
+	if os.Getenv("QT_BENCHMARK_RECORDING_DIRECTORY") != "" {
+		return recordResults(os.Getenv("QT_BENCHMARK_RECORDING_DIRECTORY"), workingDir)
+	}
+
+	return collectResults(workingDir)
+}
+
+func main() {
+
+	if err := appMain(); err != nil {
+		fmt.Printf("%s\n", err)
+		os.Exit(2)
+	}
+}
diff --git a/src/goqtestlib/runner.go b/src/goqtestlib/runner.go
index c1236d86..7e6e1b6a 100644
--- a/src/goqtestlib/runner.go
+++ b/src/goqtestlib/runner.go
@@ -54,7 +54,7 @@ const (
 
 // RunFunction is used by GenerateTestResult to run unit tests in the current environment. In
 // the agent this usually means running "make check".
-type RunFunction func() error
+type RunFunction func(extraArgs []string) error
 
 // GenerateTestResult sets up the environment for QTestLib style testing and calls the runner function for
 // to produce the test results. The repetitionsOnFailure allows for a failing test function to fail once
@@ -75,10 +75,7 @@ func GenerateTestResult(name string, resultsDirectory string, repetitionsOnFailu
 
 	os.Setenv("QT_CI_RESULTS_PATH", testResult.PathToResultsXML)
 
-	os.Setenv("TESTARGS", fmt.Sprintf("-o %s,xml -o -,txt", testResult.PathToResultsXML))
-	defer os.Setenv("TESTARGS", "")
-
-	err = runner()
+	err = runner([]string{"-o", testResult.PathToResultsXML + ",xml", "-o", "-,txt"})
 
 	os.Setenv("QT_CI_RESULTS_PATH", "")
 
@@ -106,8 +103,7 @@ func GenerateTestResult(name string, resultsDirectory string, repetitionsOnFailu
 
 			for _, testFunction := range failingFunctions {
 				for i := 0; i < repetitionsOnFailure; i++ {
-					os.Setenv("TESTARGS", testFunction)
-					if err = runner(); err != nil {
+					if err = runner([]string{testFunction}); err != nil {
 						return nil, err
 					}
 				}
diff --git a/src/goqtestlib/runner_test.go b/src/goqtestlib/runner_test.go
index de7facdd..3dcda72d 100644
--- a/src/goqtestlib/runner_test.go
+++ b/src/goqtestlib/runner_test.go
@@ -84,11 +84,11 @@ func testResultsEqual(left *ParsedTestResult, right *ParsedTestResult) bool {
 }
 
 func makeTestRunner(t *testing.T, outputToProduce ...*ParsedTestResult) RunFunction {
-	return func() error {
+	return func(extraArgs []string) error {
 		option := outputOption{}
 		flagSet := flag.NewFlagSet("testlibflagset", flag.PanicOnError)
 		flagSet.Var(&option, "o", "output specifier")
-		if err := flagSet.Parse(strings.Split(os.Getenv("TESTARGS"), " ")); err != nil {
+		if err := flagSet.Parse(extraArgs); err != nil {
 			return err
 		}
 
@@ -263,7 +263,7 @@ func TestFailingOnceWithTagTestResult(t *testing.T) {
 }
 
 func TestFailingNonTestLibTest(t *testing.T) {
-	runner := func() error {
+	runner := func([]string) error {
 		// simulate "make check" failing, but we did not write a results .xml file
 		return &exec.ExitError{}
 	}
diff --git a/src/goqtestlib/util.go b/src/goqtestlib/util.go
new file mode 100644
index 00000000..b43c5ecd
--- /dev/null
+++ b/src/goqtestlib/util.go
@@ -0,0 +1,54 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package goqtestlib
+
+import (
+	"os"
+)
+
+// SetEnvironmentVariableAndRestoreOnExit immediately sets the specified environment variable to the given value. However
+// as opposed to os.Setenv it returns a closure that will restore the old value. This is useful in conjunction with deferred
+// calls, like so: defer SetEnvironmentVariableAndRestoreOnExit("someVar", "someValue")()
+func SetEnvironmentVariableAndRestoreOnExit(variable string, value string) func() {
+	oldValue := os.Getenv(variable)
+	os.Setenv(variable, value)
+	return func() {
+		os.Setenv(variable, oldValue)
+	}
+}
diff --git a/src/goqtestlib/util_test.go b/src/goqtestlib/util_test.go
new file mode 100644
index 00000000..49f3ad69
--- /dev/null
+++ b/src/goqtestlib/util_test.go
@@ -0,0 +1,64 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package goqtestlib
+
+import (
+	"os"
+	"testing"
+)
+
+func TestEnvironmentSet(t *testing.T) {
+	testVariable := "myVariable"
+	os.Setenv(testVariable, "initialValue")
+
+	if os.Getenv(testVariable) != "initialValue" {
+		t.Fatal("Invalid initial environment variable value")
+	}
+
+	func() {
+		defer SetEnvironmentVariableAndRestoreOnExit(testVariable, "tempValue")()
+		if os.Getenv(testVariable) != "tempValue" {
+			t.Fatal("Environment variable should be set to the temporary value here")
+		}
+	}()
+
+	if os.Getenv(testVariable) != "initialValue" {
+		t.Fatal("Environment variable not restored")
+	}
+}