Added benchmark runner

Change-Id: I2d99dc094ed65ad73a47dae48a35ca709788c061
Reviewed-by: Robin Burchell <robin.burchell@viroteck.net>

4 files changed, 124 insertions, 10 deletions

diff --git a/src/goqtestlib/runner.go b/src/goqtestlib/runner.go
index c1236d86..7e6e1b6a 100644
--- a/src/goqtestlib/runner.go
+++ b/src/goqtestlib/runner.go
@@ -54,7 +54,7 @@ const (
 
 // RunFunction is used by GenerateTestResult to run unit tests in the current environment. In
 // the agent this usually means running "make check".
-type RunFunction func() error
+type RunFunction func(extraArgs []string) error
 
 // GenerateTestResult sets up the environment for QTestLib style testing and calls the runner function for
 // to produce the test results. The repetitionsOnFailure allows for a failing test function to fail once
@@ -75,10 +75,7 @@ func GenerateTestResult(name string, resultsDirectory string, repetitionsOnFailu
 
 	os.Setenv("QT_CI_RESULTS_PATH", testResult.PathToResultsXML)
 
-	os.Setenv("TESTARGS", fmt.Sprintf("-o %s,xml -o -,txt", testResult.PathToResultsXML))
-	defer os.Setenv("TESTARGS", "")
-
-	err = runner()
+	err = runner([]string{"-o", testResult.PathToResultsXML + ",xml", "-o", "-,txt"})
 
 	os.Setenv("QT_CI_RESULTS_PATH", "")
 
@@ -106,8 +103,7 @@ func GenerateTestResult(name string, resultsDirectory string, repetitionsOnFailu
 
 			for _, testFunction := range failingFunctions {
 				for i := 0; i < repetitionsOnFailure; i++ {
-					os.Setenv("TESTARGS", testFunction)
-					if err = runner(); err != nil {
+					if err = runner([]string{testFunction}); err != nil {
 						return nil, err
 					}
 				}
diff --git a/src/goqtestlib/runner_test.go b/src/goqtestlib/runner_test.go
index de7facdd..3dcda72d 100644
--- a/src/goqtestlib/runner_test.go
+++ b/src/goqtestlib/runner_test.go
@@ -84,11 +84,11 @@ func testResultsEqual(left *ParsedTestResult, right *ParsedTestResult) bool {
 }
 
 func makeTestRunner(t *testing.T, outputToProduce ...*ParsedTestResult) RunFunction {
-	return func() error {
+	return func(extraArgs []string) error {
 		option := outputOption{}
 		flagSet := flag.NewFlagSet("testlibflagset", flag.PanicOnError)
 		flagSet.Var(&option, "o", "output specifier")
-		if err := flagSet.Parse(strings.Split(os.Getenv("TESTARGS"), " ")); err != nil {
+		if err := flagSet.Parse(extraArgs); err != nil {
 			return err
 		}
 
@@ -263,7 +263,7 @@ func TestFailingOnceWithTagTestResult(t *testing.T) {
 }
 
 func TestFailingNonTestLibTest(t *testing.T) {
-	runner := func() error {
+	runner := func([]string) error {
 		// simulate "make check" failing, but we did not write a results .xml file
 		return &exec.ExitError{}
 	}
diff --git a/src/goqtestlib/util.go b/src/goqtestlib/util.go
new file mode 100644
index 00000000..b43c5ecd
--- /dev/null
+++ b/src/goqtestlib/util.go
@@ -0,0 +1,54 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package goqtestlib
+
+import (
+	"os"
+)
+
+// SetEnvironmentVariableAndRestoreOnExit immediately sets the specified environment variable to the given value. However
+// as opposed to os.Setenv it returns a closure that will restore the old value. This is useful in conjunction with deferred
+// calls, like so: defer SetEnvironmentVariableAndRestoreOnExit("someVar", "someValue")()
+func SetEnvironmentVariableAndRestoreOnExit(variable string, value string) func() {
+	oldValue := os.Getenv(variable)
+	os.Setenv(variable, value)
+	return func() {
+		os.Setenv(variable, oldValue)
+	}
+}
diff --git a/src/goqtestlib/util_test.go b/src/goqtestlib/util_test.go
new file mode 100644
index 00000000..49f3ad69
--- /dev/null
+++ b/src/goqtestlib/util_test.go
@@ -0,0 +1,64 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+package goqtestlib
+
+import (
+	"os"
+	"testing"
+)
+
+func TestEnvironmentSet(t *testing.T) {
+	testVariable := "myVariable"
+	os.Setenv(testVariable, "initialValue")
+
+	if os.Getenv(testVariable) != "initialValue" {
+		t.Fatal("Invalid initial environment variable value")
+	}
+
+	func() {
+		defer SetEnvironmentVariableAndRestoreOnExit(testVariable, "tempValue")()
+		if os.Getenv(testVariable) != "tempValue" {
+			t.Fatal("Environment variable should be set to the temporary value here")
+		}
+	}()
+
+	if os.Getenv(testVariable) != "initialValue" {
+		t.Fatal("Environment variable not restored")
+	}
+}