From bedd8eaf5d4fc917736b18975d53efb5a92f751d Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 14 Mar 2016 17:06:36 +0100 Subject: Added benchmark runner Change-Id: I2d99dc094ed65ad73a47dae48a35ca709788c061 Reviewed-by: Robin Burchell --- src/benchmarkrunner/README | 12 ++++ src/benchmarkrunner/main.go | 150 ++++++++++++++++++++++++++++++++++++++++++ src/goqtestlib/runner.go | 10 +-- src/goqtestlib/runner_test.go | 6 +- src/goqtestlib/util.go | 54 +++++++++++++++ src/goqtestlib/util_test.go | 64 ++++++++++++++++++ 6 files changed, 286 insertions(+), 10 deletions(-) create mode 100644 src/benchmarkrunner/README create mode 100644 src/benchmarkrunner/main.go create mode 100644 src/goqtestlib/util.go create mode 100644 src/goqtestlib/util_test.go (limited to 'src') diff --git a/src/benchmarkrunner/README b/src/benchmarkrunner/README new file mode 100644 index 00000000..0279fdd5 --- /dev/null +++ b/src/benchmarkrunner/README @@ -0,0 +1,12 @@ + +This little tool allows for running a series of benchmarks and collecting their results in one .zip archive, +for use with qtestcompare. + +Usage: + + (1) Change to the directory from where you'd like to run benchmarks recursively (such as tests/benchmarks) + (2) Run benchmarkrunner - this will in turn result in a call to "make benchmark" to recursively run the benchmark programs. + + +You can pass --output-file /path/to/archive.zip to store the results in a particular location. If you'd like to pass options +to qtestlib - such as "-callgrind" - then you can pass those after the parameters, i.e. `benchmarkrunner --output-file /somewhere.zip -- -callgrind` diff --git a/src/benchmarkrunner/main.go b/src/benchmarkrunner/main.go new file mode 100644 index 00000000..40e6f5b1 --- /dev/null +++ b/src/benchmarkrunner/main.go @@ -0,0 +1,150 @@ +package main + +import ( + "archive/zip" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + + "code.qt.io/qt/qtqa.git/src/goqtestlib" + "github.com/kardianos/osext" +) + +func recordResults(resultsDirectory string, workingDir string) error { + os.Unsetenv("TESTRUNNER") + + makeCommand := exec.Command("make", "benchmark") + makeCommand.Stdout = os.Stdout + makeCommand.Stderr = os.Stderr + runner := func(extraArgs []string) error { + arguments := strings.Join(extraArgs, " ") + benchmarkArgs := os.Getenv("QT_BENCHMARK_ARGS") + if benchmarkArgs != "" { + arguments = arguments + " " + benchmarkArgs + } + + defer goqtestlib.SetEnvironmentVariableAndRestoreOnExit("TESTARGS", arguments)() + return makeCommand.Run() + } + + name, err := filepath.Rel(os.Getenv("QT_BENCHMARK_BASE_DIRECTORY"), workingDir) + if err != nil { + return fmt.Errorf("Could not determine relative directory: %s", err) + } + if name == "." { + name = "testcase" + } + + repetitions := 0 + result, err := goqtestlib.GenerateTestResult(name, resultsDirectory, repetitions, runner) + if err != nil { + return err + } + + return os.Rename(result.PathToResultsXML, filepath.Join(resultsDirectory, name+".xml")) +} + +func archiveResults(resultsDir string, outputFile io.Writer) error { + archiver := zip.NewWriter(outputFile) + defer archiver.Close() + + return filepath.Walk(resultsDir, func(path string, info os.FileInfo, err error) error { + if info == nil || info.IsDir() { + return nil + } + + relativePath, err := filepath.Rel(resultsDir, path) + if err != nil { + return err + } + + sourceFile, err := os.Open(path) + if err != nil { + return err + } + defer sourceFile.Close() + + header, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + header.Name = relativePath + + file, err := archiver.CreateHeader(header) + if err != nil { + return err + } + + n, err := io.Copy(file, sourceFile) + if n != info.Size() { + return fmt.Errorf("Incorrect number of bytes written to archive for %s: Wrote %v expected %v", path, n, info.Size()) + } + return err + }) +} + +func collectResults(workingDir string) error { + self, err := osext.Executable() + if err != nil { + return fmt.Errorf("Unable to determine current executable name: %s", err) + } + + resultsDir, err := ioutil.TempDir("", "") + if err != nil { + return err + } + defer os.RemoveAll(resultsDir) + + os.Setenv("TESTRUNNER", self) + os.Setenv("QT_BENCHMARK_RECORDING_DIRECTORY", resultsDir) + os.Setenv("QT_BENCHMARK_BASE_DIRECTORY", workingDir) + os.Setenv("QT_HASH_SEED", "0") + + var outputFileName string + flag.StringVar(&outputFileName, "output-file", "results.zip", "Write collected benchmark results into specified file") + flag.Parse() + + os.Setenv("QT_BENCHMARK_ARGS", strings.Join(flag.Args(), " ")) + + makeCommand := exec.Command("make", "benchmark") + makeCommand.Stdout = os.Stdout + makeCommand.Stderr = os.Stderr + + if err := makeCommand.Run(); err != nil { + return fmt.Errorf("Error running make benchmark: %s", err) + } + + outputFile, err := os.Create(outputFileName) + if err != nil { + return fmt.Errorf("Error creating output file: %s", err) + } + defer outputFile.Close() + + return archiveResults(resultsDir, outputFile) +} + +func appMain() error { + workingDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("Error determining current working directory: %s", err) + } + + if os.Getenv("QT_BENCHMARK_RECORDING_DIRECTORY") != "" { + return recordResults(os.Getenv("QT_BENCHMARK_RECORDING_DIRECTORY"), workingDir) + } + + return collectResults(workingDir) +} + +func main() { + + if err := appMain(); err != nil { + fmt.Printf("%s\n", err) + os.Exit(2) + } +} diff --git a/src/goqtestlib/runner.go b/src/goqtestlib/runner.go index c1236d86..7e6e1b6a 100644 --- a/src/goqtestlib/runner.go +++ b/src/goqtestlib/runner.go @@ -54,7 +54,7 @@ const ( // RunFunction is used by GenerateTestResult to run unit tests in the current environment. In // the agent this usually means running "make check". -type RunFunction func() error +type RunFunction func(extraArgs []string) error // GenerateTestResult sets up the environment for QTestLib style testing and calls the runner function for // to produce the test results. The repetitionsOnFailure allows for a failing test function to fail once @@ -75,10 +75,7 @@ func GenerateTestResult(name string, resultsDirectory string, repetitionsOnFailu os.Setenv("QT_CI_RESULTS_PATH", testResult.PathToResultsXML) - os.Setenv("TESTARGS", fmt.Sprintf("-o %s,xml -o -,txt", testResult.PathToResultsXML)) - defer os.Setenv("TESTARGS", "") - - err = runner() + err = runner([]string{"-o", testResult.PathToResultsXML + ",xml", "-o", "-,txt"}) os.Setenv("QT_CI_RESULTS_PATH", "") @@ -106,8 +103,7 @@ func GenerateTestResult(name string, resultsDirectory string, repetitionsOnFailu for _, testFunction := range failingFunctions { for i := 0; i < repetitionsOnFailure; i++ { - os.Setenv("TESTARGS", testFunction) - if err = runner(); err != nil { + if err = runner([]string{testFunction}); err != nil { return nil, err } } diff --git a/src/goqtestlib/runner_test.go b/src/goqtestlib/runner_test.go index de7facdd..3dcda72d 100644 --- a/src/goqtestlib/runner_test.go +++ b/src/goqtestlib/runner_test.go @@ -84,11 +84,11 @@ func testResultsEqual(left *ParsedTestResult, right *ParsedTestResult) bool { } func makeTestRunner(t *testing.T, outputToProduce ...*ParsedTestResult) RunFunction { - return func() error { + return func(extraArgs []string) error { option := outputOption{} flagSet := flag.NewFlagSet("testlibflagset", flag.PanicOnError) flagSet.Var(&option, "o", "output specifier") - if err := flagSet.Parse(strings.Split(os.Getenv("TESTARGS"), " ")); err != nil { + if err := flagSet.Parse(extraArgs); err != nil { return err } @@ -263,7 +263,7 @@ func TestFailingOnceWithTagTestResult(t *testing.T) { } func TestFailingNonTestLibTest(t *testing.T) { - runner := func() error { + runner := func([]string) error { // simulate "make check" failing, but we did not write a results .xml file return &exec.ExitError{} } diff --git a/src/goqtestlib/util.go b/src/goqtestlib/util.go new file mode 100644 index 00000000..b43c5ecd --- /dev/null +++ b/src/goqtestlib/util.go @@ -0,0 +1,54 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtCore module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +package goqtestlib + +import ( + "os" +) + +// SetEnvironmentVariableAndRestoreOnExit immediately sets the specified environment variable to the given value. However +// as opposed to os.Setenv it returns a closure that will restore the old value. This is useful in conjunction with deferred +// calls, like so: defer SetEnvironmentVariableAndRestoreOnExit("someVar", "someValue")() +func SetEnvironmentVariableAndRestoreOnExit(variable string, value string) func() { + oldValue := os.Getenv(variable) + os.Setenv(variable, value) + return func() { + os.Setenv(variable, oldValue) + } +} diff --git a/src/goqtestlib/util_test.go b/src/goqtestlib/util_test.go new file mode 100644 index 00000000..49f3ad69 --- /dev/null +++ b/src/goqtestlib/util_test.go @@ -0,0 +1,64 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtCore module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +package goqtestlib + +import ( + "os" + "testing" +) + +func TestEnvironmentSet(t *testing.T) { + testVariable := "myVariable" + os.Setenv(testVariable, "initialValue") + + if os.Getenv(testVariable) != "initialValue" { + t.Fatal("Invalid initial environment variable value") + } + + func() { + defer SetEnvironmentVariableAndRestoreOnExit(testVariable, "tempValue")() + if os.Getenv(testVariable) != "tempValue" { + t.Fatal("Environment variable should be set to the temporary value here") + } + }() + + if os.Getenv(testVariable) != "initialValue" { + t.Fatal("Environment variable not restored") + } +} -- cgit v1.2.3