summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorRobin Burchell <robin.burchell@viroteck.net>2016-03-18 00:40:59 +0100
committerSimon Hausmann <simon.hausmann@theqtcompany.com>2016-03-21 10:52:49 +0000
commit81361d72d69c64e625e68509b36df6a4018207af (patch)
treed9584b21d4ead6836b3b4a4599ec9e1a59a27b0a /src
parent4f7931b473ecbb96dc6724992c62518a3152265b (diff)
qtestcompare: Add handling of zipped archives from benchmarkrunner.
Change-Id: I45cc05fb1e4dae2a5a688825501ba59e9567f7a0 Reviewed-by: Robin Burchell <robin.burchell@viroteck.net>
Diffstat (limited to 'src')
-rw-r--r--src/qtestcompare/main.go165
1 files changed, 136 insertions, 29 deletions
diff --git a/src/qtestcompare/main.go b/src/qtestcompare/main.go
index cb493303..27d5b08c 100644
--- a/src/qtestcompare/main.go
+++ b/src/qtestcompare/main.go
@@ -40,8 +40,12 @@
package main
import (
+ "archive/zip"
+ "encoding/xml"
"flag"
"fmt"
+ "io"
+ "io/ioutil"
"log"
"os"
"sort"
@@ -90,54 +94,52 @@ func describeChange(pChange float64) string {
}
}
-func compareSingleTestRuns(oxml string, nxml string) {
- oldTest := loadTestResult(oxml)
- newTest := loadTestResult(nxml)
-
- if oldTest.Name != newTest.Name {
- log.Fatalf("I can't compare two totally different things (old: %s, new: %s)", oldTest.Name, newTest.Name)
- return
- }
-
- // merge the test functions into a singular representation.
- mergedResults := map[string]MergedTestResult{}
+type MergedTestResults map[string]MergedTestResult
+func (results *MergedTestResults) addOldTestCase(prefix string, testCase *goqtestlib.ParsedTestResult) {
// XXX: add a way to specify what type of benchmarkresult to look for.
- for _, fn := range oldTest.Functions {
+ for _, fn := range testCase.Functions {
+ qualifiedName := prefix + fn.Name
for _, br := range fn.BenchmarkResults {
- res := mergedResults[fn.Name]
- res.Name = fn.Name
+ res := (*results)[qualifiedName]
+ res.Name = qualifiedName
if br.Metric == "WalltimeMilliseconds" {
res.OldDuration = &(br.Value)
} else if br.Metric == "InstructionReads" {
res.OldInstructionReads = &(br.Value)
}
- mergedResults[fn.Name] = res
+ (*results)[qualifiedName] = res
}
}
- for _, fn := range newTest.Functions {
+}
+
+func (results *MergedTestResults) addNewTestCase(prefix string, testCase *goqtestlib.ParsedTestResult) {
+ for _, fn := range testCase.Functions {
+ qualifiedName := prefix + fn.Name
for _, br := range fn.BenchmarkResults {
- res := mergedResults[fn.Name]
- res.Name = fn.Name
+ res := (*results)[qualifiedName]
+ res.Name = qualifiedName
if br.Metric == "WalltimeMilliseconds" {
res.NewDuration = &(br.Value)
} else if br.Metric == "InstructionReads" {
res.NewInstructionReads = &(br.Value)
}
- mergedResults[fn.Name] = res
+ (*results)[qualifiedName] = res
}
}
+}
+func (results *MergedTestResults) compare(output io.Writer) {
// convert mergedResults to a slice, and sort it for stable results.
sortedResults := []MergedTestResult{}
- for _, mr := range mergedResults {
+ for _, mr := range *results {
sortedResults = append(sortedResults, mr)
}
sort.Sort(ByName(sortedResults))
- table := tablewriter.NewWriter(os.Stdout)
+ table := tablewriter.NewWriter(output)
table.SetHeader([]string{"Test", "From", "To", "Details"})
table.SetBorder(false)
@@ -200,27 +202,132 @@ func compareSingleTestRuns(oxml string, nxml string) {
verdict = "more or less the same"
}
- table.SetFooter([]string{newTest.Name, "", "", verdict})
+ table.SetFooter([]string{"Overall result", "", "", verdict})
table.Render()
+
+}
+
+func compareSingleTestRuns(oxml string, nxml string) {
+ oldTest := loadTestResult(oxml)
+ newTest := loadTestResult(nxml)
+
+ if oldTest.Name != newTest.Name {
+ log.Fatalf("I can't compare two totally different things (old: %s, new: %s)", oldTest.Name, newTest.Name)
+ return
+ }
+
+ // merge the test functions into a singular representation.
+ mergedResults := MergedTestResults{}
+ prefix := ""
+ mergedResults.addOldTestCase(prefix, oldTest)
+ mergedResults.addNewTestCase(prefix, newTest)
+
+ mergedResults.compare(os.Stdout)
+}
+
+func unmarshalTestResult(reader io.ReadCloser) (*goqtestlib.ParsedTestResult, error) {
+ defer reader.Close()
+ xmlContents, err := ioutil.ReadAll(reader)
+ if err != nil {
+ return nil, fmt.Errorf("Error reading results xml file: %s", err)
+ }
+
+ testCase := &goqtestlib.ParsedTestResult{}
+ if err = xml.Unmarshal(xmlContents, &testCase); err != nil {
+ return nil, fmt.Errorf("Error unmarshalling testlib xml output: %s", err)
+ }
+ return testCase, nil
+}
+
+type testArchive struct {
+ reader *zip.ReadCloser
+}
+
+func openTestArchive(path string) (*testArchive, error) {
+ reader, err := zip.OpenReader(path)
+ return &testArchive{reader}, err
+}
+
+func (archive *testArchive) forEachTestCase(callback func(path string, testCase *goqtestlib.ParsedTestResult) error) error {
+ for _, f := range archive.reader.File {
+ reader, err := f.Open()
+ if err != nil {
+ log.Fatalf("Error opening entry in zip archive %s: %s", f.Name, err)
+ }
+ result, err := unmarshalTestResult(reader)
+ if err != nil {
+ log.Fatalf("Error unpacking test result for %s from zip archive: %s", f.Name, err)
+ }
+ if err := callback(f.Name, result); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (archive *testArchive) Close() error {
+ return archive.reader.Close()
+}
+
+func compareArchivedTestRuns(oarch string, narch string) {
+ fmt.Printf("Comparing zipped runs %s vs %s\n", oarch, narch)
+
+ or, err := openTestArchive(oarch)
+ if err != nil {
+ log.Fatalf("Can't open old archive: %s\n", err)
+ }
+ defer or.Close()
+
+ nr, err := openTestArchive(narch)
+ if err != nil {
+ log.Fatalf("Can't open new archive: %s\n", err)
+ }
+ defer nr.Close()
+
+ // this is a map of xml to result, so e.g:
+ // qml/binding.xml -> result.
+ mergedResults := MergedTestResults{}
+
+ or.forEachTestCase(func(path string, testCase *goqtestlib.ParsedTestResult) error {
+ mergedResults.addOldTestCase(path+"/", testCase)
+ return nil
+ })
+ nr.forEachTestCase(func(path string, testCase *goqtestlib.ParsedTestResult) error {
+ mergedResults.addNewTestCase(path+"/", testCase)
+ return nil
+ })
+
+ mergedResults.compare(os.Stdout)
}
func main() {
var nf = flag.String("new", "", "the changed XML result to compare against")
var of = flag.String("old", "", "the baseline XML result to compare against")
+
+ var na = flag.String("newarchive", "", "the changed archived results to compare against")
+ var oa = flag.String("oldarchive", "", "the baseline archived results to compare against")
flag.Parse()
nxml := *nf
oxml := *of
- if len(nxml) == 0 {
- log.Fatalf("no new provided - nothing to compare")
- return
- }
+ narch := *na
+ oarch := *oa
+
+ hasNewFile := len(nxml) > 0
+ hasOldFile := len(oxml) > 0
- if len(oxml) == 0 {
- log.Fatalf("no old provided - nothing to compare against")
+ hasNewArch := len(narch) > 0
+ hasOldArch := len(oarch) > 0
+
+ if (!hasNewFile || !hasOldFile) && (!hasNewArch || !hasOldArch) {
+ log.Fatalf("You need to provide either -new & -old, or -newarchive & -oldarchive.")
return
}
- compareSingleTestRuns(oxml, nxml)
+ if hasNewFile && hasOldFile {
+ compareSingleTestRuns(oxml, nxml)
+ } else {
+ compareArchivedTestRuns(oarch, narch)
+ }
}