1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
|
#!/usr/bin/env python
import sys
from xml.dom.minidom import parse
from dbaccess import setDatabase, execQuery, commit
from misc import isValidSHA1, getContext
# === Global functions ===============================================
def printUsage():
print (
"usage:" + sys.argv[0] +
" --help | <host> <platform> <branch> <SHA-1> <XML file> <database>")
def printVerboseUsagebbbbq():
printUsage()
print ("<host>: The physical machine on which the results were " +
"produced (e.g. barbarella or 172.24.90.79).")
print ("<platform>: The OS/compiler/architecture combination " +
"(e.g. linux-g++-32).")
print ("<branch>: The product branch (e.g. 'qt 4.6', 'qt 4.7', or " +
"'qt master').")
print ("<SHA-1>: The tested revision within the branch. Can be " +
"extracted using 'git log -1 --pretty=format:%H' (assuming the " +
"tested revision is the current head revision).")
print ("<XML file>: The results file (QTestLib output format).")
print (
"<database>: The database. One of 'bm' or 'bm-dev' (the latter " +
"intended for experimentation).")
# Returns True iff a low value indicates better performance than a high
# value for the given metric.
def lowerIsBetter(metric):
return {
"walltimemilliseconds": True,
"walltime": True,
"cputicks": True,
"instructionreads": True,
"events": True,
"bitspersecond": False,
"bytespersecond": False,
"framespersecond": False,
"fps": False
# add more if necessary ...
}[metric.lower()]
# Returns the canonical (i.e. "unaliased") form of the metric name.
def canonicalMetric(metric):
if (metric.lower() == "walltime"):
return "WalltimeMilliseconds"
if (metric.lower() == "framespersecond"):
return "fps"
return metric
# Returns True iff at least one of the given incidents indicates failure for
# the given data tag.
def matchesFailedIncident(dataTag, incidents):
for incident in incidents:
if (incident.getAttribute("type") == "fail"):
try:
dataTagElem = incident.getElementsByTagName("DataTag")[0]
except:
continue
if (dataTagElem.childNodes[0].data == dataTag):
return True
return False
# Returns results extracted from a file in QTestLib XML format
# (note: multiple top-level TestCase elements are allowed).
def extractResults(file):
def processBenchmarkResults(
results, dom, testCase, testFuncElem, testFunction, incidents):
# Loop over benchmark results ...
bmResultElems = testFuncElem.getElementsByTagName("BenchmarkResult")
for bmResultElem in bmResultElems:
# Data tag (note that "" is a valid data tag):
dataTag = bmResultElem.getAttribute("tag").strip()
# Metric:
metric = bmResultElem.getAttribute("metric").strip()
try:
lowerIsBetter_ = lowerIsBetter(metric)
except:
print(
"WARNING: skipping result for unsupported metric: >" +
metric + "<")
continue
metric = canonicalMetric(metric)
# Value:
value = float(bmResultElem.getAttribute("value"))
# Iterations (optional):
iterAttr = bmResultElem.getAttribute("iterations").strip()
if (iterAttr != ""):
try:
iterations = int(iterAttr)
assert iterations > 0
value = value / iterations
except:
raise BaseException(
"found 'iterations' attribute that is not a " +
"positive integer: " + iterAttr)
# Valid:
valid = not matchesFailedIncident(dataTag, incidents)
# Add item to array ...
results.append(
{'testCase': testCase,
'testFunction': testFunction,
'dataTag': dataTag,
'metric': metric,
'lowerIsBetter': lowerIsBetter_,
'value': value,
'valid': valid})
def processTestFunctions(results, dom, testCaseElem, testCase):
# Loop over test functions ...
testFuncElems = testCaseElem.getElementsByTagName("TestFunction")
for testFuncElem in testFuncElems:
testFunction = testFuncElem.getAttribute("name").strip()
assert testFunction != ""
incidents = testFuncElem.getElementsByTagName("Incident")
processBenchmarkResults(
results, dom, testCase, testFuncElem,
testFunction, incidents)
def processTestCases(results, dom):
# Loop over test cases ...
testCaseElems = dom.getElementsByTagName("TestCase")
for testCaseElem in testCaseElems:
testCase = testCaseElem.getAttribute("name").strip()
assert testCase != ""
processTestFunctions(
results, dom, testCaseElem, testCase)
# Load DOM structure from file:
try:
dom = parse(xmlFile)
except:
raise BaseException(sys.exc_info())
# Extract benchmark results from DOM structure:
results = []
processTestCases(results, dom)
dom.unlink()
return results
# ### 2 B DOCUMENTED!
def findOrInsertId(table, value, *args):
#print "value: >" + value + "<, ",
query_result = execQuery(
"SELECT id FROM " + table + " WHERE value = '" + str(value) + "';")
if len(query_result) == 1:
# Found, so return ID:
#print "returning existing ID: >" + str(query_result[0][0]) + "<"
return query_result[0][0]
# Not found, so insert:
query = "INSERT INTO " + table + " (value"
for i in range(0, len(args), 2):
query += ", " + str(args[i])
query += ") VALUES ('" + str(value) + "'"
for i in range(0, len(args), 2):
query += ", " + str(args[i + 1])
# ... and retrieve ID:
query += ") RETURNING id;"
query_result = execQuery(query)
assert len(query_result) == 1
#print "returning new ID: >" + str(query_result[0][0]) + "<"
return query_result[0][0]
# === Main program ===================================================
# --- Validate arguments -------------------------------------
if ((len(sys.argv) > 1) and (sys.argv[1] == "--help")):
printVerboseUsage()
sys.exit(1)
if (len(sys.argv) != 7):
printUsage()
sys.exit(1)
host = sys.argv[1]
platform = sys.argv[2]
branch = sys.argv[3]
sha1 = sys.argv[4]
if (not isValidSHA1(sha1)):
print "invalid SHA-1:", sha1
sys.exit(1)
xmlFile = sys.argv[5]
db = sys.argv[6]
# Perform a basic validation of the other arguments as well ... 2 B DONE!
try:
results = extractResults(xmlFile)
except BaseException as e:
print "failed to parse XML file:", e.args[0]
sys.exit(1)
#print results
# --- Upload to database -------------------------------------
setDatabase(db)
# Append a row to the 'upload' table (to record this upload event) ...
execQuery("INSERT INTO upload DEFAULT VALUES;", False)
# Retrieve the ID of the row we just inserted ...
uploadId = execQuery("SELECT currval('upload_id_seq');")[0][0]
hostId = findOrInsertId("host", host)
platformId = findOrInsertId("platform", platform)
branchId = findOrInsertId("branch", branch)
sha1Id = findOrInsertId("sha1", sha1)
contextId = getContext(hostId, platformId, branchId, sha1Id)
if contextId == -1:
contextId = execQuery("INSERT INTO context"
" (hostId, platformId, branchId, sha1Id)"
" VALUES (%d, %d, %d, %d)"
" RETURNING id;"
% (hostId, platformId, branchId, sha1Id))[0][0]
# Append rows to the 'result' table ...
for result in results:
benchmark = (
result['testCase'] + ":" + result['testFunction'] + "(" +
str(result['dataTag']) + ")")
benchmarkId = findOrInsertId("benchmark", benchmark)
metricId = findOrInsertId(
"metric", result['metric'], "lowerIsBetter",
result['lowerIsBetter'])
query = (
"INSERT INTO result"
" (contextId, benchmarkId, value, valid, metricId, uploadId)"
" VALUES (%d, %d, %f, %s, %d, %d);"
% (contextId, benchmarkId, result['value'], result['valid'],
metricId, uploadId))
execQuery(query, False)
# Make sure everything is written to the database:
commit()
print "uploading done"
|
