1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
|
#!/usr/bin/env python3
#############################################################################
##
## Copyright (C) 2015 The Qt Company Ltd.
## Contact: http://www.qt.io/licensing/
##
## This file is part of the release tools of the Qt Toolkit.
##
## $QT_BEGIN_LICENSE:LGPL21$
## Commercial License Usage
## Licensees holding valid commercial Qt licenses may use this file in
## accordance with the commercial license agreement provided with the
## Software or, alternatively, in accordance with the terms contained in
## a written agreement between you and The Qt Company. For licensing terms
## and conditions see http://www.qt.io/terms-conditions. For further
## information use the contact form at http://www.qt.io/contact-us.
##
## GNU Lesser General Public License Usage
## Alternatively, this file may be used under the terms of the GNU Lesser
## General Public License version 2.1 or version 3 as published by the Free
## Software Foundation and appearing in the file LICENSE.LGPLv21 and
## LICENSE.LGPLv3 included in the packaging of this file. Please review the
## following information to ensure the GNU Lesser General Public License
## requirements will be met: https://www.gnu.org/licenses/lgpl.html and
## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
##
## As a special exception, The Qt Company gives you certain additional
## rights. These rights are described in The Qt Company LGPL Exception
## version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
##
## $QT_END_LICENSE$
##
#############################################################################
# Regenerate all tests' output.
#
# Usage: cd to the build directory corresponding to this script's
# location; invoke this script; optionally pass the names of sub-dirs
# to limit which tests to regenerate expected_* files for.
import os
import subprocess
class Fail (Exception): pass
class Cleaner (object):
"""Tool to clean up test output to make diff-ing runs useful.
We care about whether tests pass or fail - if that changes,
something that matters has happened - and we care about some
changes to what they say when they do fail; but we don't care
exactly what line of what file the failing line of code now
occupies, nor do we care how many milliseconds each test took to
run; and changes to the Qt version number mean nothing to us.
Create one singleton instance; it'll do mildly expensive things
once and you can use its .clean() method to tidy up your test
output."""
def __init__(self, here, command):
"""Set up the details we need for later cleaning.
Takes two parameters: here is $PWD and command is how this
script was invoked, from which we'll work out where it is; in
a shadow build, the former is the build tree's location
corresponding to this last. Checks $PWD does look as expected
in a build tree - raising Fail() if not - then invokes qmake
to discover Qt version (saved as .version for the benefit of
clients) and prepares the sequence of (regex, replace) pairs
that .clean() needs to do its job."""
self.version, self.__replace = self.__getPatterns(here, command)
import re
@staticmethod
def __getPatterns(here, command,
patterns = (
# Timings:
(r'( *<Duration msecs=)"[\d\.]+"/>', r'\1"0"/>'), # xml, lightxml
(r'(Totals:.*,) *[0-9.]+ms', r'\1 0ms'), # txt
# Benchmarks:
(r'[0-9,.]+( (?:CPU ticks|msecs) per iteration \(total:) [0-9,.]+ ', r'0\1 0, '), # txt
(r'(,"(?:CPUTicks|WalltimeMilliseconds)"),\d+,\d+,', r'\1,0,0,'), # csv
(r'(<BenchmarkResult metric="(?:CPUTicks|WalltimeMilliseconds)".*\bvalue=)"[^"]+"', r'\1"0"'), # xml, lightxml
# Build details:
(r'(Config: Using QtTest library).*', r'\1'), # txt
(r'( *<QtBuild)>[^<]+</QtBuild>', r'\1/>'), # xml, lightxml
(r'(<property value=")[^"]+(" name="QtBuild"/>)', r'\1\2'), # xunitxml
# Line numbers in source files:
(r'(Loc: \[[^[\]()]+)\(\d+\)', r'\1(0)'), # txt
# (r'(\[Loc: [^[\]()]+)\(\d+\)', r'\1(0)'), # teamcity
(r'(<Incident.*\bfile=.*\bline=)"\d+"', r'\1"0"'), # lightxml, xml
),
precook = re.compile):
"""Private implementation details of __init__()."""
qmake = ('..',) * 4 + ('bin', 'qmake')
qmake = os.path.join(*qmake)
if os.path.sep in command:
scriptPath = os.path.abspath(command)
elif os.path.exists(command):
# e.g. if you typed "python3 generate_expected_output.py"
scriptPath = os.path.join(here, command)
else:
# From py 3.2: could use os.get_exec_path() here.
for d in os.environ.get('PATH', '').split(os.pathsep):
scriptPath = os.path.join(d, command)
if os.path.isfile(scriptPath):
break
else: # didn't break
raise Fail('Unable to find', command, 'in $PATH')
# Are we being run from the right place ?
myNames = scriptPath.split(os.path.sep)
if not (here.split(os.path.sep)[-5:] == myNames[-6:-1]
and os.path.isfile(qmake)):
raise Fail('Run', myNames[-1], 'in its directory of a completed build')
try:
qtver = subprocess.check_output([qmake, '-query', 'QT_VERSION'])
except OSError as what:
raise Fail(what.strerror)
qtver = qtver.strip().decode('utf-8')
scriptPath = os.path.dirname(scriptPath) # ditch leaf file-name
sentinel = os.path.sep + 'qtbase' + os.path.sep # '/qtbase/'
# Identify the path prefix of our qtbase ancestor directory
# (source, build and $PWD, when different); trim such prefixes
# off all paths we see.
roots = tuple(r[:r.find(sentinel) + 1].encode('unicode-escape').decode('utf-8')
for r in set((here, scriptPath, os.environ.get('PWD', '')))
if sentinel in r)
patterns += tuple((root, r'') for root in roots) + (
(r'\.'.join(qtver.split('.')), r'@INSERT_QT_VERSION_HERE@'),)
if any('-' in r for r in roots):
# Our xml formats replace hyphens with a character entity:
patterns += tuple((root.replace('-', '�*2D;'), r'')
for root in roots if '-' in root)
return qtver, tuple((precook(p), r) for p, r in patterns)
del re
def clean(self, data):
"""Remove volatile details from test output.
Takes the full test output as a single (possibly huge)
multi-line string; iterates over cleaned lines of output."""
for line in data.split('\n'):
# Replace all occurrences of each regex:
for searchRe, replaceExp in self.__replace:
line = searchRe.sub(replaceExp, line)
yield line
def generateTestData(testname, clean,
formats = ('xml', 'csv', 'txt', 'xunitxml', 'lightxml'), # +'teamcity' in 5.7
extraArgs = {
"commandlinedata": "fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2",
"benchlibcallgrind": "-callgrind",
"benchlibeventcounter": "-eventcounter",
"benchliboptions": "-eventcounter",
"benchlibtickcounter": "-tickcounter",
"badxml": "-eventcounter",
"benchlibcounting": "-eventcounter",
"printdatatags": "-datatags",
"printdatatagswithglobaltags": "-datatags",
"silent": "-silent",
"verbose1": "-v1",
"verbose2": "-v2",
}):
"""Run one test and save its cleaned results.
Required arguments are the name of the test directory (the binary
it contains is expected to have the same name) and a function
that'll clean a test-run's output; see Cleaner.clean().
"""
# MS-Win: shall need to add .exe to this
path = os.path.join(testname, testname)
if not os.path.isfile(path):
print("Warning: directory", testname, "contains no test executable")
return
print(" running", testname)
for format in formats:
cmd = [path, '-' + format]
if testname in extraArgs:
cmd += extraArgs[testname].split()
data = subprocess.Popen(cmd, stdout=subprocess.PIPE,
universal_newlines=True).communicate()[0]
with open('expected_' + testname + '.' + format, 'w') as out:
out.write('\n'.join(clean(data))) # write() appends a newline, too
def main(name, *args):
"""Minimal argument parsing and driver for the real work"""
os.environ['LC_ALL'] = 'C'
herePath = os.getcwd()
cleaner = Cleaner(herePath, name)
tests = args if args else [d for d in os.listdir('.') if os.path.isdir(d)]
print("Generating", len(tests), "test results for", cleaner.version, "in:", herePath)
for path in tests:
generateTestData(path, cleaner.clean)
if __name__ == '__main__':
# Executed when script is run, not when imported (e.g. to debug)
import sys
if sys.platform.startswith('win'):
print("This script does not work on Windows.")
exit()
try:
main(*sys.argv)
except Fail as what:
sys.stderr.write('Failed: ' + ' '.join(what.args) + '\n')
exit(1)
|
