diff options
Diffstat (limited to 'tests/auto/testlib/selftests/generate_expected_output.py')
-rwxr-xr-x | tests/auto/testlib/selftests/generate_expected_output.py | 259 |
1 files changed, 177 insertions, 82 deletions
diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py index 66a75a304f..202c4cc426 100755 --- a/tests/auto/testlib/selftests/generate_expected_output.py +++ b/tests/auto/testlib/selftests/generate_expected_output.py @@ -27,89 +27,184 @@ ## ############################################################################# -#regenerate all test's output +# Regenerate all tests' output. +# +# Usage: cd to the build directory corresponding to this script's +# location; invoke this script; optionally pass the names of sub-dirs +# to limit which tests to regenerate expected_* files for. import os -import sys import subprocess -import re - -formats = ['xml', 'txt', 'xunitxml', 'lightxml', 'teamcity'] - -qtver = subprocess.check_output(['qmake', '-query', 'QT_VERSION']).strip().decode('utf-8') -rootPath = os.getcwd() - -isWindows = sys.platform == 'win32' - -replacements = [ - (qtver, r'@INSERT_QT_VERSION_HERE@'), - (r'Config: Using QtTest library.*', r'Config: Using QtTest library'), # Build string in text logs - (rootPath.encode('unicode-escape').decode('utf-8'), r''), - (r'( *)<Duration msecs="[\d\.]+"/>', r'\1<Duration msecs="0"/>'), - (r'( *)<QtBuild>[^<]+</QtBuild>', r'\1<QtBuild/>'), # Build element in xml, lightxml - (r'<property value="[^"]+" name="QtBuild"/>', r'<property value="" name="QtBuild"/>') # Build in xunitxml -] - -extraArgs = { - "commandlinedata": "fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2", - "benchlibcallgrind": "-callgrind", - "benchlibeventcounter": "-eventcounter", - "benchliboptions": "-eventcounter", - "benchlibtickcounter": "-tickcounter", - "badxml": "-eventcounter", - "benchlibcounting": "-eventcounter", - "printdatatags": "-datatags", - "printdatatagswithglobaltags": "-datatags", - "silent": "-silent", - "verbose1": "-v1", - "verbose2": "-v2", -} - -# Replace all occurrences of searchExp in one file -def replaceInFile(file): - import sys - import fileinput - for line in fileinput.input(file, inplace=1): - for searchExp, replaceExp in replacements: - line = re.sub(searchExp, replaceExp, line) - sys.stdout.write(line) - -def subdirs(): - result = [] - for path in os.listdir('.'): - if os.path.isdir('./' + path): - result.append(path) - return result - -def getTestForPath(path): - if isWindows: - testpath = path + '\\' + path + '.exe' - else: - testpath = path + '/' + path - return testpath - -def generateTestData(testname): - print(" running " + testname) + +class Fail (Exception): pass + +class Cleaner (object): + """Tool to clean up test output to make diff-ing runs useful. + + We care about whether tests pass or fail - if that changes, + something that matters has happened - and we care about some + changes to what they say when they do fail; but we don't care + exactly what line of what file the failing line of code now + occupies, nor do we care how many milliseconds each test took to + run; and changes to the Qt version number mean nothing to us. + + Create one singleton instance; it'll do mildly expensive things + once and you can use its .clean() method to tidy up your test + output.""" + + def __init__(self, here, command): + """Set up the details we need for later cleaning. + + Takes two parameters: here is $PWD and command is how this + script was invoked, from which we'll work out where it is; in + a shadow build, the former is the build tree's location + corresponding to this last. Checks $PWD does look as expected + in a build tree - raising Fail() if not - then invokes qmake + to discover Qt version (saved as .version for the benefit of + clients) and prepares the sequence of (regex, replace) pairs + that .clean() needs to do its job.""" + self.version, self.__replace = self.__getPatterns(here, command) + + import re + @staticmethod + def __getPatterns(here, command, + patterns = ( + # Timings: + (r'( *<Duration msecs=)"[\d\.]+"/>', r'\1"0"/>'), # xml, lightxml + (r'(Totals:.*,) *[0-9.]+ms', r'\1 0ms'), # txt + # Benchmarks: + (r'[0-9,.]+( (?:CPU ticks|msecs) per iteration \(total:) [0-9,.]+ ', r'0\1 0, '), # txt + (r'(<BenchmarkResult metric="(?:CPUTicks|WalltimeMilliseconds)".*\bvalue=)"[^"]+"', r'\1"0"'), # xml, lightxml + # Build details: + (r'(Config: Using QtTest library).*', r'\1'), # txt + (r'( *<QtBuild)>[^<]+</QtBuild>', r'\1/>'), # xml, lightxml + (r'(<property value=")[^"]+(" name="QtBuild"/>)', r'\1\2'), # xunitxml + # Line numbers in source files: + (r'(Loc: \[[^[\]()]+)\(\d+\)', r'\1(0)'), # txt + (r'(\[Loc: [^[\]()]+)\(\d+\)', r'\1(0)'), # teamcity + (r'(<Incident.*\bfile=.*\bline=)"\d+"', r'\1"0"'), # lightxml, xml + ), + precook = re.compile): + """Private implementation details of __init__().""" + + qmake = ('..',) * 4 + ('bin', 'qmake') + qmake = os.path.join(*qmake) + + if os.path.sep in command: + scriptPath = os.path.abspath(command) + elif os.path.exists(command): + # e.g. if you typed "python3 generate_expected_output.py" + scriptPath = os.path.join(here, command) + else: + # From py 3.2: could use os.get_exec_path() here. + for d in os.environ.get('PATH', '').split(os.pathsep): + scriptPath = os.path.join(d, command) + if os.path.isfile(scriptPath): + break + else: # didn't break + raise Fail('Unable to find', command, 'in $PATH') + + # Are we being run from the right place ? + myNames = scriptPath.split(os.path.sep) + if not (here.split(os.path.sep)[-5:] == myNames[-6:-1] + and os.path.isfile(qmake)): + raise Fail('Run', myNames[-1], 'in its directory of a completed build') + + try: + qtver = subprocess.check_output([qmake, '-query', 'QT_VERSION']) + except OSError as what: + raise Fail(what.strerror) + qtver = qtver.strip().decode('utf-8') + + scriptPath = os.path.dirname(scriptPath) # ditch leaf file-name + sentinel = os.path.sep + 'qtbase' + os.path.sep # '/qtbase/' + # Identify the path prefix of our qtbase ancestor directory + # (source, build and $PWD, when different); trim such prefixes + # off all paths we see. + roots = tuple(r[:r.find(sentinel) + 1].encode('unicode-escape').decode('utf-8') + for r in set((here, scriptPath, os.environ.get('PWD', ''))) + if sentinel in r) + patterns += tuple((root, r'') for root in roots) + ( + (r'\.'.join(qtver.split('.')), r'@INSERT_QT_VERSION_HERE@'),) + if any('-' in r for r in roots): + # Our xml formats replace hyphens with a character entity: + patterns += tuple((root.replace('-', '�*2D;'), r'') + for root in roots if '-' in root) + + return qtver, tuple((precook(p), r) for p, r in patterns) + del re + + def clean(self, data): + """Remove volatile details from test output. + + Takes the full test output as a single (possibly huge) + multi-line string; iterates over cleaned lines of output.""" + for line in data.split('\n'): + # Replace all occurrences of each regex: + for searchRe, replaceExp in self.__replace: + line = searchRe.sub(replaceExp, line) + yield line + +def generateTestData(testname, clean, + formats = ('xml', 'txt', 'xunitxml', 'lightxml', 'teamcity'), + extraArgs = { + "commandlinedata": "fiveTablePasses fiveTablePasses:fiveTablePasses_data1 -v2", + "benchlibcallgrind": "-callgrind", + "benchlibeventcounter": "-eventcounter", + "benchliboptions": "-eventcounter", + "benchlibtickcounter": "-tickcounter", + "badxml": "-eventcounter", + "benchlibcounting": "-eventcounter", + "printdatatags": "-datatags", + "printdatatagswithglobaltags": "-datatags", + "silent": "-silent", + "verbose1": "-v1", + "verbose2": "-v2", + }): + """Run one test and save its cleaned results. + + Required arguments are the name of the test directory (the binary + it contains is expected to have the same name) and a function + that'll clean a test-run's output; see Cleaner.clean(). + """ + # MS-Win: shall need to add .exe to this + path = os.path.join(testname, testname) + if not os.path.isfile(path): + print("Warning: directory", testname, "contains no test executable") + return + + print(" running", testname) for format in formats: - cmd = [getTestForPath(testname) + ' -' + format + ' ' + extraArgs.get(testname, '')] - result = 'expected_' + testname + '.' + format - data = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).communicate()[0] - out = open(result, 'wb') - out.write(data) - out.close() - replaceInFile(result) - -if isWindows: - print("This script does not work on Windows.") - exit() - -tests = sys.argv[1:] -os.environ['LC_ALL'] = 'C' -if len(tests) == 0: - tests = subdirs() -print("Generating " + str(len(tests)) + " test results for: " + qtver + " in: " + rootPath) -for path in tests: - if os.path.isfile(getTestForPath(path)): - generateTestData(path) - else: - print("Warning: directory " + path + " contains no test executable") + cmd = [path, '-' + format] + if testname in extraArgs: + cmd += extraArgs[testname].split() + + data = subprocess.Popen(cmd, stdout=subprocess.PIPE, + universal_newlines=True).communicate()[0] + with open('expected_' + testname + '.' + format, 'w') as out: + out.write('\n'.join(clean(data))) # write() appends a newline, too + +def main(name, *args): + """Minimal argument parsing and driver for the real work""" + os.environ['LC_ALL'] = 'C' + herePath = os.getcwd() + cleaner = Cleaner(herePath, name) + + tests = args if args else [d for d in os.listdir('.') if os.path.isdir(d)] + print("Generating", len(tests), "test results for", cleaner.version, "in:", herePath) + for path in tests: + generateTestData(path, cleaner.clean) + +if __name__ == '__main__': + # Executed when script is run, not when imported (e.g. to debug) + import sys + + if sys.platform.startswith('win'): + print("This script does not work on Windows.") + exit() + + try: + main(*sys.argv) + except Fail as what: + sys.stderr.write('Failed: ' + ' '.join(what.args) + '\n') + exit(1) |