diff options
Diffstat (limited to 'tests/auto/testlib/selftests/generate_expected_output.py')
-rwxr-xr-x | tests/auto/testlib/selftests/generate_expected_output.py | 262 |
1 files changed, 150 insertions, 112 deletions
diff --git a/tests/auto/testlib/selftests/generate_expected_output.py b/tests/auto/testlib/selftests/generate_expected_output.py index 581ff38006..350d20fa27 100755 --- a/tests/auto/testlib/selftests/generate_expected_output.py +++ b/tests/auto/testlib/selftests/generate_expected_output.py @@ -1,62 +1,47 @@ #!/usr/bin/env python3 -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the release tools of the Qt Toolkit. -## -## $QT_BEGIN_LICENSE:GPL-EXCEPT$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3 as published by the Free Software -## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -# Regenerate all tests' output. -# -# Usage: cd to the build directory corresponding to this script's -# location; invoke this script; optionally pass the names of sub-dirs -# to limit which tests to regenerate expected_* files for. -# -# The saved test output is used by ./tst_selftests.cpp, which compares -# it to the output of each test, ignoring various boring changes. -# This script canonicalises the parts that would exhibit those boring -# changes, so as to avoid noise in git (and conflicts in merges) for -# the saved copies of the output. +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 +from argparse import ArgumentParser, RawTextHelpFormatter import os import subprocess import re +import sys + + +USAGE = """ +Regenerate all tests' output. + +Usage: cd to the build directory containing the directories with +the subtest binaries, invoke this script; optionally pass the names of sub-dirs +and formats to limit which tests to regenerate expected_* files for. + +The saved test output is used by ./tst_selftests.cpp, which compares +it to the output of each test, ignoring various boring changes. +This script canonicalises the parts that would exhibit those boring +changes, so as to avoid noise in git (and conflicts in merges) for +the saved copies of the output. +""" + + +DEFAULT_FORMATS = ['xml', 'txt', 'junitxml', 'lightxml', 'teamcity', 'tap', 'csv'] TESTS = ['assert', 'badxml', 'benchlibcallgrind', 'benchlibcounting', 'benchlibeventcounter', 'benchliboptions', 'benchlibtickcounter', 'benchlibwalltime', 'blacklisted', 'cmptest', 'commandlinedata', 'counting', 'crashes', 'datatable', 'datetime', 'deleteLater', - 'deleteLater_noApp', 'differentexec', 'exceptionthrow', 'expectfail', - 'failcleanup', 'faildatatype', 'failfetchtype', 'failinit', - 'failinitdata', 'fetchbogus', 'findtestdata', 'float', 'globaldata', - 'longstring', 'maxwarnings', 'multiexec', 'pairdiagnostics', 'pass', + 'deleteLater_noApp', 'differentexec', 'eventloop', 'exceptionthrow', + 'expectfail', "extendedcompare", 'failcleanup', 'failcleanuptestcase', + 'faildatatype', 'failfetchtype', 'failinit', 'failinitdata', + 'fetchbogus', 'findtestdata', 'float', 'globaldata', 'longstring', + 'maxwarnings', 'mouse', 'multiexec', 'pairdiagnostics', 'pass', 'printdatatags', 'printdatatagswithglobaltags', 'qexecstringlist', - 'signaldumper', 'silent', 'singleskip', 'skip', 'skipcleanup', - 'skipinit', 'skipinitdata', 'sleep', 'strcmp', 'subtest', 'testlib', - 'tuplediagnostics', 'verbose1', 'verbose2', 'verifyexceptionthrown', - 'warnings', 'watchdog', 'xunit', 'keyboard'] + 'signaldumper', 'silent', 'silent_fatal', 'singleskip', 'skip', + 'skipblacklisted', 'skipcleanup', 'skipcleanuptestcase', 'skipinit', + 'skipinitdata', 'sleep', 'strcmp', 'subtest', 'testlib', 'tuplediagnostics', + 'verbose1', 'verbose2', 'verifyexceptionthrown', 'warnings', 'watchdog', + 'junit', 'keyboard'] class Fail (Exception): pass @@ -75,24 +60,31 @@ class Cleaner (object): once and you can use its .clean() method to tidy up your test output.""" - def __init__(self, here, command): + def __init__(self): """Set up the details we need for later cleaning. - Takes two parameters: here is os.getcwd() and command is how - this script was invoked, from which we'll work out where it - is; in a shadow build, the former is the build tree's location - corresponding to this last. Saves the directory of this + Saves the directory of this script as self.sourceDir, so client can find tst_selftests.cpp there. Checks here does look as expected in a build tree - - raising Fail() if not - then invokes qmake to discover Qt + raising Fail() if not - then retrieves the Qt version (saved as .version for the benefit of clients) and prepares the sequence of (regex, replace) pairs that .clean() needs to do its job.""" - self.version, self.sourceDir, self.__replace = self.__getPatterns(here, command) + self.version, self.sourceDir, self.__replace = self.__getPatterns() + + @staticmethod + def _read_qt_version(qtbase_dir): + cmake_conf_file = os.path.join(qtbase_dir, '.cmake.conf') + with open(cmake_conf_file) as f: + for line in f: + # set(QT_REPO_MODULE_VERSION "6.1.0") + if 'set(QT_REPO_MODULE_VERSION' in line: + return line.strip().split('"')[1] + + raise RuntimeError("Someone broke .cmake.conf formatting again") @staticmethod - def __getPatterns(here, command, - patterns = ( + def __getPatterns(patterns = ( # Timings: (r'( *<Duration msecs=)"[\d\.]+"/>', r'\1"0"/>'), # xml, lightxml (r'(Totals:.*,) *[0-9.]+ms', r'\1 0ms'), # txt @@ -105,6 +97,7 @@ class Cleaner (object): (r'(Config: Using QtTest library).*', r'\1'), # txt (r'( *<QtBuild)>[^<]+</QtBuild>', r'\1/>'), # xml, lightxml (r'(<property name="QtBuild" value=")[^"]+"', r'\1"'), # junitxml + (r'(<testsuite .*? hostname=")[^"]+(".*>)', r'\1@HOSTNAME@\2'), # junit # Line numbers in source files: (r'(ASSERT: ("|").*("|") in file .*, line) \d+', r'\1 0'), # lightxml (r'(Loc: \[[^[\]()]+)\(\d+\)', r'\1(0)'), # txt @@ -127,38 +120,13 @@ class Cleaner (object): precook = re.compile): """Private implementation details of __init__().""" - qmake = ('..',) * 4 + ('bin', 'qmake') - qmake = os.path.join(*qmake) - - if os.path.sep in command: - scriptPath = os.path.abspath(command) - elif os.path.exists(command): - # e.g. if you typed "python3 generate_expected_output.py" - scriptPath = os.path.join(here, command) - else: - # From py 3.2: could use os.get_exec_path() here. - for d in os.environ.get('PATH', '').split(os.pathsep): - scriptPath = os.path.join(d, command) - if os.path.isfile(scriptPath): - break - else: # didn't break - raise Fail('Unable to find', command, 'in $PATH') - # Are we being run from the right place ? - scriptPath, myName = os.path.split(scriptPath) + scriptPath = os.path.dirname(os.path.abspath(__file__)) hereNames, depth = scriptPath.split(os.path.sep), 5 hereNames = hereNames[-depth:] # path components from qtbase down assert hereNames[0] == 'qtbase', ('Script moved: please correct depth', hereNames) - if not (here.split(os.path.sep)[-depth:] == hereNames - and os.path.isfile(qmake)): - raise Fail('Run', myName, 'in its directory of a completed build') - - try: - qtver = subprocess.check_output([qmake, '-query', 'QT_VERSION']) - except OSError as what: - raise Fail(what.strerror) - qtver = qtver.strip().decode('utf-8') - + qtbase_dir = os.path.realpath(os.path.join(scriptPath, '..', '..', '..', '..')) + qtver = Cleaner._read_qt_version(qtbase_dir) hereNames = tuple(hereNames) # Add path to specific sources and to tst_*.cpp if missing (for in-source builds): patterns += ((r'(^|[^/])\b(qtestcase.cpp)\b', r'\1qtbase/src/testlib/\2'), @@ -174,7 +142,7 @@ class Cleaner (object): # (source, build and $PWD, when different); trim such prefixes # off all paths we see. roots = tuple(r[:r.find(sentinel) + 1].encode('unicode-escape').decode('utf-8') - for r in set((here, scriptPath, os.environ.get('PWD', ''))) + for r in set((os.getcwd(), scriptPath, os.environ.get('PWD', ''))) if sentinel in r) patterns += tuple((root, r'') for root in roots) + ( (r'\.'.join(qtver.split('.')), r'@INSERT_QT_VERSION_HERE@'),) @@ -207,28 +175,38 @@ class Scanner (object): def __init__(self): pass - def subdirs(self, given): + def subdirs(self, given, skip_callgrind=False): if given: for d in given: if not os.path.isdir(d): print('No such directory:', d, '- skipped') + elif skip_callgrind and d == 'benchlibcallgrind': + pass # Skip this test, as requeted. elif d in TESTS: yield d else: - print('Directory', d, 'is not in the list of tests') + print(f'Directory {d} is not in the list of tests') else: - for d in TESTS: + tests = TESTS + if skip_callgrind: + tests.remove('benchlibcallgrind') + missing = 0 + for d in tests: if os.path.isdir(d): yield d else: - print('directory ', d, " doesn't exist, was it removed?") + missing += 1 + print(f"directory {d} doesn't exist, was it removed?") + if missing == len(tests): + print(USAGE) + del re -# Keep in sync with tst_selftests.cpp's processEnvironment(): +# Keep in sync with tst_selftests.cpp's testEnvironment(): def baseEnv(platname=None, - keep=('PATH', 'QT_QPA_PLATFORM'), + keep=('PATH', 'QT_QPA_PLATFORM', 'QTEST_THROW_ON_FAIL', 'QTEST_THROW_ON_SKIP', 'ASAN_OPTIONS'), posix=('HOME', 'USER', 'QEMU_SET_ENV', 'QEMU_LD_PREFIX'), - nonapple=('DISPLAY', 'XAUTHLOCALHOSTNAME'), # and XDG_* + nonapple=('DISPLAY', 'XAUTHORITY', 'XAUTHLOCALHOSTNAME'), # and XDG_* # Don't actually know how to test for QNX, so this is ignored: qnx=('GRAPHICS_ROOT', 'TZ'), # Probably not actually relevant @@ -275,9 +253,9 @@ def testEnv(testname, "watchdog": { "QTEST_FUNCTION_TIMEOUT": "100" }, }, # Must match tst_Selftests::runSubTest_data(): - crashers = ("assert", "blacklisted", "crashes", "crashedterminate", + crashers = ("assert", "crashes", "crashedterminate", "exceptionthrow", "faildatatype", "failfetchtype", - "fetchbogus", "silent", "watchdog")): + "fetchbogus", "silent_fatal", "watchdog")): """Determine the environment in which to run a test.""" data = baseEnv() if testname in crashers: @@ -286,16 +264,54 @@ def testEnv(testname, data.update(extraEnv[testname]) return data -def generateTestData(testname, clean, - formats = ('xml', 'txt', 'junitxml', 'lightxml', 'teamcity', 'tap')): +def shouldIgnoreTest(testname, format): + """Test whether to exclude a test/format combination. + + See TestLogger::shouldIgnoreTest() in tst_selftests.cpp; it starts + with various exclusions for opt-in tests, platform dependencies + and tool availability; we ignore those, as we need the test data + to be present when those exclusions aren't in effect. + + In the remainder, exclude what it always excludes. + """ + if format != 'txt': + if testname in ("differentexec", + "multiexec", + "qexecstringlist", + "benchliboptions", + "printdatatags", + "printdatatagswithglobaltags", + "silent", + "silent_fatal", + "crashes", + "benchlibcallgrind", + "float", + "sleep"): + return True + + if testname == "badxml" and not format.endswith('xml'): + return True + + # Skip benchlib* for teamcity, and everything else for csv: + if format == ('teamcity' if testname.startswith('benchlib') else 'csv'): + return True + + if testname == "junit" and format != "junitxml": + return True + + return False + +def generateTestData(test_path, expected_path, clean, formats): """Run one test and save its cleaned results. - Required arguments are the name of the test directory (the binary - it contains is expected to have the same name) and a function - that'll clean a test-run's output; see Cleaner.clean(). + Required arguments are the path to test directory (the binary + it contains is expected to have the same name), a function + that'll clean a test-run's output; see Cleaner.clean() and a list of + formats. """ # MS-Win: shall need to add .exe to this - path = os.path.join(testname, testname) + testname = os.path.basename(test_path) + path = os.path.join(test_path, testname) if not os.path.isfile(path): print("Warning: directory", testname, "contains no test executable") return @@ -303,27 +319,49 @@ def generateTestData(testname, clean, # Prepare environment in which to run tests: env = testEnv(testname) - print(" running", testname) for format in formats: - cmd = [path, '-' + format] + if shouldIgnoreTest(testname, format): + continue + print(f' running {testname}/{format}') + cmd = [path, f'-{format}'] + expected_file = f'expected_{testname}.{format}' data = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env, universal_newlines=True).communicate()[0] - with open('expected_' + testname + '.' + format, 'w') as out: + with open(os.path.join(expected_path, expected_file), 'w') as out: out.write('\n'.join(clean(data))) # write() appends a newline, too -def main(name, *args): - """Minimal argument parsing and driver for the real work""" - herePath = os.getcwd() - cleaner = Cleaner(herePath, name) +def main(argv): + """Argument parsing and driver for the real work""" + argument_parser = ArgumentParser(description=USAGE, formatter_class=RawTextHelpFormatter) + argument_parser.add_argument('--formats', '-f', + help='Comma-separated list of formats') + argument_parser.add_argument('--skip-callgrind', '-s', action='store_true', + help='Skip the (no longer expensive) benchlib callgrind test') + argument_parser.add_argument('subtests', help='subtests to regenerate', + nargs='*', type=str) + + options = argument_parser.parse_args(argv[1:]) + formats = options.formats.split(',') if options.formats else DEFAULT_FORMATS - tests = tuple(Scanner().subdirs(args)) - print("Generating", len(tests), "test results for", cleaner.version, "in:", herePath) + cleaner = Cleaner() + src_dir = cleaner.sourceDir + + if not options.skip_callgrind: + # Skip it, even if not requested, when valgrind isn't available: + try: + probe = subprocess.Popen(['valgrind', '--version'], stdout=subprocess.PIPE, + env=testEnv('benchlibcallgrind'), universal_newlines=True) + except FileNotFoundError: + options.skip_callgrind = True + print("Failed to find valgrind, skipping benchlibcallgrind test") + + tests = tuple(Scanner().subdirs(options.subtests, options.skip_callgrind)) + print("Generating", len(tests), "test results for", cleaner.version, "in:", src_dir) for path in tests: - generateTestData(path, cleaner.clean) + generateTestData(path, src_dir, cleaner.clean, formats) if __name__ == '__main__': # Executed when script is run, not when imported (e.g. to debug) - import sys baseEnv(sys.platform) # initializes its cache if sys.platform.startswith('win'): @@ -331,7 +369,7 @@ if __name__ == '__main__': exit() try: - main(*sys.argv) + main(sys.argv) except Fail as what: sys.stderr.write('Failed: ' + ' '.join(what.args) + '\n') exit(1) |