diff options
| -rw-r--r-- | build_history/blacklist.txt | 3 | ||||
| -rw-r--r-- | sources/pyside2/tests/registry/existence_test.py | 6 | ||||
| -rw-r--r-- | sources/pyside2/tests/registry/init_platform.py | 6 | ||||
| -rw-r--r-- | sync.profile | 19 | ||||
| -rw-r--r-- | testing/__init__.py | 61 | ||||
| -rw-r--r-- | testing/blacklist.py | 125 | ||||
| -rw-r--r-- | testing/buildlog.py | 121 | ||||
| -rw-r--r-- | testing/command.py | 332 | ||||
| -rw-r--r-- | testing/helper.py | 73 | ||||
| -rw-r--r-- | testing/parser.py | 150 | ||||
| -rw-r--r-- | testing/runner.py | 219 | ||||
| -rw-r--r-- | testrunner.py | 804 |
12 files changed, 1092 insertions, 827 deletions
diff --git a/build_history/blacklist.txt b/build_history/blacklist.txt index 61297600c..468a10c32 100644 --- a/build_history/blacklist.txt +++ b/build_history/blacklist.txt @@ -78,6 +78,3 @@ win32 linux darwin -# Disable signature existence test for now on dev -[registry::existence_test] - qt5.11 diff --git a/sources/pyside2/tests/registry/existence_test.py b/sources/pyside2/tests/registry/existence_test.py index 8f3c568a3..3c5b7e545 100644 --- a/sources/pyside2/tests/registry/existence_test.py +++ b/sources/pyside2/tests/registry/existence_test.py @@ -131,16 +131,16 @@ if not have_refmodule and is_ci and qtVersion()[:2] in tested_versions: """ generate_all() sys.stderr.flush() - print("BEGIN_FILE", refPath, file=sys.stderr) + print("BEGIN_FILE", shortpath, file=sys.stderr) with open(refPath) as f: print(f.read(), file=sys.stderr) - print("END_FILE", refPath, file=sys.stderr) + print("END_FILE", shortpath, file=sys.stderr) sys.stderr.flush() raise RuntimeError(dedent(""" {line} ** This is the initial call. You should check this file in: ** {} - **""").format(refPath, line=79 * "*")) + **""").format(shortpath, line=79 * "*")) if __name__ == '__main__': unittest.main() diff --git a/sources/pyside2/tests/registry/init_platform.py b/sources/pyside2/tests/registry/init_platform.py index ffe6e22fc..a8e692d98 100644 --- a/sources/pyside2/tests/registry/init_platform.py +++ b/sources/pyside2/tests/registry/init_platform.py @@ -229,7 +229,9 @@ def enum_all(): return ret def generate_all(): - with open(refPath(), "w") as outfile, open(sourcepath) as f: + refPath = getRefPath() + module = os.path.basename(os.path.splitext(refPath)[0]) + with open(refPath, "w") as outfile, open(sourcepath) as f: fmt = Formatter(outfile) enu = SimplifyingEnumerator(fmt) lines = f.readlines() @@ -254,7 +256,7 @@ def generate_all(): def __main__(): print("+++ generating {}. You should probably check this file in." - .format(refpath)) + .format(getRefPath())) generate_all() if __name__ == "__main__": diff --git a/sync.profile b/sync.profile deleted file mode 100644 index 9f28f73da..000000000 --- a/sync.profile +++ /dev/null @@ -1,19 +0,0 @@ -# Module dependencies. -# Every module that is required to build this module should have one entry. -# Each of the module version specifiers can take one of the following values: -# - A specific Git revision. -# - any git symbolic ref resolvable from the module's repository (e.g. "refs/heads/master" to track master branch) -# - an empty string to use the same branch under test (dependencies will become "refs/heads/master" if we are in the master branch) -# -%dependencies = ( - "qttools" => "refs/heads/5.6", - "qtx11extras" => "refs/heads/5.6", - "qtsvg" => "refs/heads/5.6", - "qtmultimedia" => "refs/heads/5.6", - "qtdeclarative" => "refs/heads/5.6", - "qtxmlpatterns" => "refs/heads/5.6", - "qtbase" => "refs/heads/5.6", - "qtwebview" => "refs/heads/5.6", - "qtwebsockets" => "refs/heads/5.6", -); - diff --git a/testing/__init__.py b/testing/__init__.py new file mode 100644 index 000000000..a540251cc --- /dev/null +++ b/testing/__init__.py @@ -0,0 +1,61 @@ +############################################################################# +## +## Copyright (C) 2017 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from __future__ import print_function + +import sys +from . import command + +main = command.main + +# modify print so that it always flushes +__builtins__["orig_print"] = __builtins__["print"] + +def print_flushed(*args, **kw): + orig_print(*args, **kw) + sys.stdout.flush() + +__builtins__["print"] = print_flushed + +print = print_flushed + +# We also could use "python -u" to get unbuffered output. +# This method is better since it needs no change of the interface. + +# eof diff --git a/testing/blacklist.py b/testing/blacklist.py new file mode 100644 index 000000000..b11b7db6f --- /dev/null +++ b/testing/blacklist.py @@ -0,0 +1,125 @@ +############################################################################# +## +## Copyright (C) 2017 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from __future__ import print_function + +from .helper import decorate, StringIO +from .buildlog import builds + + +class BlackList(object): + def __init__(self, blname): + if blname == None: + f = StringIO() + self.raw_data = [] + else: + with open(blname) as f: + self.raw_data = f.readlines() + # keep all lines, but see what is not relevant + lines = self.raw_data[:] + + def filtered_line(line): + if '#' in line: + line = line[0:line.index('#')] + return line.split() + + # now put every bracketed line in a test + # and use subsequent identifiers for a match + def is_test(fline): + return fline and fline[0].startswith("[") + + self.tests = {} + + if not lines: + # nothing supplied + return + + self.index = {} + for idx, line in enumerate(lines): + fline = filtered_line(line) + if not fline: + continue + if is_test(fline): + break + # we have a global section + name = '' + self.tests[name] = [] + for idx, line in enumerate(lines): + fline = filtered_line(line) + if is_test(fline): + # a new name + name = decorate(fline[0][1:-1]) + self.tests[name] = [] + self.index[name] = idx + elif fline: + # a known name with a new entry + self.tests[name].append(fline) + + def find_matching_line(self, test): + """ + Take a test result. + Find a line in the according blacklist file where all keys of the line are found. + If line not found, do nothing. + if line found and test passed, it is a BPASS. + If line found and test failed, it is a BFAIL. + """ + passed = test.passed + classifiers = set(builds.classifiers) + + if "" in self.tests: + # this is a global section + for line in self.tests[""]: + keys = set(line) + if keys <= classifiers: + # found a match! + return line + mod_name = test.mod_name + if mod_name not in self.tests and decorate(mod_name) not in self.tests: + return None + if mod_name in self.tests: + thing = mod_name + else: + thing = decorate(mod_name) + for line in self.tests[thing]: + keys = set(line) + if keys <= classifiers: + # found a match! + return line + else: + return None # nothing found diff --git a/testing/buildlog.py b/testing/buildlog.py new file mode 100644 index 000000000..a040d5d34 --- /dev/null +++ b/testing/buildlog.py @@ -0,0 +1,121 @@ +############################################################################# +## +## Copyright (C) 2017 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from __future__ import print_function + +import os +import sys +from collections import namedtuple + +from .helper import script_dir + +LogEntry = namedtuple("LogEntry", ["log_dir", "build_dir"]) + + +class BuildLog(object): + """ + This class is a convenience wrapper around a list of log entries. + + The list of entries is sorted by date and checked for consistency. + For simplicity and readability, the log entries are named tuples. + + """ + def __init__(self): + history_dir = os.path.join(script_dir, 'build_history') + build_history = [] + for timestamp in os.listdir(history_dir): + log_dir = os.path.join(history_dir, timestamp) + if not os.path.isdir(log_dir): + continue + fpath = os.path.join(log_dir, 'build_dir.txt') + if not os.path.exists(fpath): + print("Warning: %s not found, skipped" % fpath) + continue + with open(fpath) as f: + build_dir = f.read().strip() + if not os.path.exists(build_dir): + rel_dir, low_part = os.path.split(build_dir) + rel_dir, two_part = os.path.split(rel_dir) + if two_part.startswith("pyside") and two_part.endswith("build"): + build_dir = os.path.abspath(os.path.join(two_part, low_part)) + if os.path.exists(build_dir): + print("Note: build_dir was probably moved.") + else: + print("Warning: missing build dir %s" % build_dir) + continue + entry = LogEntry(log_dir, build_dir) + build_history.append(entry) + # we take the latest build for now. + build_history.sort() + self.history = build_history + self._buildno = None + + def set_buildno(self, buildno): + self.history[buildno] # test + self._buildno = buildno + + @property + def selected(self): + if self._buildno is None: + return None + if self.history is None: + return None + return self.history[self._buildno] + + @property + def classifiers(self): + if not self.selected: + raise ValueError('+++ No build with the configuration found!') + # Python2 legacy: Correct 'linux2' to 'linux', recommended way. + platform = 'linux' if sys.platform.startswith('linux') else sys.platform + res = [platform] + # the rest must be guessed from the given filename + path = self.selected.build_dir + base = os.path.basename(path) + res.extend(base.split('-')) + # add all the python and qt subkeys + for entry in res: + parts = entry.split(".") + for idx in range(len(parts)): + key = ".".join(parts[:idx]) + if key not in res: + res.append(key) + return res + +builds = BuildLog() diff --git a/testing/command.py b/testing/command.py new file mode 100644 index 000000000..67dd05a03 --- /dev/null +++ b/testing/command.py @@ -0,0 +1,332 @@ +############################################################################# +## +## Copyright (C) 2017 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from __future__ import print_function + +""" +testrunner +========== + +Provide an interface to the pyside tests. +----------------------------------------- + +This program can only be run if PySide was build with tests enabled. +All tests are run in a single pass, and if not blacklisted, an error +is raised at the end of the run. + +Recommended build process: +There is no need to install the project. +Building the project with something like + + python setup.py build --build-tests --qmake=<qmakepath> --ignore-git --debug + +is sufficient. The tests are run by changing into the latest build dir and there +into pyside2, then 'make test'. + + +New testing policy: +------------------- + +The tests are now run 5 times, and errors are reported +when they appear at least 3 times. With the variable COIN_RERUN_FAILED_ONLY it is +possible to configure if all tests should be rerun or the failed ones, only. + +The full mode can be tested locally by setting + + export COIN_RERUN_FAILED_ONLY=0 +""" + +import os +import sys +import argparse +from textwrap import dedent +from collections import OrderedDict +from timeit import default_timer as timer + +from .helper import script_dir, decorate +from .buildlog import builds +from .blacklist import BlackList +from .runner import TestRunner +from .parser import TestParser + +# Should we repeat only failed tests? +COIN_RERUN_FAILED_ONLY = True +COIN_THRESHOLD = 3 # report error if >= +COIN_TESTING = 5 # number of runs + +if (os.environ.get("COIN_RERUN_FAILED_ONLY", "1").lower() in + "0 f false n no".split()): + COIN_RERUN_FAILED_ONLY = False + +def test_project(project, args, blacklist, runs): + ret = [] + for idx in range(runs): + index = idx + 1 + runner = TestRunner(builds.selected, project, index) + print() + print("********* Start testing of %s *********" % project) + print("Config: Using", " ".join(builds.classifiers)) + print() + if os.path.exists(runner.logfile) and args.skip: + print("Parsing existing log file:", runner.logfile) + else: + if index > 1 and COIN_RERUN_FAILED_ONLY: + rerun = rerun_list + if not rerun: + print("--- no re-runs found, stopping before test {} ---" + .format(index)) + break + else: + rerun = None + runner.run("RUN {}:".format(idx + 1), rerun, 10 * 60) + result = TestParser(runner.logfile) + r = 5 * [0] + rerun_list = [] + print() + for test, res in result.iter_blacklist(blacklist): + print("RES {}:".format(index), end=" ") + print("%-6s" % res, decorate(test) + "()") + r[0] += 1 if res == "PASS" else 0 + r[1] += 1 if res == "FAIL!" else 0 + r[2] += 1 if res == "SKIPPED" else 0 # not yet supported + r[3] += 1 if res == "BFAIL" else 0 + r[4] += 1 if res == "BPASS" else 0 + if res not in ("PASS", "BPASS"): + rerun_list.append(test) + print() + print("Totals:", sum(r), "tests.", + "{} passed, {} failed, {} skipped, {} blacklisted, {} bpassed." + .format(*r)) + print() + print("********* Finished testing of %s *********" % project) + print() + ret.append(r) + + return ret + +def main(): + # create the top-level command parser + start_time = timer() + all_projects = "shiboken2 pyside2 pyside2-tools".split() + tested_projects = "shiboken2 pyside2".split() + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=dedent("""\ + Run the tests for some projects, default = '{}'. + + Testing is now repeated up to {rep} times, and errors are + only reported if they occur {thr} or more times. + The environment variable COIN_RERUN_FAILED_ONLY controls if errors + are only repeated if there are errors. The default is "1". + """.format("' '".join(tested_projects), thr=COIN_THRESHOLD, rep=COIN_TESTING))) + subparsers = parser.add_subparsers(dest="subparser_name") + + # create the parser for the "test" command + parser_test = subparsers.add_parser("test") + group = parser_test.add_mutually_exclusive_group(required=False) + blacklist_default = os.path.join(script_dir, 'build_history', 'blacklist.txt') + group.add_argument("--blacklist", "-b", type=argparse.FileType('r'), + default=blacklist_default, + help='a Qt blacklist file (default: {})'.format(blacklist_default)) + parser_test.add_argument("--skip", action='store_true', + help="skip the tests if they were run before") + parser_test.add_argument("--environ", nargs='+', + help="use name=value ... to set environment variables") + parser_test.add_argument("--buildno", default=-1, type=int, + help="use build number n (0-based), latest = -1 (default)") + parser_test.add_argument("--projects", nargs='+', type=str, + default=tested_projects, + choices=all_projects, + help="use '{}'' (default) or other projects" + .format("' '".join(tested_projects))) + parser_getcwd = subparsers.add_parser("getcwd") + parser_getcwd.add_argument("filename", type=argparse.FileType('w'), + help="write the build dir name into a file") + parser_getcwd.add_argument("--buildno", default=-1, type=int, + help="use build number n (0-based), latest = -1 (default)") + parser_list = subparsers.add_parser("list") + args = parser.parse_args() + + if hasattr(args, "buildno"): + try: + builds.set_buildno(args.buildno) + except IndexError: + print("history out of range. Try '%s list'" % __file__) + sys.exit(1) + + if args.subparser_name == "getcwd": + print(builds.selected.build_dir, file=args.filename) + print(builds.selected.build_dir, "written to file", args.filename.name) + sys.exit(0) + elif args.subparser_name == "test": + pass # we do it afterwards + elif args.subparser_name == "list": + rp = os.path.relpath + print() + print("History") + print("-------") + for idx, build in enumerate(builds.history): + print(idx, rp(build.log_dir), rp(build.build_dir)) + print() + print("Note: only the last history entry of a folder is valid!") + sys.exit(0) + else: + parser.print_help() + sys.exit(1) + + if args.blacklist: + args.blacklist.close() + bl = BlackList(args.blacklist.name) + else: + bl = BlackList(None) + if args.environ: + for line in args.environ: + things = line.split("=") + if len(things) != 2: + raise ValueError("you need to pass one or more name=value pairs.") + key, value = things + os.environ[key] = value + + print(dedent("""\ + System: + Platform={platform} + Executable={executable} + Version={version_lf} + API version={api_version} + + Environment:""") + .format(version_lf=sys.version.replace("\n", " "), **sys.__dict__)) + for key, value in sorted(os.environ.items()): + print(" {}={}".format(key, value)) + print() + + q = 5 * [0] + + runs = COIN_TESTING + fail_crit = COIN_THRESHOLD + # now loop over the projects and accumulate + for project in args.projects: + res = test_project(project, args, bl, runs) + for idx, r in enumerate(res): + q = list(map(lambda x, y: x+y, r, q)) + + if len(args.projects) > 1: + print("All above projects:", sum(q), "tests.", + "{} passed, {} failed, {} skipped, {} blacklisted, {} bpassed." + .format(*q)) + print() + + tot_res = OrderedDict() + for project in args.projects: + for idx in range(runs): + index = idx + 1 + runner = TestRunner(builds.selected, project, index) + result = TestParser(runner.logfile) + for test, res in result.iter_blacklist(bl): + key = project + ":" + test + tot_res.setdefault(key, []) + tot_res[key].append(res) + tot_flaky = 0 + print("*" * 79) + print("**") + print("* Summary Of All Tests") + print("*") + empty = True + for test, res in tot_res.items(): + pass__c = res.count("PASS") + bpass_c = res.count("BPASS") + fail__c = res.count("FAIL!") + bfail_c = res.count("BFAIL") + fail2_c = fail__c + bfail_c + if pass__c == len(res): + continue + elif bpass_c == runs and runs > 1: + msg = "Remove blacklisting; test passes" + elif fail__c == runs: + msg = "Newly detected Real test failure!" + elif bfail_c == runs: + msg = "Keep blacklisting ;-(" + elif fail2_c > 0 and fail2_c < len(res): + msg = "Flaky test" + tot_flaky += 1 + else: + continue + empty = False + padding = 6 * runs + txt = " ".join(("{:<{width}}".format(piece, width=5) for piece in res)) + txt = (txt + padding * " ")[:padding] + testpad = 36 + if len(test) < testpad: + test += (testpad - len(test)) * " " + print(txt, decorate(test), msg) + if empty: + print("* (empty)") + print("*") + print("**") + print("*" * 79) + print() + if runs > 1: + print("Total flaky tests: errors but not always = {}".format(tot_flaky)) + print() + else: + print("For info about flaky tests, we need to perform more than one run.") + print("Please activate the COIN mode: 'export QTEST_ENVIRONMENT=ci'") + print() + # nag us about unsupported projects + ap, tp = set(all_projects), set(tested_projects) + if ap != tp: + print("+++++ Note: please support", " ".join(ap-tp), "+++++") + print() + + stop_time = timer() + used_time = stop_time - start_time + # Now create an error if the criterion is met: + try: + err_crit = "'FAIL! >= {}'".format(fail_crit) + for res in tot_res.values(): + if res.count("FAIL!") >= fail_crit: + raise ValueError("At least one failure was not blacklisted " + "and met the criterion {}" + .format(err_crit)) + print("No test met the error criterion {}".format(err_crit)) + finally: + print() + print("Total time of whole Python script = {:0.2f} sec".format(used_time)) + print() +# eof diff --git a/testing/helper.py b/testing/helper.py new file mode 100644 index 000000000..8ea3e6a6b --- /dev/null +++ b/testing/helper.py @@ -0,0 +1,73 @@ +############################################################################# +## +## Copyright (C) 2017 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from __future__ import print_function + +import os +import sys +from collections import namedtuple + +PY3 = sys.version_info[0] == 3 # from the six module +from subprocess import PIPE +if PY3: + from subprocess import TimeoutExpired + from io import StringIO +else: + class SubprocessError(Exception): pass + # this is a fake, just to keep the source compatible. + # timeout support is in python 3.3 and above. + class TimeoutExpired(SubprocessError): pass + from StringIO import StringIO + + +script_dir = os.path.dirname(os.path.dirname(__file__)) + +def decorate(mod_name): + """ + Write the combination of "modulename_funcname" + in the Qt-like form "modulename::funcname" + """ + if "_" not in mod_name: + return mod_name + if "::" in mod_name: + return mod_name + name, rest = mod_name.split("_", 1) + return name + "::" + rest + +#eof diff --git a/testing/parser.py b/testing/parser.py new file mode 100644 index 000000000..d54abb1fa --- /dev/null +++ b/testing/parser.py @@ -0,0 +1,150 @@ +############################################################################# +## +## Copyright (C) 2017 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from __future__ import print_function + +import os +import re +from collections import namedtuple +from .helper import StringIO + +_EXAMPLE = """ +Example output: + +ip1 n sharp mod_name code tim +----------------------------------------------------------------------------------------- +114/391 Test #114: QtCore_qfileinfo_test-42 ........................ Passed 0.10 sec + Start 115: QtCore_qfile_test +115/391 Test #115: QtCore_qfile_test ...............................***Failed 0.11 sec + Start 116: QtCore_qflags_test + +We will only look for the dotted lines and calculate everything from that. +The summary statistics at the end will be ignored. That allows us to test +this functionality with short timeout values. + +Note the field "mod_name". I had split this before, but it is necessary +to use the combination as the key, because the test names are not unique. +""" + +# validation of our pattern: + +_TEST_PAT = r""" + ^ # start + \s* # any whitespace ==: WS + ([0-9]+)/([0-9]+) # ip1 "/" n + \s+ # some WS + Test # "Test" + \s+ # some WS + \# # sharp symbol "#" + ([0-9]+) # sharp + : # colon symbol ':' + \s+ # some WS + ([\w-]+) # mod_name + .*? # whatever (non greedy) + ( # + (Passed) # either "Passed", None + | # + \*\*\*(\w+.*?) # or None, "Something" + ) # code + \s+ # some WS + ([0-9]+\.[0-9]+) # tim + \s+ # some WS + sec # "sec" + \s* # any WS + $ # end + """ +assert re.match(_TEST_PAT, _EXAMPLE.splitlines()[5], re.VERBOSE) +assert len(re.match(_TEST_PAT, _EXAMPLE.splitlines()[5], re.VERBOSE).groups()) == 8 +assert len(re.match(_TEST_PAT, _EXAMPLE.splitlines()[7], re.VERBOSE).groups()) == 8 + +TestResult = namedtuple("TestResult", ["idx", "mod_name", "passed", + "code", "time"]) +def _parse_tests(test_log): + """ + Create a TestResult object for every entry. + """ + result = [] + if isinstance(test_log, StringIO): + lines = test_log.readlines() + elif test_log is not None and os.path.exists(test_log): + with open(test_log) as f: + lines = f.readlines() + else: + lines = [] + pat = _TEST_PAT + for line in lines: + match = re.match(pat, line, re.VERBOSE) + if match: + idx, n, sharp, mod_name, much_stuff, code1, code2, tim = tup = match.groups() + # either code1 or code2 is None + code = code1 or code2 + idx, n, code, tim = int(idx), int(n), code.lower(), float(tim) + res = TestResult(idx, mod_name, code == "passed", code, tim) + result.append(res) + return result + + +class TestParser(object): + def __init__(self, test_log): + self._result = _parse_tests(test_log) + + @property + def result(self): + return self._result + + def __len__(self): + return len(self._result) + + def iter_blacklist(self, blacklist): + bl = blacklist + for line in self._result: + mod_name = line.mod_name + passed = line.passed + match = bl.find_matching_line(line) + if not passed: + if match: + res = "BFAIL" + else: + res = "FAIL!" + else: + if match: + res = "BPASS" + else: + res = "PASS" + yield mod_name, res diff --git a/testing/runner.py b/testing/runner.py new file mode 100644 index 000000000..f86bdeebf --- /dev/null +++ b/testing/runner.py @@ -0,0 +1,219 @@ +############################################################################# +## +## Copyright (C) 2017 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from __future__ import print_function + +import os +import sys +import re +import subprocess +import inspect + +from collections import namedtuple +from textwrap import dedent + +from .buildlog import builds +from .helper import decorate, PY3, TimeoutExpired + +sys.path.append('..') +from utils import detectClang + +class TestRunner(object): + def __init__(self, log_entry, project, index): + self.log_entry = log_entry + built_path = log_entry.build_dir + self.test_dir = os.path.join(built_path, project) + log_dir = log_entry.log_dir + if index is not None: + self.logfile = os.path.join(log_dir, project + ".{}.log".format(index)) + else: + self.logfile = os.path.join(log_dir, project + ".log") + os.environ['CTEST_OUTPUT_ON_FAILURE'] = '1' + self._setupClang() + self._setup() + + def _setupClang(self): + if sys.platform != "win32": + return + clangDir = detectClang() + if clangDir[0]: + clangBinDir = os.path.join(clangDir[0], 'bin') + path = os.environ.get('PATH') + if not clangBinDir in path: + os.environ['PATH'] = clangBinDir + os.pathsep + path + print("Adding %s as detected by %s to PATH" % (clangBinDir, clangDir[1])) + + def _find_ctest(self): + """ + Find ctest in the Makefile + + We no longer use make, but the ctest command directly. + It is convenient to look for the ctest program using the Makefile. + This serves us two purposes: + + - there is no dependency of the PATH variable, + - each project is checked whether ctest was configured. + """ + make_path = os.path.join(self.test_dir, "Makefile") + look_for = "--force-new-ctest-process" + line = None + with open(make_path) as makefile: + for line in makefile: + if look_for in line: + break + else: + # We have probably forgotten to build the tests. + # Give a nice error message with a shortened but exact path. + rel_path = os.path.relpath(make_path) + msg = dedent("""\n + {line} + ** ctest is not in '{}'. + * Did you forget to build the tests with '--build-tests' in setup.py? + """).format(rel_path, line=79 * "*") + raise RuntimeError(msg) + # the ctest program is on the left to look_for + assert line, "Did not find {}".format(look_for) + ctest = re.search(r'(\S+|"([^"]+)")\s+' + look_for, line).groups() + return ctest[1] or ctest[0] + + def _setup(self): + self.ctestCommand = self._find_ctest() + + def _run(self, cmd_tuple, label, timeout): + """ + Perform a test run in a given build + + The build can be stopped by a keyboard interrupt for testing + this script. Also, a timeout can be used. + + After the change to directly using ctest, we no longer use + "--force-new-ctest-process". Until now this has no drawbacks + but was a little faster. + """ + + self.cmd = cmd_tuple + # We no longer use the shell option. It introduces wrong handling + # of certain characters which are not yet correctly escaped: + # Especially the "^" caret char is treated as an escape, and pipe symbols + # without a caret are interpreted as such which leads to weirdness. + # Since we have all commands with explicit paths and don't use shell + # commands, this should work fine. + print(dedent("""\ + running {cmd} + in {test_dir} + """).format(**self.__dict__)) + ctest_process = subprocess.Popen(self.cmd, + cwd=self.test_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + def py_tee(input, output, label): + ''' + A simple (incomplete) tee command in Python + + This script simply logs everything from input to output + while the output gets some decoration. The specific reason + to have this script at all is: + + - it is necessary to have some decoration as prefix, since + we run commands several times + + - collecting all output and then decorating is not nice if + you have to wait for a long time + + The special escape is for the case of an embedded file in + the output. + ''' + def xprint(*args, **kw): + print(*args, file=output, **kw) + + # 'for line in input:' would read into too large chunks + while True: + line = input.readline() + if not line: + break + labelled = True + if line.startswith('BEGIN_FILE'): + labelled = False + txt = line.rstrip() + xprint(label, txt) if label and labelled else xprint(txt) + if line.startswith('END_FILE'): + labelled = True + + tee_src = dedent("""\ + from __future__ import print_function + import sys + {} + py_tee(sys.stdin, sys.stdout, '{label}') + """).format(dedent(inspect.getsource(py_tee)), label=label) + tee_cmd = (sys.executable, "-E", "-u", "-c", tee_src) + tee_process = subprocess.Popen(tee_cmd, + cwd=self.test_dir, + stdin=ctest_process.stdout) + try: + comm = tee_process.communicate + output = (comm(timeout=timeout) if PY3 else comm())[0] + except (TimeoutExpired, KeyboardInterrupt): + print() + print("aborted, partial result") + ctest_process.kill() + outs, errs = ctest_process.communicate() + # ctest lists to a temp file. Move it to the log + tmp_name = self.logfile + ".tmp" + if os.path.exists(tmp_name): + if os.path.exists(self.logfile): + os.unlink(self.logfile) + os.rename(tmp_name, self.logfile) + self.partial = True + else: + self.partial = False + finally: + print("End of the test run") + print() + tee_process.wait() + + def run(self, label, rerun, timeout): + cmd = self.ctestCommand, "--output-log", self.logfile + if rerun is not None: + # cmd += ("--rerun-failed",) + # For some reason, this worked never in the script file. + # We pass instead the test names as a regex: + words = "^(" + "|".join(rerun) + ")$" + cmd += ("--tests-regex", words) + self._run(cmd, label, timeout) +# eof diff --git a/testrunner.py b/testrunner.py index 557363ff6..e3a30a07e 100644 --- a/testrunner.py +++ b/testrunner.py @@ -37,806 +37,10 @@ ## ############################################################################# -from __future__ import print_function -from utils import detectClang +from __future__ import print_function, absolute_import -""" -testrunner - -Provide an interface to the pyside tests. - -- find the latest build dir. - This is found by the build_history in setup.py, - near the end of pyside_build.run() - -- run 'make test' and record the output - (not ready) - -- compare the result list with the current blacklist - -- return the correct error status - (zero if expected includes observed, else 1) - -Recommended build process: -There is no need to install the project. -Building the project with something like - - python setup.py build --build-tests --qmake=<qmakepath> --ignore-git --debug - -is sufficient. The tests are run by changing into the latest build dir and there -into pyside2, then 'make test'. - -""" - -import os import sys -import re -import subprocess -import zipfile -import argparse - -PY3 = sys.version_info[0] == 3 # from the six module -from subprocess import PIPE -if PY3: - from subprocess import TimeoutExpired - from io import StringIO -else: - class SubprocessError(Exception): pass - # this is a fake, just to keep the source compatible. - # timeout support is in python 3.3 and above. - class TimeoutExpired(SubprocessError): pass - from StringIO import StringIO -from collections import namedtuple - -# Change the cwd to our source dir -try: - this_file = __file__ -except NameError: - this_file = sys.argv[0] -this_file = os.path.abspath(this_file) -if os.path.dirname(this_file): - os.chdir(os.path.dirname(this_file)) -script_dir = os.getcwd() - -LogEntry = namedtuple("LogEntry", ["log_dir", "build_dir"]) - -def setupClang(): - if sys.platform != "win32": - return - clangDir = detectClang() - if clangDir[0]: - clangBinDir = os.path.join(clangDir[0], 'bin') - path = os.environ.get('PATH') - if not clangBinDir in path: - os.environ['PATH'] = clangBinDir + os.pathsep + path - print("Adding %s as detected by %s to PATH" % (clangBinDir, clangDir[1])) - -class BuildLog(object): - """ - This class is a convenience wrapper around a list of log entries. - - The list of entries is sorted by date and checked for consistency. - For simplicity and readability, the log entries are named tuples. - - """ - def __init__(self, script_dir=script_dir): - history_dir = os.path.join(script_dir, 'build_history') - build_history = [] - for timestamp in os.listdir(history_dir): - log_dir = os.path.join(history_dir, timestamp) - if not os.path.isdir(log_dir): - continue - fpath = os.path.join(log_dir, 'build_dir.txt') - if not os.path.exists(fpath): - print("Warning: %s not found, skipped" % fpath) - continue - with open(fpath) as f: - build_dir = f.read().strip() - if not os.path.exists(build_dir): - rel_dir, low_part = os.path.split(build_dir) - rel_dir, two_part = os.path.split(rel_dir) - if two_part.startswith("pyside") and two_part.endswith("build"): - build_dir = os.path.abspath(os.path.join(two_part, low_part)) - if os.path.exists(build_dir): - print("Note: build_dir was probably moved.") - else: - print("Warning: missing build dir %s" % build_dir) - continue - entry = LogEntry(log_dir, build_dir) - build_history.append(entry) - # we take the latest build for now. - build_history.sort() - self.history = build_history - self._buildno = None - - def set_buildno(self, buildno): - self.history[buildno] # test - self._buildno = buildno - - @property - def selected(self): - if self._buildno is None: - return None - if self.history is None: - return None - return self.history[self._buildno] - - @property - def classifiers(self): - if not self.selected: - raise ValueError('+++ No build with the configuration found!') - # Python2 legacy: Correct 'linux2' to 'linux', recommended way. - platform = 'linux' if sys.platform.startswith('linux') else sys.platform - res = [platform] - # the rest must be guessed from the given filename - path = self.selected.build_dir - base = os.path.basename(path) - res.extend(base.split('-')) - # add all the python and qt subkeys - for entry in res: - parts = entry.split(".") - for idx in range(len(parts)): - key = ".".join(parts[:idx]) - if key not in res: - res.append(key) - return res - - -class TestRunner(object): - def __init__(self, log_entry, project): - self.log_entry = log_entry - built_path = log_entry.build_dir - self.test_dir = os.path.join(built_path, project) - log_dir = log_entry.log_dir - self.logfile = os.path.join(log_dir, project + ".log") - os.environ['CTEST_OUTPUT_ON_FAILURE'] = '1' - self._setup() - - def _setup(self): - if sys.platform == 'win32': - # Windows: Helper implementing 'which' command using 'where.exe' - def winWhich(binary): - cmd = ['where.exe', binary] - stdOut = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout - result = stdOut.readlines() - stdOut.close() - if len(result) > 0: - return re.compile('\\s+').sub(' ', result[0].decode('utf-8')) - return None - - self.makeCommand = 'nmake' - qmakeSpec = os.environ.get('QMAKESPEC') - if qmakeSpec is not None and 'g++' in qmakeSpec: - self.makeCommand = 'mingw32-make' - # Can 'tee' be found in the environment (MSYS-git installation with usr/bin in path)? - self.teeCommand = winWhich('tee.exe') - if self.teeCommand is None: - git = winWhich('git.exe') - if not git: - # In COIN we have only git.cmd in path - git = winWhich('git.cmd') - if 'cmd' in git: - # Check for a MSYS-git installation with 'cmd' in the path and grab 'tee' from usr/bin - index = git.index('cmd') - self.teeCommand = git[0:index] + 'bin\\tee.exe' - if not os.path.exists(self.teeCommand): - self.teeCommand = git[0:index] + 'usr\\bin\\tee.exe' # git V2.8.X - if not os.path.exists(self.teeCommand): - raise "Cannot locate 'tee' command" - - else: - self.makeCommand = 'make' - self.teeCommand = 'tee' - - def run(self, timeout = 300): - """ - perform a test run in a given build. The build can be stopped by a - keyboard interrupt for testing this script. Also, a timeout can - be used. - """ - - if sys.platform == "win32": - cmd = (self.makeCommand, 'test') - tee_cmd = (self.teeCommand, self.logfile) - print("running", cmd, 'in', self.test_dir, ',\n logging to', self.logfile, 'using ', tee_cmd) - make = subprocess.Popen(cmd, cwd=self.test_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - tee = subprocess.Popen(tee_cmd, cwd=self.test_dir, stdin=make.stdout, shell=True) - else: - cmd = (self.makeCommand, 'test') - tee_cmd = (self.teeCommand, self.logfile) - print("running", cmd, 'in', self.test_dir, ',\n logging to', self.logfile, 'using ', tee_cmd) - make = subprocess.Popen(cmd, cwd=self.test_dir, stdout=subprocess.PIPE) - tee = subprocess.Popen(tee_cmd, cwd=self.test_dir, stdin=make.stdout) - make.stdout.close() - try: - if PY3: - output = tee.communicate(timeout=timeout)[0] - else: - output = tee.communicate()[0] - except (TimeoutExpired, KeyboardInterrupt): - print() - print("aborted") - tee.kill() - make.kill() - outs, errs = tee.communicate() - finally: - print("End of the test run") - tee.wait() - - -_EXAMPLE = """ -Example output: - -ip1 n sharp mod_name code tim ------------------------------------------------------------------------------------------ -114/391 Test #114: QtCore_qfileinfo_test-42 ........................ Passed 0.10 sec - Start 115: QtCore_qfile_test -115/391 Test #115: QtCore_qfile_test ...............................***Failed 0.11 sec - Start 116: QtCore_qflags_test - -We will only look for the dotted lines and calculate everything from that. -The summary statistics at the end will be ignored. That allows us to test -this functionality with short timeout values. - -Note the field "mod_name". I had split this before, but it is necessary -to use the combination as the key, because the test names are not unique. -""" - -# validation of our pattern: - -_TEST_PAT = r""" - ^ # start - \s* # any whitespace ==: WS - ([0-9]+)/([0-9]+) # ip1 "/" n - \s+ # some WS - Test # "Test" - \s+ # some WS - \# # sharp symbol "#" - ([0-9]+) # sharp - : # colon symbol ':' - \s+ # some WS - ([\w-]+) # mod_name - .*? # whatever (non greedy) - ( # - (Passed) # either "Passed", None - | # - \*\*\*(\w+.*?) # or None, "Something" - ) # code - \s+ # some WS - ([0-9]+\.[0-9]+) # tim - \s+ # some WS - sec # "sec" - \s* # any WS - $ # end - """ -assert re.match(_TEST_PAT, _EXAMPLE.splitlines()[5], re.VERBOSE) -assert len(re.match(_TEST_PAT, _EXAMPLE.splitlines()[5], re.VERBOSE).groups()) == 8 -assert len(re.match(_TEST_PAT, _EXAMPLE.splitlines()[7], re.VERBOSE).groups()) == 8 - -TestResult = namedtuple("TestResult", ["idx", "mod_name", "passed", - "code", "time"]) - - -class TestParser(object): - def __init__(self, test_log): - self._result = _parse_tests(test_log) - - @property - def result(self): - return self._result - - def __len__(self): - return len(self._result) - - def iter_blacklist(self, blacklist): - bl = blacklist - for line in self._result: - mod_name = line.mod_name - passed = line.passed - match = bl.find_matching_line(line) - if not passed: - if match: - res = "BFAIL" - else: - res = "FAIL" - else: - if match: - res = "BPASS" - else: - res = "PASS" - yield mod_name, res - - -class BlackList(object): - def __init__(self, blname): - if blname == None: - f = StringIO() - self.raw_data = [] - else: - with open(blname) as f: - self.raw_data = f.readlines() - # keep all lines, but see what is not relevant - lines = self.raw_data[:] - - def filtered_line(line): - if '#' in line: - line = line[0:line.index('#')] - return line.split() - - # now put every bracketed line in a test - # and use subsequent identifiers for a match - def is_test(fline): - return fline and fline[0].startswith("[") - - self.tests = {} - - if not lines: - # nothing supplied - return - - self.index = {} - for idx, line in enumerate(lines): - fline = filtered_line(line) - if not fline: - continue - if is_test(fline): - break - # we have a global section - name = '' - self.tests[name] = [] - for idx, line in enumerate(lines): - fline = filtered_line(line) - if is_test(fline): - # a new name - name = decorate(fline[0][1:-1]) - self.tests[name] = [] - self.index[name] = idx - elif fline: - # a known name with a new entry - self.tests[name].append(fline) - - def find_matching_line(self, test): - """ - Take a test result. - Find a line in the according blacklist file where all keys of the line are found. - If line not found, do nothing. - if line found and test passed, it is a BPASS. - If line found and test failed, it is a BFAIL. - """ - passed = test.passed - classifiers = set(builds.classifiers) - - if "" in self.tests: - # this is a global section - for line in self.tests[""]: - keys = set(line) - if keys <= classifiers: - # found a match! - return line - mod_name = test.mod_name - if mod_name not in self.tests and decorate(mod_name) not in self.tests: - return None - if mod_name in self.tests: - thing = mod_name - else: - thing = decorate(mod_name) - for line in self.tests[thing]: - keys = set(line) - if keys <= classifiers: - # found a match! - return line - else: - return None # noting found - - -""" -Simplified blacklist file -------------------------- - -A comment reaches from '#' to the end of line. -The file starts with an optional global section. -A test is started with a [square-bracketed] section name. -A line matches if all keys in the line are found. -If a line matches, the corresponding test is marked BFAIL or BPASS depending if the test passed or -not. - -Known keys are: - -darwin -win32 -linux -... - -qt5.6.1 -qt5.6.2 -... - -py3 -py2 - -32bit -64bit - -debug -release -""" - -""" -Data Folding v2 -=============== - -In the first layout of data folding, we distinguished complete domains -like "debug/release" and incomplete domains like "ubuntu/win32" which -can be extended to any number. - -This version is simpler. We do a first pass over all data and collect -all data. Therefore, incomplete domains do not exist. The definition -of the current members of the domain goes into a special comment at -the beginning of the file. - - -Compressing a blacklist ------------------------ - -When we have many samples of data, it is very likely to get very similar -entries. The redundancy is quite high, and we would like to compress -data without loosing information. - -Consider the following data set: - -[some::sample_test] - darwin qt5.6.1 py3 64bit debug - darwin qt5.6.1 py3 64bit release - darwin qt5.6.1 py2 64bit debug - darwin qt5.6.1 py2 64bit release - win32 qt5.6.1 py3 64bit debug - win32 qt5.6.1 py3 64bit release - win32 qt5.6.1 py2 64bit debug - win32 qt5.6.1 py2 64bit release - -The keys "debug" and "release" build the complete set of keys in their -domain. When sorting the lines, we can identify all similar entries which -are only different by the keys "debug" and "release". - -[some::sample_test] - darwin qt5.6.1 py3 64bit - darwin qt5.6.1 py2 64bit - win32 qt5.6.1 py3 64bit - win32 qt5.6.1 py2 64bit - -We can do the same for "py3" and "py2", because we have again the complete -set of possible keys available: - -[some::sample_test] - darwin qt5.6.1 64bit - win32 qt5.6.1 64bit - -The operating system has the current keys "darwin" and "win32". -They are kept in a special commend, and we get: - -# COMPRESSION: darwin win32 -[some::sample_test] - qt5.6.1 64bit - - -Expanding a blacklist ---------------------- - -All of the above steps are completely reversible. - - -Alternate implementation ------------------------- - -Instead of using a special comment, I am currently in favor of -the following: - -The global section gets the complete set of variables, like so - -# Globals - darwin win32 linux - qt5.6.1 qt5.6.2 - py3 py2 - 32bit 64bit - debug release -[some::sample_test] - qt5.6.1 64bit - -This approach has the advantage that it does not depend on comments. -The lines in the global section can always added without any conflict, -because these test results are impossible. Therefore, we list all our -keys without adding anything that could influence a test. -It makes also sense to have everything explicitly listed here. -""" - -def _parse_tests(test_log): - """ - Create a TestResult object for every entry. - """ - result = [] - if isinstance(test_log, StringIO): - lines = test_log.readlines() - elif test_log is not None and os.path.exists(test_log): - with open(test_log) as f: - lines = f.readlines() - else: - lines = [] - pat = _TEST_PAT - for line in lines: - match = re.match(pat, line, re.VERBOSE) - if match: - idx, n, sharp, mod_name, much_stuff, code1, code2, tim = tup = match.groups() - # either code1 or code2 is None - code = code1 or code2 - if idx != sharp: - raise ValueError("inconsistent log lines or program error: %s" % tup) - idx, n, code, tim = int(idx), int(n), code.lower(), float(tim) - res = TestResult(idx, mod_name, code == "passed", code, tim) - result.append(res) - return result - -def decorate(mod_name): - """ - Write the combination of "modulename_funcname" - in the Qt-like form "modulename::funcname" - """ - if "_" not in mod_name: - return mod_name - if "::" in mod_name: - return mod_name - name, rest = mod_name.split("_", 1) - return name + "::" + rest - -def create_read_write(filename): - if os.path.isfile(filename): - # existing file, open for read and write - return open(filename, 'r+') - elif os.path.exists(filename): - # a directory? - raise argparse.ArgumentTypeError(None, "invalid file argument: %s" % filename) - else: - try: - return open(filename, 'w') - except IOError: - raise argparse.ArgumentError(None, "cannot create file: %s" % filename) - -def learn_blacklist(fname, result, selected): - with open(fname, "r+") as f: - _remove_from_blacklist(f.name) - _add_to_blacklist(f.name, result) - _update_header(f.name, selected) - -def _remove_from_blacklist(old_blname): - # get rid of existing classifiers - classifierset = set(builds.classifiers) - - # for every line, remove comments and see if the current set if an exact - # match. We will touch only exact matches. - def filtered_line(line): - if '#' in line: - line = line[0:line.index('#')] - return line.split() - - with open(old_blname, "r") as f: - lines = f.readlines() - deletions = [] - for idx, line in enumerate(lines): - fline = filtered_line(line) - if not fline: - continue - if '[' in fline[0]: - # a heading line - continue - if set(fline) == classifierset: - deletions.append(idx) - while deletions: - delete = deletions.pop() - del lines[delete] - # remove all blank lines, but keep comments - for idx, line in reversed(list(enumerate(lines))): - if not line.split(): - del lines[idx] - # remove all consecutive sections, but keep comments - for idx, line in reversed(list(enumerate(lines))): - fline = line.split() - if fline and fline[0].startswith("["): - if idx+1 == len(lines): - # remove section at the end - del lines[idx] - continue - gline = lines[idx+1].split() - if gline and gline[0].startswith("["): - # next section right after this, remove this - del lines[idx] - with open(old_blname, "w") as f: - f.writelines(lines) - -def _add_to_blacklist(old_blname, result): - # insert new classifiers - classifiers = " " + " ".join(builds.classifiers) + "\n" - insertions = [] - additions = [] - old_bl = BlackList(old_blname) - lines = old_bl.raw_data[:] - if lines and not lines[-1].endswith("\n"): - lines[-1] += "\n" - for test in result: - if test.passed: - continue - if test.mod_name in old_bl.tests: - # the test is already there, add to the first line - idx = old_bl.index[test.mod_name] - insertions.append(idx) - if decorate(test.mod_name) in old_bl.tests: - # the same, but the name was decorated - idx = old_bl.index[decorate(test.mod_name)] - insertions.append(idx) - else: - # the test is new, append it to the end - additions.append("[" + decorate(test.mod_name) + "]\n") - while insertions: - this = insertions.pop() - lines[this] += classifiers - for line in additions: - lines.append(line) - lines.append(classifiers) - # now write the data out - with open(old_blname, "r+") as f: - f.writelines(lines) - -def _update_header(old_blname, selected): - with open(old_blname) as f: - lines = f.readlines() - classifierset = set(builds.classifiers) - for idx, line in reversed(list(enumerate(lines))): - fline = line.split() - if fline and fline[0].startswith('#'): - if set(fline) >= classifierset: - del lines[idx] - - classifiers = " ".join(builds.classifiers) - path = selected.log_dir - base = os.path.basename(path) - test = '### test date = %s classifiers = %s\n' % (base, classifiers) - lines.insert(0, test) - with open(old_blname, "w") as f: - f.writelines(lines) - - -if __name__ == '__main__': - # create the top-level parser - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(dest="subparser_name") - - # create the parser for the "test" command - parser_test = subparsers.add_parser("test") - group = parser_test.add_mutually_exclusive_group(required=False) - blacklist_default = os.path.join(script_dir, 'build_history', 'blacklist.txt') - group.add_argument("--blacklist", "-b", type=argparse.FileType('r'), - default=blacklist_default, - help='a Qt blacklist file (default: {})'.format(blacklist_default)) - group.add_argument("--learn", "-l", type=create_read_write, - help="add new entries to a blacklist file") - parser_test.add_argument("--skip", action='store_true', - help="skip the tests if they were run before") - parser_test.add_argument("--environ", nargs='+', - help="use name=value ... to set environment variables") - parser_test.add_argument("--buildno", default=-1, type=int, - help="use build number n (0-based), latest = -1 (default)") - all_projects = "shiboken2 pyside2 pyside2-tools".split() - tested_projects = "shiboken2 pyside2".split() - parser_test.add_argument("--projects", nargs='+', type=str, - default=tested_projects, - choices=all_projects, - help="use 'pyside2' (default) or other projects") - parser_getcwd = subparsers.add_parser("getcwd") - parser_getcwd.add_argument("filename", type=argparse.FileType('w'), - help="write the build dir name into a file") - parser_getcwd.add_argument("--buildno", default=-1, type=int, - help="use build number n (0-based), latest = -1 (default)") - parser_list = subparsers.add_parser("list") - args = parser.parse_args() - - print("System:\n Platform=%s\n Executable=%s\n Version=%s\n API version=%s\n\nEnvironment:" % - (sys.platform, sys.executable, sys.version.replace("\n", " "), sys.api_version)) - for v in sorted(os.environ.keys()): - print(" %s=%s" % (v, os.environ[v])) - - builds = BuildLog(script_dir) - if hasattr(args, "buildno"): - try: - builds.set_buildno(args.buildno) - except IndexError: - print("history out of range. Try '%s list'" % __file__) - sys.exit(1) - - if args.subparser_name == "getcwd": - print(builds.selected.build_dir, file=args.filename) - print(builds.selected.build_dir, "written to file", args.filename.name) - sys.exit(0) - elif args.subparser_name == "test": - pass # we do it afterwards - elif args.subparser_name == "list": - rp = os.path.relpath - print() - print("History") - print("-------") - for idx, build in enumerate(builds.history): - print(idx, rp(build.log_dir), rp(build.build_dir)) - print() - print("Note: only the last history entry of a folder is valid!") - sys.exit(0) - else: - parser.print_help() - sys.exit(1) - - if args.blacklist: - args.blacklist.close() - bl = BlackList(args.blacklist.name) - elif args.learn: - args.learn.close() - learn_blacklist(args.learn.name, result.result, builds.selected) - bl = BlackList(args.learn.name) - else: - bl = BlackList(None) - if args.environ: - for line in args.environ: - things = line.split("=") - if len(things) != 2: - raise ValueError("you need to pass one or more name=value pairs.") - key, value = things - os.environ[key] = value - - q = 5 * [0] - - setupClang() - - # now loop over the projects and accumulate - for project in args.projects: - runner = TestRunner(builds.selected, project) - if os.path.exists(runner.logfile) and args.skip: - print("Parsing existing log file:", runner.logfile) - else: - runner.run(10 * 60) - result = TestParser(runner.logfile) - r = 5 * [0] - print("********* Start testing of %s *********" % project) - print("Config: Using", " ".join(builds.classifiers)) - for test, res in result.iter_blacklist(bl): - print("%-6s" % res, ":", decorate(test) + "()") - r[0] += 1 if res == "PASS" else 0 - r[1] += 1 if res == "FAIL" else 0 - r[2] += 1 if res == "SKIPPED" else 0 # not yet supported - r[3] += 1 if res == "BFAIL" else 0 - r[4] += 1 if res == "BPASS" else 0 - print("Totals:", sum(r), "tests.", - "{} passed, {} failed, {} skipped, {} blacklisted, {} bpassed." - .format(*r)) - print("********* Finished testing of %s *********" % project) - print() - q = list(map(lambda x, y: x+y, r, q)) - - if len(args.projects) > 1: - print("All above projects:", sum(q), "tests.", - "{} passed, {} failed, {} skipped, {} blacklisted, {} bpassed." - .format(*q)) - print() - - # nag us about unsupported projects - ap, tp = set(all_projects), set(tested_projects) - if ap != tp: - print("+++++ Note: please support", " ".join(ap-tp), "+++++") - print() +import testing +import testing.blacklist # just to be sure it's us... - for project in args.projects: - runner = TestRunner(builds.selected, project) - result = TestParser(runner.logfile) - for test, res in result.iter_blacklist(bl): - if res == "FAIL": - raise ValueError("At least one failure was not blacklisted") - # the makefile does run, although it does not find any tests. - # We simply check if any tests were found. - if len(result) == 0: - path = builds.selected.build_dir - project = os.path.join(path, args.project) - raise ValueError("there are no tests in %s" % project) +testing.main() |