diff options
Diffstat (limited to 'testing/command.py')
| -rw-r--r-- | testing/command.py | 332 |
1 files changed, 332 insertions, 0 deletions
diff --git a/testing/command.py b/testing/command.py new file mode 100644 index 000000000..67dd05a03 --- /dev/null +++ b/testing/command.py @@ -0,0 +1,332 @@ +############################################################################# +## +## Copyright (C) 2017 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of PySide2. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from __future__ import print_function + +""" +testrunner +========== + +Provide an interface to the pyside tests. +----------------------------------------- + +This program can only be run if PySide was build with tests enabled. +All tests are run in a single pass, and if not blacklisted, an error +is raised at the end of the run. + +Recommended build process: +There is no need to install the project. +Building the project with something like + + python setup.py build --build-tests --qmake=<qmakepath> --ignore-git --debug + +is sufficient. The tests are run by changing into the latest build dir and there +into pyside2, then 'make test'. + + +New testing policy: +------------------- + +The tests are now run 5 times, and errors are reported +when they appear at least 3 times. With the variable COIN_RERUN_FAILED_ONLY it is +possible to configure if all tests should be rerun or the failed ones, only. + +The full mode can be tested locally by setting + + export COIN_RERUN_FAILED_ONLY=0 +""" + +import os +import sys +import argparse +from textwrap import dedent +from collections import OrderedDict +from timeit import default_timer as timer + +from .helper import script_dir, decorate +from .buildlog import builds +from .blacklist import BlackList +from .runner import TestRunner +from .parser import TestParser + +# Should we repeat only failed tests? +COIN_RERUN_FAILED_ONLY = True +COIN_THRESHOLD = 3 # report error if >= +COIN_TESTING = 5 # number of runs + +if (os.environ.get("COIN_RERUN_FAILED_ONLY", "1").lower() in + "0 f false n no".split()): + COIN_RERUN_FAILED_ONLY = False + +def test_project(project, args, blacklist, runs): + ret = [] + for idx in range(runs): + index = idx + 1 + runner = TestRunner(builds.selected, project, index) + print() + print("********* Start testing of %s *********" % project) + print("Config: Using", " ".join(builds.classifiers)) + print() + if os.path.exists(runner.logfile) and args.skip: + print("Parsing existing log file:", runner.logfile) + else: + if index > 1 and COIN_RERUN_FAILED_ONLY: + rerun = rerun_list + if not rerun: + print("--- no re-runs found, stopping before test {} ---" + .format(index)) + break + else: + rerun = None + runner.run("RUN {}:".format(idx + 1), rerun, 10 * 60) + result = TestParser(runner.logfile) + r = 5 * [0] + rerun_list = [] + print() + for test, res in result.iter_blacklist(blacklist): + print("RES {}:".format(index), end=" ") + print("%-6s" % res, decorate(test) + "()") + r[0] += 1 if res == "PASS" else 0 + r[1] += 1 if res == "FAIL!" else 0 + r[2] += 1 if res == "SKIPPED" else 0 # not yet supported + r[3] += 1 if res == "BFAIL" else 0 + r[4] += 1 if res == "BPASS" else 0 + if res not in ("PASS", "BPASS"): + rerun_list.append(test) + print() + print("Totals:", sum(r), "tests.", + "{} passed, {} failed, {} skipped, {} blacklisted, {} bpassed." + .format(*r)) + print() + print("********* Finished testing of %s *********" % project) + print() + ret.append(r) + + return ret + +def main(): + # create the top-level command parser + start_time = timer() + all_projects = "shiboken2 pyside2 pyside2-tools".split() + tested_projects = "shiboken2 pyside2".split() + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=dedent("""\ + Run the tests for some projects, default = '{}'. + + Testing is now repeated up to {rep} times, and errors are + only reported if they occur {thr} or more times. + The environment variable COIN_RERUN_FAILED_ONLY controls if errors + are only repeated if there are errors. The default is "1". + """.format("' '".join(tested_projects), thr=COIN_THRESHOLD, rep=COIN_TESTING))) + subparsers = parser.add_subparsers(dest="subparser_name") + + # create the parser for the "test" command + parser_test = subparsers.add_parser("test") + group = parser_test.add_mutually_exclusive_group(required=False) + blacklist_default = os.path.join(script_dir, 'build_history', 'blacklist.txt') + group.add_argument("--blacklist", "-b", type=argparse.FileType('r'), + default=blacklist_default, + help='a Qt blacklist file (default: {})'.format(blacklist_default)) + parser_test.add_argument("--skip", action='store_true', + help="skip the tests if they were run before") + parser_test.add_argument("--environ", nargs='+', + help="use name=value ... to set environment variables") + parser_test.add_argument("--buildno", default=-1, type=int, + help="use build number n (0-based), latest = -1 (default)") + parser_test.add_argument("--projects", nargs='+', type=str, + default=tested_projects, + choices=all_projects, + help="use '{}'' (default) or other projects" + .format("' '".join(tested_projects))) + parser_getcwd = subparsers.add_parser("getcwd") + parser_getcwd.add_argument("filename", type=argparse.FileType('w'), + help="write the build dir name into a file") + parser_getcwd.add_argument("--buildno", default=-1, type=int, + help="use build number n (0-based), latest = -1 (default)") + parser_list = subparsers.add_parser("list") + args = parser.parse_args() + + if hasattr(args, "buildno"): + try: + builds.set_buildno(args.buildno) + except IndexError: + print("history out of range. Try '%s list'" % __file__) + sys.exit(1) + + if args.subparser_name == "getcwd": + print(builds.selected.build_dir, file=args.filename) + print(builds.selected.build_dir, "written to file", args.filename.name) + sys.exit(0) + elif args.subparser_name == "test": + pass # we do it afterwards + elif args.subparser_name == "list": + rp = os.path.relpath + print() + print("History") + print("-------") + for idx, build in enumerate(builds.history): + print(idx, rp(build.log_dir), rp(build.build_dir)) + print() + print("Note: only the last history entry of a folder is valid!") + sys.exit(0) + else: + parser.print_help() + sys.exit(1) + + if args.blacklist: + args.blacklist.close() + bl = BlackList(args.blacklist.name) + else: + bl = BlackList(None) + if args.environ: + for line in args.environ: + things = line.split("=") + if len(things) != 2: + raise ValueError("you need to pass one or more name=value pairs.") + key, value = things + os.environ[key] = value + + print(dedent("""\ + System: + Platform={platform} + Executable={executable} + Version={version_lf} + API version={api_version} + + Environment:""") + .format(version_lf=sys.version.replace("\n", " "), **sys.__dict__)) + for key, value in sorted(os.environ.items()): + print(" {}={}".format(key, value)) + print() + + q = 5 * [0] + + runs = COIN_TESTING + fail_crit = COIN_THRESHOLD + # now loop over the projects and accumulate + for project in args.projects: + res = test_project(project, args, bl, runs) + for idx, r in enumerate(res): + q = list(map(lambda x, y: x+y, r, q)) + + if len(args.projects) > 1: + print("All above projects:", sum(q), "tests.", + "{} passed, {} failed, {} skipped, {} blacklisted, {} bpassed." + .format(*q)) + print() + + tot_res = OrderedDict() + for project in args.projects: + for idx in range(runs): + index = idx + 1 + runner = TestRunner(builds.selected, project, index) + result = TestParser(runner.logfile) + for test, res in result.iter_blacklist(bl): + key = project + ":" + test + tot_res.setdefault(key, []) + tot_res[key].append(res) + tot_flaky = 0 + print("*" * 79) + print("**") + print("* Summary Of All Tests") + print("*") + empty = True + for test, res in tot_res.items(): + pass__c = res.count("PASS") + bpass_c = res.count("BPASS") + fail__c = res.count("FAIL!") + bfail_c = res.count("BFAIL") + fail2_c = fail__c + bfail_c + if pass__c == len(res): + continue + elif bpass_c == runs and runs > 1: + msg = "Remove blacklisting; test passes" + elif fail__c == runs: + msg = "Newly detected Real test failure!" + elif bfail_c == runs: + msg = "Keep blacklisting ;-(" + elif fail2_c > 0 and fail2_c < len(res): + msg = "Flaky test" + tot_flaky += 1 + else: + continue + empty = False + padding = 6 * runs + txt = " ".join(("{:<{width}}".format(piece, width=5) for piece in res)) + txt = (txt + padding * " ")[:padding] + testpad = 36 + if len(test) < testpad: + test += (testpad - len(test)) * " " + print(txt, decorate(test), msg) + if empty: + print("* (empty)") + print("*") + print("**") + print("*" * 79) + print() + if runs > 1: + print("Total flaky tests: errors but not always = {}".format(tot_flaky)) + print() + else: + print("For info about flaky tests, we need to perform more than one run.") + print("Please activate the COIN mode: 'export QTEST_ENVIRONMENT=ci'") + print() + # nag us about unsupported projects + ap, tp = set(all_projects), set(tested_projects) + if ap != tp: + print("+++++ Note: please support", " ".join(ap-tp), "+++++") + print() + + stop_time = timer() + used_time = stop_time - start_time + # Now create an error if the criterion is met: + try: + err_crit = "'FAIL! >= {}'".format(fail_crit) + for res in tot_res.values(): + if res.count("FAIL!") >= fail_crit: + raise ValueError("At least one failure was not blacklisted " + "and met the criterion {}" + .format(err_crit)) + print("No test met the error criterion {}".format(err_crit)) + finally: + print() + print("Total time of whole Python script = {:0.2f} sec".format(used_time)) + print() +# eof |