diff options
| author | Christian Tismer <tismer@stackless.com> | 2017-12-21 19:44:25 +0100 |
|---|---|---|
| committer | Christian Tismer <tismer@stackless.com> | 2017-12-22 12:52:54 +0000 |
| commit | a18994c2ff7836a5b6998f6aee24583d920dad80 (patch) | |
| tree | 9e6f0e1c1df5e5f1ae0acc5d09bd91326d9fce89 /testing | |
| parent | 5c7e140036e26d8dad27ef8b7da984f093e97787 (diff) | |
small enhancement for testrunner
The new testrunner now has the multiple test feature.
We refine and correct the summary view a little bit, remove
more dead code, add some comments and improve the output slightly.
Change-Id: I152bf5cbe2171fb07de7e88054a42fc767dc14c0
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Simo Fält <simo.falt@qt.io>
Diffstat (limited to 'testing')
| -rw-r--r-- | testing/command.py | 36 | ||||
| -rw-r--r-- | testing/helper.py | 1 | ||||
| -rw-r--r-- | testing/runner.py | 1 |
3 files changed, 18 insertions, 20 deletions
diff --git a/testing/command.py b/testing/command.py index 612672c5a..67dd05a03 100644 --- a/testing/command.py +++ b/testing/command.py @@ -94,19 +94,6 @@ if (os.environ.get("COIN_RERUN_FAILED_ONLY", "1").lower() in "0 f false n no".split()): COIN_RERUN_FAILED_ONLY = False -def create_read_write(filename): - if os.path.isfile(filename): - # existing file, open for read and write - return open(filename, 'r+') - elif os.path.exists(filename): - # a directory? - raise argparse.ArgumentTypeError(None, "invalid file argument: %s" % filename) - else: - try: - return open(filename, 'w') - except IOError: - raise argparse.ArgumentError(None, "cannot create file: %s" % filename) - def test_project(project, args, blacklist, runs): ret = [] for idx in range(runs): @@ -121,6 +108,10 @@ def test_project(project, args, blacklist, runs): else: if index > 1 and COIN_RERUN_FAILED_ONLY: rerun = rerun_list + if not rerun: + print("--- no re-runs found, stopping before test {} ---" + .format(index)) + break else: rerun = None runner.run("RUN {}:".format(idx + 1), rerun, 10 * 60) @@ -247,8 +238,6 @@ def main(): q = 5 * [0] - runs = 1 - fail_crit = 1 runs = COIN_TESTING fail_crit = COIN_THRESHOLD # now loop over the projects and accumulate @@ -275,15 +264,16 @@ def main(): tot_res[key].append(res) tot_flaky = 0 print("*" * 79) + print("**") + print("* Summary Of All Tests") print("*") - print("* Summary Of All Tests") - print("*") - print() + empty = True for test, res in tot_res.items(): pass__c = res.count("PASS") bpass_c = res.count("BPASS") fail__c = res.count("FAIL!") bfail_c = res.count("BFAIL") + fail2_c = fail__c + bfail_c if pass__c == len(res): continue elif bpass_c == runs and runs > 1: @@ -292,18 +282,24 @@ def main(): msg = "Newly detected Real test failure!" elif bfail_c == runs: msg = "Keep blacklisting ;-(" - elif fail__c > 0 and fail__c < len(res): + elif fail2_c > 0 and fail2_c < len(res): msg = "Flaky test" tot_flaky += 1 else: continue + empty = False padding = 6 * runs - txt = " ".join(((piece + " ")[:5] for piece in res)) + txt = " ".join(("{:<{width}}".format(piece, width=5) for piece in res)) txt = (txt + padding * " ")[:padding] testpad = 36 if len(test) < testpad: test += (testpad - len(test)) * " " print(txt, decorate(test), msg) + if empty: + print("* (empty)") + print("*") + print("**") + print("*" * 79) print() if runs > 1: print("Total flaky tests: errors but not always = {}".format(tot_flaky)) diff --git a/testing/helper.py b/testing/helper.py index 552955b86..8ea3e6a6b 100644 --- a/testing/helper.py +++ b/testing/helper.py @@ -70,3 +70,4 @@ def decorate(mod_name): name, rest = mod_name.split("_", 1) return name + "::" + rest +#eof diff --git a/testing/runner.py b/testing/runner.py index 90076f582..a993c3a5c 100644 --- a/testing/runner.py +++ b/testing/runner.py @@ -148,6 +148,7 @@ class TestRunner(object): def xprint(*args, **kw): print(*args, file=output, **kw) + # 'for line in input:' would read into too large chunks while True: line = input.readline() if not line: |