diff options
Diffstat (limited to 'testing')
| -rw-r--r-- | testing/blacklist.py | 225 | ||||
| -rw-r--r-- | testing/command.py | 24 |
2 files changed, 14 insertions, 235 deletions
diff --git a/testing/blacklist.py b/testing/blacklist.py index b6fd73f73..b11b7db6f 100644 --- a/testing/blacklist.py +++ b/testing/blacklist.py @@ -122,227 +122,4 @@ class BlackList(object): # found a match! return line else: - return None # noting found - - -""" -Simplified blacklist file -------------------------- - -A comment reaches from '#' to the end of line. -The file starts with an optional global section. -A test is started with a [square-bracketed] section name. -A line matches if all keys in the line are found. -If a line matches, the corresponding test is marked BFAIL or BPASS depending if the test passed or -not. - -Known keys are: - -darwin -win32 -linux -... - -qt5.6.1 -qt5.6.2 -... - -py3 -py2 - -32bit -64bit - -debug -release -""" - -""" -Data Folding v2 -=============== - -In the first layout of data folding, we distinguished complete domains -like "debug/release" and incomplete domains like "ubuntu/win32" which -can be extended to any number. - -This version is simpler. We do a first pass over all data and collect -all data. Therefore, incomplete domains do not exist. The definition -of the current members of the domain goes into a special comment at -the beginning of the file. - - -Compressing a blacklist ------------------------ - -When we have many samples of data, it is very likely to get very similar -entries. The redundancy is quite high, and we would like to compress -data without loosing information. - -Consider the following data set: - -[some::sample_test] - darwin qt5.6.1 py3 64bit debug - darwin qt5.6.1 py3 64bit release - darwin qt5.6.1 py2 64bit debug - darwin qt5.6.1 py2 64bit release - win32 qt5.6.1 py3 64bit debug - win32 qt5.6.1 py3 64bit release - win32 qt5.6.1 py2 64bit debug - win32 qt5.6.1 py2 64bit release - -The keys "debug" and "release" build the complete set of keys in their -domain. When sorting the lines, we can identify all similar entries which -are only different by the keys "debug" and "release". - -[some::sample_test] - darwin qt5.6.1 py3 64bit - darwin qt5.6.1 py2 64bit - win32 qt5.6.1 py3 64bit - win32 qt5.6.1 py2 64bit - -We can do the same for "py3" and "py2", because we have again the complete -set of possible keys available: - -[some::sample_test] - darwin qt5.6.1 64bit - win32 qt5.6.1 64bit - -The operating system has the current keys "darwin" and "win32". -They are kept in a special commend, and we get: - -# COMPRESSION: darwin win32 -[some::sample_test] - qt5.6.1 64bit - - -Expanding a blacklist ---------------------- - -All of the above steps are completely reversible. - - -Alternate implementation ------------------------- - -Instead of using a special comment, I am currently in favor of -the following: - -The global section gets the complete set of variables, like so - -# Globals - darwin win32 linux - qt5.6.1 qt5.6.2 - py3 py2 - 32bit 64bit - debug release -[some::sample_test] - qt5.6.1 64bit - -This approach has the advantage that it does not depend on comments. -The lines in the global section can always added without any conflict, -because these test results are impossible. Therefore, we list all our -keys without adding anything that could influence a test. -It makes also sense to have everything explicitly listed here. -""" - -def learn_blacklist(fname, result, selected): - with open(fname, "r+") as f: - _remove_from_blacklist(f.name) - _add_to_blacklist(f.name, result) - _update_header(f.name, selected) - -def _remove_from_blacklist(old_blname): - # get rid of existing classifiers - classifierset = set(builds.classifiers) - - # for every line, remove comments and see if the current set is an exact - # match. We will touch only exact matches. - def filtered_line(line): - if '#' in line: - line = line[0:line.index('#')] - return line.split() - - with open(old_blname, "r") as f: - lines = f.readlines() - deletions = [] - for idx, line in enumerate(lines): - fline = filtered_line(line) - if not fline: - continue - if '[' in fline[0]: - # a heading line - continue - if set(fline) == classifierset: - deletions.append(idx) - while deletions: - delete = deletions.pop() - del lines[delete] - # remove all blank lines, but keep comments - for idx, line in reversed(list(enumerate(lines))): - if not line.split(): - del lines[idx] - # remove all consecutive sections, but keep comments - for idx, line in reversed(list(enumerate(lines))): - fline = line.split() - if fline and fline[0].startswith("["): - if idx+1 == len(lines): - # remove section at the end - del lines[idx] - continue - gline = lines[idx+1].split() - if gline and gline[0].startswith("["): - # next section right after this, remove this - del lines[idx] - with open(old_blname, "w") as f: - f.writelines(lines) - -def _add_to_blacklist(old_blname, result): - # insert new classifiers - classifiers = " " + " ".join(builds.classifiers) + "\n" - insertions = [] - additions = [] - old_bl = BlackList(old_blname) - lines = old_bl.raw_data[:] - if lines and not lines[-1].endswith("\n"): - lines[-1] += "\n" - for test in result: - if test.passed: - continue - if test.mod_name in old_bl.tests: - # the test is already there, add to the first line - idx = old_bl.index[test.mod_name] - insertions.append(idx) - if decorate(test.mod_name) in old_bl.tests: - # the same, but the name was decorated - idx = old_bl.index[decorate(test.mod_name)] - insertions.append(idx) - else: - # the test is new, append it to the end - additions.append("[" + decorate(test.mod_name) + "]\n") - while insertions: - this = insertions.pop() - lines[this] += classifiers - for line in additions: - lines.append(line) - lines.append(classifiers) - # now write the data out - with open(old_blname, "r+") as f: - f.writelines(lines) - -def _update_header(old_blname, selected): - with open(old_blname) as f: - lines = f.readlines() - classifierset = set(builds.classifiers) - for idx, line in reversed(list(enumerate(lines))): - fline = line.split() - if fline and fline[0].startswith('#'): - if set(fline) >= classifierset: - del lines[idx] - - classifiers = " ".join(builds.classifiers) - path = selected.log_dir - base = os.path.basename(path) - test = '### test date = %s classifiers = %s\n' % (base, classifiers) - lines.insert(0, test) - with open(old_blname, "w") as f: - f.writelines(lines) + return None # nothing found diff --git a/testing/command.py b/testing/command.py index 4d9629989..5d37f63e6 100644 --- a/testing/command.py +++ b/testing/command.py @@ -70,10 +70,11 @@ into pyside2, then 'make test'. import os import sys import argparse +from textwrap import dedent from .helper import script_dir, decorate from .buildlog import builds -from .blacklist import BlackList, learn_blacklist +from .blacklist import BlackList from .runner import TestRunner from .parser import TestParser @@ -102,8 +103,6 @@ def main(): group.add_argument("--blacklist", "-b", type=argparse.FileType('r'), default=blacklist_default, help='a Qt blacklist file (default: {})'.format(blacklist_default)) - group.add_argument("--learn", "-l", type=create_read_write, - help="add new entries to a blacklist file") parser_test.add_argument("--skip", action='store_true', help="skip the tests if they were run before") parser_test.add_argument("--environ", nargs='+', @@ -154,10 +153,6 @@ def main(): if args.blacklist: args.blacklist.close() bl = BlackList(args.blacklist.name) - elif args.learn: - args.learn.close() - learn_blacklist(args.learn.name, result.result, builds.selected) - bl = BlackList(args.learn.name) else: bl = BlackList(None) if args.environ: @@ -168,10 +163,17 @@ def main(): key, value = things os.environ[key] = value - print("System:\n Platform=%s\n Executable=%s\n Version=%s\n API version=%s\n\nEnvironment:" % - (sys.platform, sys.executable, sys.version.replace("\n", " "), sys.api_version)) - for v in sorted(os.environ.keys()): - print(" %s=%s" % (v, os.environ[v])) + print(dedent("""\ + System: + Platform={platform} + Executable={executable} + Version={version_lf} + API version={api_version} + + Environment:""") + .format(version_lf=sys.version.replace("\n", " "), **sys.__dict__)) + for key, value in sorted(os.environ.items()): + print(" {}={}".format(key, value)) print() q = 5 * [0] |