diff options
| author | Daniel Smith <daniel.smith@qt.io> | 2019-07-03 16:22:04 +0200 |
|---|---|---|
| committer | Daniel Smith <daniel.smith@qt.io> | 2019-08-29 17:29:25 +0200 |
| commit | 8a79bcdf467d9e1e16821fdb3aff9ee8e3c652dd (patch) | |
| tree | 765272f70fb7af6e5a5e966f576f2ebbdffef17c /scripts | |
| parent | 68a9cf7cf8169589bfd1e9539a8695e661d1b036 (diff) | |
Add a blacklisting tool to help keep blacklists updated
This script pulls blacklist stats from the database and updates
blacklist files with any test platform configurations that have
failed at least once in the past 60 days. This action effectively
un-blacklists platforms that have only passed in the past 60 days.
The idea here is that some blacklisted tests will get fixed, either
by code updates or OS fixes that cause a blacklisted test to start
passing. Automatic mode will only ever update or remove existing
blacklist entries. A report is printed at application exit with
modified and skipped test info.
Must run against a qt5 directory or submodule with --qt5dir
Can be run in interactive mode with --interactive
Can fast-forward to a desired test with --fastForward
Change-Id: Ia278e7f179197c72113649b40a081fd5165acf8e
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/coin/blacklist_tool/README.md | 43 | ||||
| -rw-r--r-- | scripts/coin/blacklist_tool/blacklistTool.py | 1513 | ||||
| -rw-r--r-- | scripts/coin/blacklist_tool/platformEnums.py | 250 |
3 files changed, 1806 insertions, 0 deletions
diff --git a/scripts/coin/blacklist_tool/README.md b/scripts/coin/blacklist_tool/README.md new file mode 100644 index 00000000..3ae08a55 --- /dev/null +++ b/scripts/coin/blacklist_tool/README.md @@ -0,0 +1,43 @@ +# Easy Blacklist Maintenance Tool + +### Requirements +1. Python 3 +2. COIN database read access (testresults.qt.io) +3. Python modules: + 1. argparse + 2. influxdb + 3. prettytable + 4. PyInquirer + +### Usage +#### Example: +`python3 blacklistTool.py --qt5dir ~/qt5/ --interactive` + +#### Parameters: +[Required] `--qt5dir <path to directory of a qt5 checkout or a single submodule inside qt5>` + +[Optional] `--interactive` Enables interactive mode + +[Optional] `--fastForward <testName>` Runs queries for blacklisted tests as usual, but fast forwards +the script to the specified test name. + +#### Optional Environment variables +`INFLUX_DB_URL` The hostname where the coin database resides. Defaults to 'testresults.qt.io' +`INFLUX_DB_PORT` The port to connect to influxdb with. SSL is required. Defaults to port 443 +`INFLUX_DB_USER` The username used to login to the COIN database +`INFLUX_DB_PASSWORD` The password used to login to the COIN database + +#### Notes +- **Interactive mode:** This is the recommended mode of operation. You will be given +a chance to enter your database username and password manually, as well as edit +the query used to retrieve blacklisted tests. +- **Automatic mode:** If a testname is found but is only a partial match, f.ex. +`[tryAcquireWithTimeout:0.2s]` versus `[tryAcquireWithTimeout]`, a report of the +mismatch will be printed upon completion of the script. +**Interactive mode** you'll be asked what to do `(edit existing, replace, or delete)`. +- **All modes:** When a test is an exact match, and has had 0 failures on any platforms in +the past 60 days (default period), the test will be removed completely from the blacklist. +- **All modes:** If a test is deleted from the blacklist and no tests remain in it, the +BLACKLIST file will be deleted. +- **All modes:** If a blacklist item's failing configurations is unchanged, but the original + file contains trailing newlines, it may be rewritten to remove the newlines. diff --git a/scripts/coin/blacklist_tool/blacklistTool.py b/scripts/coin/blacklist_tool/blacklistTool.py new file mode 100644 index 00000000..40c5265c --- /dev/null +++ b/scripts/coin/blacklist_tool/blacklistTool.py @@ -0,0 +1,1513 @@ +############################################################################ +## +# Copyright (C) 2019 The Qt Company Ltd. +# Contact: https://www.qt.io/licensing/ +## +# This file is part of the Quality Assurance module of the Qt Toolkit. +## +# $QT_BEGIN_LICENSE:GPL-EXCEPT$ +# Commercial License Usage +# Licensees holding valid commercial Qt licenses may use this file in +# accordance with the commercial license agreement provided with the +# Software or, alternatively, in accordance with the terms contained in +# a written agreement between you and The Qt Company. For licensing terms +# and conditions see https://www.qt.io/terms-conditions. For further +# information use the contact form at https://www.qt.io/contact-us. +## +# GNU General Public License Usage +# Alternatively, this file may be used under the terms of the GNU +# General Public License version 3 as published by the Free Software +# Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +# included in the packaging of this file. Please review the following +# information to ensure the GNU General Public License requirements will +# be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +# $QT_END_LICENSE$ +## +############################################################################# + + +from __future__ import print_function, unicode_literals +import os +import sys +import argparse +import time +import atexit +import re +from prettytable import PrettyTable +from influxdb import InfluxDBClient +from influxdb import exceptions +from platformEnums import OS, COMPILER, PLATFORM +from enum import Enum +from PyInquirer import style_from_dict, Token, prompt, Separator +from pathlib import Path + + +# Setup a clear screen function for cleaning the output window. +def clear(): + """Clear the console screen using the OS built-in methods.""" + if sys.platform == "win32": + os.system('cls') + else: + os.system('clear') + + +# Set style for interactive interface +style = style_from_dict({ + Token.QuestionMark: '#E91E63 bold', + Token.Selected: '#673AB7 bold', + Token.Separator: '#e9c01e bold', + Token.Disabled: '#8D021F bold', + Token.Instruction: '', # default + Token.Answer: '#2196f3 bold', + Token.Question: '#ffff99 bold', +}) + + +class PlatformData(Enum): + """Enum to make accessing database results more human readable in code.""" + host_arch = 0 + host_compiler = 1 + host_os = 2 + host_os_version = 3 + target_arch = 4 + target_compiler = 5 + target_os = 6 + target_os_version = 7 + +INFLUX_DB_URL = "testresults.qt.io" if not os.environ.get("INFLUX_DB_URL") else os.environ.get("INFLUX_DB_URL") +INFLUX_DB_PORT = 443 if not os.environ.get("INFLUX_DB_PORT") else int(os.environ.get("INFLUX_DB_PORT")) +fastForward = False +modifiedFiles = set() +partialMatchesSkipped = list() + + +def onExit(): + """Print out a report following completion of the script.""" + + print("\n\n\nModified files during this run:") + print("\n".join(modifiedFiles)) + + print("\nBlacklist test cases that found parial matches (NOT MODIFIED):") + for item in partialMatchesSkipped: + print(f""" +Test: [{item['testname']}] + File path: + {item['blacklistPath']} + Partial match found in file: {item['matchText']} + Existing platforms for {item['matchText']}: + {item['existingBlacklist']} + Suggested new platforms: + {item['newBlacklistItems']} + Test Results dashboard: + {item['testCaseDashboardURL']} +""") + + +atexit.register(onExit) + + +clear() # Clear the screen and get ready! + + +class editHelper(): + + def displayModifiedTable(deletedLines: str, addedLines: str) -> None: + """Displays the proposed edits to a blacklist file in a table format.""" + addRemoveTable = PrettyTable( + ["Old Blacklist Entry", "New Blacklist Entry"]) + addRemoveTable.align["Old Blacklist Entry"] = "l" + addRemoveTable.align["New Blacklist Entry"] = "l" + addRemoveTable.add_row([deletedLines.strip(), addedLines.strip()]) + print(f"\n\n Updated Blacklist for [{testname[2]}]:\n{addRemoveTable}") + + def printFailingConfigs(failedPlatforms: list) -> None: + """Displays failing configurations as reported by the database + in a table format.""" + # Prepare the table for display. It's pretty, and informational too! + failingConfigs = PrettyTable(["Host Arch", "Host Compiler", "Host OS", "Host OS Version", + "Target Arch", "Target Compiler", "Target OS", + "Target OS Version"]) + for platform in failedPlatforms: + failingConfigs.add_row(platform) + + print( + f"\nFAILING CONFIGS for {os.path.normpath(os.sep.join(testname))}:\n{failingConfigs}") + + def paintHeader(testname: tuple, blacklistedTestData: dict, failedPlatforms: list = []) -> None: + """Clears the screen and prints information relating + to the current blacklist and test case""" + clear() + print(f"\nOpening {blacklistedTestData['filePath']}") + print(f"Test Case: [{testname[2]}]") + print(f"""\nTestresults dashboard for [\ +{testname[2] if testname[2].find(':') < 0 else testname[2][:testname[2].find(':') + 1]}\ +]:\n +{blacklistedTestData['dashboardURL']}""") + if blacklistedTestData['blacklistSnip']: + print("\nCurrent Blacklist entry:\n=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=") + print(blacklistedTestData['blacklistSnip']) + print("=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=\n") + else: + print(f"\nTest [{testname[2]}] not found in blacklist...\n") + if failedPlatforms: + editHelper.printFailingConfigs(failedPlatforms) + + def checkFailingPlatformSaturation(platforms: list, platformType: str = "") -> bool: + """Returns true of the count of platforms passed is more than 3/5 + of the count list of active platforms of the same OS or platform type. + Platforms passed are assumed to be of the same OS family such + as ['windows-10 msvc-2017', 'windows7-sp1'] and will check against that + type if platformType (such as 'xcb') is not passed + + Example: ubuntu-18.04 is failing, and ubuntu-18.04, rhel-7.4, rhel-7.6, and opensuse-leap + are acive. This would mean only 25% of linux type platforms are failing, so don't use the + general term 'linux' here. + """ + + activePlatformCountofThisType = 0 + + if not platformType: # check against the OS type passed. + try: + # Get the actual type of the platforms passed. + platformType = [OS(x).isOfType for x in OS if OS( + x).normalizedValue == platforms[0].split()[0]][0] + except IndexError: + pass + + for platform in activePlatforms: + try: + # Count how many active platforms of the same type there currently are + if [OS(x).isOfType for x in OS if OS(x).normalizedValue == platform.split()[0] + ][0] == platformType: + activePlatformCountofThisType += 1 + except IndexError: + pass + + else: # Check against the platformType passed. + for platform in activePlatforms: + try: + if platformType in OS.getCanBe(platform.split(" ")[0]): + activePlatformCountofThisType += 1 + except IndexError: + pass + + for platform in platforms.copy(): + if platform not in activePlatforms: + platforms.pop(platforms.index(platform)) + + if activePlatformCountofThisType and float(len(platforms) / + activePlatformCountofThisType) >= 0.6: + return True + else: + return False + + def locateBlacklist(blacklistPath: str, testnameTuple: tuple) -> dict: + """Open the BLACKLIST file and try to locate the test in question. + Returns a dict object with data relating to the test case.""" + + returnObject = { + "filePath": "", + "fileExists": False, + "startPos": None, + "endPos": None, + "blacklistSnip": "", + "partialMatch": False, + "matchText": "", + "notFound": False, + "AdditionalLinesToKeep": set(), + "dashboardURL": "" + } + + returnObject["dashboardURL"] = f"\ +https://testresults.qt.io/grafana/d/000000009/coin-single\ +-test-details?orgId=1&var-project={testnameTuple[0]}&var-\ +testcase={testnameTuple[1]}&var-testfunction\ +={testnameTuple[2] if testnameTuple[2].find(':') < 0 else testnameTuple[2][:testnameTuple[2].find(':') + 1] + ']'}\ +&var-branch=dev&var-inter=24h&from=now-60d&to=now" + + # Our initial path begins with 'qt/' and ends with the testname. Drop that and add BLACKLIST + # Raw blacklist path appears as "qt/[module]/tests/auto/[testname]/[testCase]" + path = Path(args.qt5dir, os.sep.join( + Path(blacklistPath).parts[1:-1]), "BLACKLIST") + # Some tests are run from a "/test/" subdirectory, but the blacklist is in the main + # directory, one level up. + if not path.exists() and f"{os.sep}test{os.sep}" in str(path): + path = Path(os.sep.join(path.parts[:-2]), "BLACKLIST") + # Clean up the path in case there are any non-uniform path separators. + path = os.path.normpath(path) + print(f"opening {path}\n") + print(f"Searching for test: [{testnameTuple[2]}]...") + if not os.path.exists(path): + returnObject["filePath"] = path + returnObject["fileExists"] = False + print("BLACKLIST File does not exist...") + return returnObject + + returnObject["filePath"] = path + returnObject["fileExists"] = True + + with open(path, mode="r", newline='') as blacklist: + blacklistRaw = blacklist.read() + # Locate the test name and get the bounds up until the next test item. + testnameLoc = blacklistRaw.find(f"[{testnameTuple[2]}]") + if testnameLoc < 0: + print(f"Test [{testnameTuple[2]}] not found in blacklist...\n") + returnObject["notFound"] = True + # Try to find a match with a sub-test and save it for manual review. + testnameLoc = blacklistRaw.find(f"[{testnameTuple[2]}:") + if testnameLoc >= 0: + endOfLine = blacklistRaw.find("\n", testnameLoc) + returnObject["partialMatch"] = True + returnObject["startPos"] = testnameLoc + returnObject["matchText"] = blacklistRaw[testnameLoc: endOfLine] + print( + f"Partial test match in blacklist: \ +'{blacklistRaw[testnameLoc: endOfLine]}'") + else: + returnObject["notFound"] = True + else: + returnObject["startPos"] = testnameLoc + endOfLine = blacklistRaw.find("\n", testnameLoc) + returnObject["matchText"] = blacklistRaw[testnameLoc: endOfLine] + + if not returnObject["notFound"] or returnObject["partialMatch"]: + # Look for the position of the next test name, or return -1 (end of file) + # if our test is the last test. + returnObject["endPos"] = blacklistRaw.find("[", endOfLine) if blacklistRaw.find( + "[", endOfLine) > 0 else len(blacklistRaw) - 1 + returnObject["blacklistSnip"] = blacklistRaw[returnObject["startPos"]: returnObject["endPos"]] + # Find comments to keep and add them to the list. + for line in returnObject["blacklistSnip"].splitlines(): + if line.strip().startswith('#'): + returnObject["AdditionalLinesToKeep"].add(line.strip()) + + return returnObject + + def generateNewBlacklist(blacklistedTestData: dict, failedPlatforms: list) -> set: + """Use the target OS and compiler versions to write up a list of properly formatted + blacklist entries.\nThis does not preserve the existing list.""" + newBlacklist = set() + + if blacklistedTestData["AdditionalLinesToKeep"]: + newBlacklist.update(blacklistedTestData["AdditionalLinesToKeep"]) + for target in failedPlatforms: + if target[PlatformData.target_os_version.value] == OS.Windows_10.name: + newBlacklist.add( + f"{OS[target[PlatformData.target_os_version.value]].normalizedValue} \ +{COMPILER[target[PlatformData.target_compiler.value]].value}") + else: + newBlacklist.add( + OS[target[PlatformData.target_os_version.value]].normalizedValue) + return sorted(newBlacklist) + + def deleteLines(blacklistedTestData: dict, preserveFile: bool, dryRun: bool) -> str: + """Delete the old blacklist entry from the file. If the deleted + entry was the only entry in the file and deleteLines was not told to keep + the BLACKLIST file, it will be deleted.\n + Set 'dryRun' if no changes should be made.\n + Returns a snip of what was cut out from the file, or what would + be if dryRun is set.""" + existingBlacklistData = "" + delete = False + if blacklistedTestData["startPos"] == 0 and blacklistedTestData["endPos"] is None: + delete = True # The given test spans the whole file. Delete it. + with open(blacklistedTestData["filePath"], mode="r+", newline='') as blacklist: + existingBlacklistData = blacklist.read() + else: # Snip out the test and rewrite the file. + with open(blacklistedTestData["filePath"], mode="r+", newline='') as blacklist: + existingBlacklistData = blacklist.read() + if not blacklistedTestData["startPos"] == 0: + beforeTestText = existingBlacklistData[0: + blacklistedTestData["startPos"]] + else: + beforeTestText = "" + if blacklistedTestData["endPos"] and not (blacklistedTestData["endPos"] >= + len(existingBlacklistData)): + afterTestText = existingBlacklistData[blacklistedTestData["endPos"]:] + else: + afterTestText = "" + + if len(beforeTestText + afterTestText) < 3: + beforeTestText = "" + afterTestText = "" + delete = True + if not dryRun: + blacklist.seek(0) # Reset position in file. + blacklist.write(beforeTestText + afterTestText) + blacklist.truncate() + + if delete and not preserveFile and not dryRun: + print("Deleted blacklist with 0 entries...") + # Delete the empty blacklist file. We'll create a new one if another test needs adding. + os.remove(blacklistedTestData["filePath"]) + + # Return the snippet of what's being deleted. + return existingBlacklistData[blacklistedTestData['startPos']: blacklistedTestData['endPos']] + + def writeNewEntry(blacklistedTestData: dict, linesToAdd: list, linesToDelete: str) -> dict: + """Delete the old entry (if applicable) and write the new one.\n + Returns a dict of the added and deleted snippets.""" + addedLinesSet = set(linesToAdd) + deletedLinesSet = set() + if linesToDelete: + deletedLinesSet.update(linesToDelete.splitlines()[1:]) + + # Don't rewrite the file if the lines to write are the same as the existing lines + # regardless of ordering. + if addedLinesSet.symmetric_difference(deletedLinesSet): + deletedLines = "" + if linesToDelete: + deletedLines = editHelper.deleteLines( + blacklistedTestData, True, False) + with open(blacklistedTestData["filePath"], mode="r+", newline='') as blacklist: + blacklistRaw = blacklist.read() + blacklist.seek(0) + if blacklistedTestData['startPos'] is None: + startPos = len(blacklistRaw) + else: + startPos = blacklistedTestData['startPos'] + linesToWrite = '' + f'[{testname[2]}]\n' + \ + '\n'.join(linesToAdd) + '\n' + blacklist.write( + blacklistRaw[:startPos] + linesToWrite + blacklistRaw[startPos:]) + blacklist.truncate() + # Return what's being changed. + return {"addedLines": linesToWrite, "deletedLines": deletedLines} + else: + # No change to the file was necessary + return {"addedLines": "", "deletedLines": ""} + + def determineEditRequired(existingItems: list, newItems: list) -> bool: + """Compare the list of new and old items in the blacklist + entry. Determine if there's any changes.""" + if set(existingItems).symmetric_difference(set(newItems)): + return True + else: + return False + + def getEdits(testname: tuple, blacklistedTestData: dict, failedPlatforms: list, + existingBlacklistItems: list, action: str) -> list: + """Ask the user a series of prommpts to generate a new blacklist + and provide feedback to confirm if the new list if correct.""" + success = False + while not success: + # Start the interactive editor + linesToAdd = editEntry( + testname[2], False, failedPlatforms, existingBlacklistItems, action) + # Add comment lines back in at the top. This is slightly destructive + # and may result in a comment relating to a specific platform + # appearing out of order, but it's better than dropping it. + for index, line in enumerate(blacklistedTestData['AdditionalLinesToKeep']): + linesToAdd.insert(index, line) + editHelper.displayModifiedTable( + "\n".join(existingBlacklistItems), "\n".join(linesToAdd)) + usrinput = prompt([ + { + "type": 'confirm', + 'message': "Is the new blacklist correct?", + "name": "confirm" + } + ]) + if usrinput['confirm']: + success = True + usrinput = prompt( + [{ + "type": 'confirm', + 'message': "Do you wish to perform any manual edits?", + "name": "confirm", + "default": False + }] + ) + if usrinput['confirm']: + success = False + usrinput = prompt([ + { + "type": 'editor', + 'message': "Manually edit the new entries for the blacklist.", + "name": "editor", + "default": "\n".join(linesToAdd), + "eargs": { + "editor": "default", + "ext": ".txt" + } + } + ]) + linesToAdd = usrinput['editor'].splitlines() + editHelper.displayModifiedTable( + "\n".join(existingBlacklistItems), "\n".join(linesToAdd)) + usrinput = prompt([ + { + "type": 'confirm', + 'message': "Is the new blacklist correct?", + "name": "confirm" + } + ]) + if usrinput['confirm']: + success = True + + if not success: + print("Resetting editor...") + time.sleep(1) + clear() + editHelper.paintHeader( + testname, blacklistedTestData, failedPlatforms) + return linesToAdd + + +def getActionToPerform(testname: str, blacklistedTestName: str, + hasFailures: bool, notInBlacklist: bool) -> str: + """Prompt the user for an appropriate action to take for a given test.\n + Options available change based on the context of the test in question.""" + questions = list() + # Set a bool if the found testname and the original search name are the same + isFullMatch = testname == f"[{blacklistedTestName}]" + message = "" + + if not notInBlacklist: + if isFullMatch: + message = f"Select the action to take on {testname}" + else: + message = f"""{testname} is a partial match +{f', but [{blacklistedTestName}] has 0 failures...' if not hasFailures else '.'} +What should we do?""" + questions.append( + { + 'type': 'list', + 'name': 'action', + 'message': message, + 'default': 'edit', + 'choices': [ + { + 'name': 'Edit existing', + 'value': 'edit' + } + ] + } + ) + + if hasFailures and not isFullMatch: + questions[0]['choices'].append( + { + 'name': f"Replace with [{testname[1:testname.find(':')]}]", + 'value': 'replace' + } + ) + else: + questions[0]['choices'].append( + { + 'name': 'Delete entry', + 'value': 'delete' + } + ) + + questions[0]['choices'].append( + { + 'name': f'Abort / Skip', + 'value': 'abort' + } + ) + + elif notInBlacklist and hasFailures: + questions.append( + { + 'type': 'list', + 'name': 'action', + 'message': f"{testname} is not in the existing blacklist. What should we do?", + 'default': 'abort', + 'choices': [ + { + 'name': f'Abort / Skip', + 'value': 'abort' + }, + { + 'name': f'Add [{testname}]', + 'value': 'add' + } + ] + } + ) + else: + return 'edit' + + # Abort is always available. + + answers = prompt(questions, style=style) + return (answers["action"]) + + +def editEntry(testname: str, isPartialMatch: bool, failedPlatformsRaw: list, + alreadyBlacklisted: list, action: str) -> list: + """Present the user with a series of prompts that make blacklisting and whitelisting + suggestions.\n + Failing Platform Saturation is tested against activePlatforms. If >60% of the active platforms + of a given type are failing, the general platform term will be used instead. Whitelist + suggestions will be made for the remaining acive, but passing platforms in this case.""" + + # There are a lot of lambda function here that generate or filter down lists. + # Often, the purpose is looking at the list of options to present to the user, + # but removing "Separator" objects from the list that would otherwise cause + # exceptions when examining the list data. + + # Other lambdas generate lists of related items from platformEnums.py, + # looking at various relationships between platform targets, os families + # and how a given target relates to general platform terms. + + print(f"\nEntry {action} mode for {testname}") + + allPlatforms = list() + failedPlatforms = list() + relatedPlatforms = set() + markedAsCIFlaky = False + platformCollection = dict() + whitelistPreChecked = dict() + + # Build a checkbox list for failed platforms. + # Pre-tick the options that are already in the blacklist. + if failedPlatformsRaw: + for platform in failedPlatformsRaw: + if platform[-1] == OS.Windows_10.name: + newitem = { + 'checked': True, 'name': f"{OS[platform[-1]].normalizedValue} \ +{COMPILER[platform[-3]].value}"} + else: + newitem = {'checked': True, + 'name': OS[platform[-1]].normalizedValue} + + if newitem not in failedPlatforms: + failedPlatforms.append(newitem) + + generalPlatform = OS[platform[-1]].osFamily + if generalPlatform in [PLATFORM(x).normalizedValue for x in PLATFORM]: + if generalPlatform not in platformCollection: + platformCollection[generalPlatform] = set() + platformCollection[generalPlatform].add( + f"{OS[platform[-1]].normalizedValue} {COMPILER[platform[-3]].value}" + if platform[-1] == OS.Windows_10.name else OS[platform[-1]].normalizedValue) + relatedPlatforms.add(generalPlatform) + + # Write a separator with information about general platform names to the list of options. + if alreadyBlacklisted or relatedPlatforms: + failedPlatforms.append(Separator( + '== General platform Types:==\n See https://doc.qt.io/qt-5/qguiapplication.html#\ +platformName-prop')) + + # Build the general platforms list to present in the first prompt. Avoid duplicates + # Since we're looking at platform names like 'osx', 'windows', 'rhel' + for item in alreadyBlacklisted: + # Set a flag if the existing line contains 'ci' such as "macos-10.12 ci" + # Use this flag later to pop ci back onto edited entries. + if re.search(r'\bci\b', item): + markedAsCIFlaky = True + item = item.split(" ")[0] + if item not in failedPlatforms and item in [PLATFORM(x).normalizedValue for x in PLATFORM]: + relatedPlatforms.add(item) + + # Add the general platform names if it doesn't already exist in the first half + # of the list (already blacklisted). + # If an item passes the failing platforms saturation test, pre-check it. + for item in relatedPlatforms: + if item not in [ + x for x in filter(lambda y: type(y) != + Separator, failedPlatforms) if x['name'] == item + ]: + if item == "*" and not editHelper.checkFailingPlatformSaturation( + [x['name'] for x in filter(lambda y: type(y) != + Separator, failedPlatforms) + ], "*"): + failedPlatforms.append({'checked': False, 'name': item}) + else: + failedPlatforms.append({'checked': True, 'name': item}) + + # Run the saturation test on all other targets to determine which + # general platform names we should use instead of blacklisting individual targets. + for item in platformCollection: + useGlobalPlatformTerm = editHelper.checkFailingPlatformSaturation( + list(platformCollection[item]), item) + index = None + try: + index = failedPlatforms.index([x for x in filter(lambda y: type( + y) != Separator, failedPlatforms) if x['name'] == item][0]) + except IndexError: + print(f"WARN: {item} not in list of Failed Platforms") + # Generally shouldn't happen, as any platform in platformCollection + # Should theoretically be in the failedPlatforms list. + continue + if useGlobalPlatformTerm: + failedPlatforms[index]['checked'] = True + else: + failedPlatforms[index]['checked'] = False + + # Look at each of the failed platforms and determine if + # a majority of that platform failed. If so, add the + # platform family name / type to the list of options + # and check it. + tempFailedPlatforms = [x['name'] for x in filter( + lambda y: type(y) != Separator, failedPlatforms)] + for platformType in set([OS.getType(x) for x in tempFailedPlatforms]): + # Filter the list of platforms to pass down to ones of the same type. + checked = False + index = None + try: + index = failedPlatforms.index([x for x in filter(lambda y: type( + y) != Separator, failedPlatforms) if x['name'] == item][0]) + except IndexError: + pass + + if editHelper.checkFailingPlatformSaturation( + [x for x in + filter(lambda y: OS.getType(y) == + platformType, tempFailedPlatforms) + ], platformType): + checked = True + # What platforms are in the active platforms of this type + # but have not failed? Save this for later so we can + # auto-check whitelist options. + if not whitelistPreChecked.get(platformType, []): + whitelistPreChecked[platformType] = list() + tempSet = set( + [x for x in filter(lambda y: OS.getType(y) == + platformType, activePlatforms) + ] + ).difference([ + x for x in filter(lambda y: OS.getType(y) == platformType, + tempFailedPlatforms) + ]) + if tempSet: + whitelistPreChecked[platformType].extend(list(tempSet)) + + # Check off msvc compilers in whitelist choices if windows 10 was a failed platform. + if OS.Windows_10.normalizedValue in tempFailedPlatforms and (platformType in [ + OS.Windows_10.normalizedValue, + PLATFORM.WINDOWS.normalizedValue + ]): + # Search through the raw list of failed platforms. The target compiler exists at + # index -3, and the target OS version at index -1 + failedWin10Compilers = set([COMPILER.getNormalizedValue( + x[-3]) for x in filter(lambda y: OS.Windows_10.name == y[-1], failedPlatformsRaw)]) + if not whitelistPreChecked.get(platformType, []): + whitelistPreChecked[platformType] = list() + + # Gather the list of compilers (x), and check to see which ones are currently + # active in the CI (y), but filter the active list down to only MSVC compilers (z). + # Get the difference of the sets, returning only a set of passing compilers + # which are active in the CI. + passingWin10Compilers = set([x.value for x in COMPILER if [ + x.value for y in filter(lambda z: 'msvc' in z, activePlatforms) + if x.value in y] + ] + ).difference(failedWin10Compilers) + whitelistPreChecked[platformType].extend(passingWin10Compilers) + + if index: + failedPlatforms[index]['checked'] = checked + else: + failedPlatforms.append({'name': platformType, 'checked': checked}) + + # build a checkbox list for all possible platform configs. + for key in OS: + checked = False + if key == OS.Windows_10: + for compiler in COMPILER: + if compiler.value.lower().startswith('msvc'): + allPlatforms.append( + {'checked': checked, 'name': f"{key.normalizedValue} {compiler.value}"}) + else: + allPlatforms.append( + {'checked': checked, 'name': key.normalizedValue}) + + if alreadyBlacklisted: + allPlatforms.append(Separator( + '== General platform Types:==\n See https://doc.qt.io/qt-5/qguiapplication.html#\ +platformName-prop') + ) + + for key in [PLATFORM(x).normalizedValue for x in PLATFORM]: + allPlatforms.append({'checked': False, 'name': key}) + + # Get ready to show the first prompt. + # This prompt will show platforms that are already in the blacklist, + # any new failed platforms, and general platform name suggestions. + firstAnswers = list() + + # Only present the prompt if there were any new failed platforms. + # It's possible we're in edit mode without any failures, in the case + # that the user is adding a new test or editing one that the database + # reported all-passing. + if failedPlatforms: + questions = [ + { + 'type': 'checkbox', + 'name': 'platformEdit', + 'message': '[BLACKLIST] The below have failed at least once in the past 60 days. \ +Select any to add to the blacklist. (All pre-selected by default)', + 'choices': failedPlatforms + } + ] + firstAnswers = prompt(questions, style=style) + + # Clear the list of related platforms and re-add only the ones that the user selected. + relatedPlatforms.clear() + for answer in firstAnswers['platformEdit']: + if answer in [PLATFORM(x).normalizedValue for x in PLATFORM]: + relatedPlatforms.add(answer) + + print("\n") # Visual spacer in-between prompts. + + keywordDisabledItems = list() + # Tick checkbox in allPlatforms for platforms that were selected in the first prompt. + for index, platform in enumerate(allPlatforms): + if type(platform) == Separator: + continue + + # The following block checks the selected list of platforms and disables + # specific entries that are covered by a general term such as 'osx' or + # 'xcb'. This avoids needing to manually uncheck platforms in the list + # in order to avoid redundant blacklisting. + tempOSEnumIs = None + + # The line below will return the list of "canBe" from the OS enum if + # the item exists in OS and was selected. If a value is passed in + # "platform" that isn't in OS, None will be returned. + tempOSEnum = [OS(x).canBe for x in OS if x.normalizedValue in platform['name'] + or platform['name'] in x.osFamily or platform['name'] in x.isOfType] + + # If not None or empty, check to see if the selected platform's canBe list + # contains a keyword from the list of selected related platforms. + # If it is, deselect the item so only the general platform keyword is selected. + if tempOSEnum: + tempOSEnumIs = [x for x in tempOSEnum[0] + if x in relatedPlatforms] + + if platform['name'] in firstAnswers['platformEdit'] and not tempOSEnumIs and platform['name'] != '*': + platform['checked'] = True + elif tempOSEnumIs or (platform['name'] == '*' and '*' in relatedPlatforms): + platform['disabled'] = f"Already selected for blacklisting by keyword \ +'{tempOSEnumIs[0] if tempOSEnumIs else '*'}'" + # Keep a list of indexes we mark as disabled. + keywordDisabledItems.append(index) + else: + platform['checked'] = False + + # Second prompt. Asks to add any additional platforms from the list of all possible + # blacklist options. + blacklistAnswers = list() + + # Only prompt for additional platforms if '*' was not selected. + if '*' not in relatedPlatforms: + + questions = [ + { + 'type': 'checkbox', + 'name': 'allPlatforms', + 'message': f'[BLACKLIST] Check any {"additional " if failedPlatforms else ""}\ +platforms to add.', + 'choices': allPlatforms + } + ] + + # Ask the prompt. + # Includes a quick conversion to set and back to list to strip out duplicates + blacklistAnswers = list( + set(prompt(questions, style=style)['allPlatforms'])) + + # Add disabled platform blacklist choices back into the list of blacklist answers + try: + # Find the separator in the list if there is one, start looking at + # platforms after that index. + sepIndex = allPlatforms.index( + [x for x in allPlatforms if type(x) == Separator][0]) + except ValueError or IndexError: + sepIndex = 0 + # Make a set of the disabled platform choices + tempDisabledGeneralPlatforms = set() + for item in allPlatforms[sepIndex + 1:]: + if item.get('disabled', None): + tempDisabledGeneralPlatforms.add(item['name']) + blacklistAnswers.extend(tempDisabledGeneralPlatforms) + else: + blacklistAnswers = ['*'] + + # Reset general platforms to ask about in the whitelist. + generalPlatforms = set() + + for blindex, item in enumerate(blacklistAnswers): + # Filter out unselectable separator choices from the allPlatforms + # list and search for the dict item that was selected in + # blacklistAnswers. Disable those items so they cannot be selected + # when whitelisting. + + # TODO: Is this redundant now that the whitelist options get filtered down anyway??? + + for index, platform in enumerate(allPlatforms): + if type(platform) == Separator: + continue + if platform["name"] == item: + allPlatforms[index]["disabled"] = "Already selected for blacklisting" + # Only ask for whitelisting if a blanket platform type is selected. + if item in [PLATFORM(x).normalizedValue for x in PLATFORM]: + generalPlatforms.add((item, blindex)) + + # Un-disable the items that would be blacklisted by the general platform keyword + # So they can be selected in the whitelist. + for index in keywordDisabledItems: + if allPlatforms[index]['name'] not in [x[0] for x in generalPlatforms]: + allPlatforms[index]["disabled"] = False + + # remove options from the list that are not of the same family. + for answer in blacklistAnswers.copy(): + if not PLATFORM.getIsRootType(answer) and [ + True for x in filter(lambda y: + PLATFORM.getFamily(y) == PLATFORM.getFamily( + answer), blacklistAnswers + ) + if PLATFORM.getIsRootType(x) is True + ]: + blacklistAnswers.pop(blacklistAnswers.index(answer)) + + for index, blAnswer in enumerate(blacklistAnswers): + whitelistChoices = set() + + # Get the basic list of whitelist choices based on explicitly related platforms to blAnswer + tempList = PLATFORM.getCanBe(blAnswer) + if tempList: + for choice in tempList: + # Don't add self. i.e. Don't add 'ubuntu' if 'ubuntu' is being blacklisted. + if choice not in ["*", blAnswer, f"{OS.getFamily(blAnswer)}", + f"{OS.getType(blAnswer)}"]: + whitelistChoices.add(choice) + + # Build the list of other related platforms and OSes that + # would be blacklisted by blAnswer. Duplicates are okay + # and will be stripped out later. + tempList = list() + tempList.extend(OS.getCanBe(blAnswer)) + tempList.extend(OS.getFamilyMembers(blAnswer)) + for member in OS.getTypeMembers(blAnswer): + tempList.extend([member, OS.getFamily(member)]) + + for choice in tempList: + if choice not in ["*", blAnswer, f"{OS.getFamily(blAnswer)}", + f"{OS.getType(blAnswer)}"]: + whitelistChoices.add(choice) + + # Add msvc options to the whitelist for windows 10. + if ('windows' in blAnswer and 'windows-10' in [x['name'] for x in + filter(lambda y: type(y) != Separator, + failedPlatforms) + ]) or blAnswer == '*': + for compiler in [COMPILER(x) for x in COMPILER if 'msvc' in COMPILER(x).value]: + whitelistChoices.add(compiler.value) + + whitelistChoicesFormatted = [{'name': x} for x in whitelistChoices] + + # Pre-check choices that would be blacklisted by a general term + # but have not failed recently and are active platforms in the CI. + for choice in whitelistPreChecked.get(blAnswer, []): + if PLATFORM.getIsRootType(blAnswer): + if OS.getFamily(choice) not in [OS.getFamily(x) for x in blacklistAnswers]: + whitelistChoicesFormatted[whitelistChoicesFormatted.index( + { + 'name': OS.getFamily(choice) + } + )] = { + 'name': OS.getFamily(choice), 'checked': True + } + else: + if choice in [x for x in whitelistChoices]: + whitelistChoicesFormatted[whitelistChoicesFormatted.index( + { + 'name': choice + } + )] = { + 'name': choice, 'checked': True + } + else: + if choice in [x for x in whitelistChoices]: + whitelistChoicesFormatted[whitelistChoicesFormatted.index( + { + 'name': choice + } + )] = { + 'name': choice, 'checked': True + } + + # Prompt the user for whitelist options for this blacklisted + # answer if there are any possible combinations. + if whitelistChoices: + questions = [ + { + 'type': 'checkbox', + 'name': 'exceptions', + 'message': f'[WHITELIST] Check any exceptions to \ +add to the WHITELIST for {blAnswer}.', + 'choices': sorted(whitelistChoicesFormatted, key=lambda i: i['name']) + } + ] + + whitelistAnswers = prompt(questions, style=style)['exceptions'] + + # Build up the whitelist on top of any applicable general terms provided. + # See platformEnums::OS for more information about what can be applied + # to which platform terms. + for item in whitelistAnswers: + if 'windows-10' in item and 'msvc' in item: + for compiler in [ + COMPILER(x) for x in COMPILER if x.value == item.split(' ')[1] + ]: + blacklistAnswers[index] = f"{blacklistAnswers[index]} !{compiler.value}" + else: + blacklistAnswers[index] = f"{blacklistAnswers[index]} !{item}" + + # Prompt the user for which options to mark with 'ci' if any existing + # blacklist items were marked with 'ci' + # Marking a blacklist line with 'ci' makes the line only take effect + # inside of COIN. This is useful if the test is flaky or failing explicitly + # due to a known COIN bug or infrastructure issue, but passes normally in + # a real-world environment. + if markedAsCIFlaky: + + choices = [{'name': x} for x in blacklistAnswers] + questions = [ + { + 'type': 'checkbox', + 'name': 'markForCI', + 'message': f'[CI ONLY] At least one item in the existing blacklist was marked \ +with \'ci\'. Select any new items to mark with the \'ci\' designation.', + 'choices': choices + } + ] + + flakyAnswers = prompt(questions, style=style)['markForCI'] + + for item in flakyAnswers: + blacklistAnswers[blacklistAnswers.index(item)] = f"{item} ci" + + return blacklistAnswers + + +def appendPartialMatchSkipped(testname: list, blacklistedTestData: dict): + partialMatchesSkipped.append( + { + "blacklistPath": blacklistedTestData["filePath"], + "testname": testname[2], + "matchText": blacklistedTestData["matchText"], + "existingBlacklist": '\n '.join(blacklistedTestData["blacklistSnip"]), + "newBlacklistItems": '\n\ + '.join(editHelper.generateNewBlacklist(blacklistedTestData)), + "testCaseDashboardURL": blacklistedTestData['dashboardURL'] + } + ) + + +def processItem(testname: list, failedPlatforms: list): + + global fastForward # Make this global editable in this scope. + + if fastForward: + # Fast-Forward takes a test name (see arg parsing in main()) + # This skips forward in the results to the selected test if it exists. + print(f"Fast Forwarding to {args.fastForward}...") + if testname[2] == args.fastForward: + clear() + # If we found the test, cancel the fast forward. + fastForward = False + else: + return + + blacklistPath = os.sep.join(testname) + + # Find the current test in the blacklist or touch a new file. + # Return start and end bounds for the test. + blacklistedTestData = editHelper.locateBlacklist(blacklistPath, testname) + + # Abort is the default action in automatic mode. + # This skips the test if a partial match is found. + action = "abort" + haveAction = False + + if not blacklistedTestData: + # Return this iteration if there was a critical error + # such as being unable to touch a new file. + return + elif not blacklistedTestData['fileExists'] or (blacklistedTestData['notFound'] + and not blacklistedTestData['partialMatch']): + editHelper.paintHeader(testname, blacklistedTestData, failedPlatforms) + if args.interactive: + # If in interactive mode, ask the user if the test should be added. + if failedPlatforms: + action = getActionToPerform(testname[2], "", True, True) + haveAction = True + if action == 'abort': + input(f"\nEdit Aborted...\nPress Return to continue...") + clear() + return # Skip this test + elif action == 'add' and not blacklistedTestData['fileExists']: + # Touch the file since it doesn't exist and try to create it. + # This can only occur in interactive mode. + try: + # Initialize a blank file if it doesn't exist. + open( + blacklistedTestData['filePath'], mode="a", newline="") + except FileNotFoundError: + print( + f"Error writing to file at {blacklistedTestData['filePath']}... \ +Is your qt5 repository fully up-to-date?") + input( + f"Press Return to continue. Please update [{testname[2]}] in \ +{blacklistedTestData['filePath']} manually.") + clear() + else: + print( + f"\nDatabase provided no failing platforms for [{testname[2]}]. \ +Nothing to do...") + input(f"\nPress Return to continue...") + clear() + return # Skip this test + else: + # Return this iteration if the blacklist file doesn't exist or + # the test isn't in the blacklist. + return + + existingBlacklistItems = list() + + # Did we find a whole or partial match? Print the current blacklist + # and ask for action if in interactive mode. + if not blacklistedTestData["notFound"] or blacklistedTestData["partialMatch"]: + existingBlacklistItems = blacklistedTestData['blacklistSnip'].splitlines()[ + 1:] + for index, item in enumerate(existingBlacklistItems): + existingBlacklistItems[index] = existingBlacklistItems[index].strip( + ) + existingBlacklistItems = sorted(existingBlacklistItems) + + if args.interactive and blacklistedTestData["partialMatch"]: + # Get the action to perform if we found a partial match since this is a special case. + # The user may wish to update the partial match, delete it and add a new test case, + # or abort and skip the case. + if not haveAction: + editHelper.paintHeader( + testname, blacklistedTestData, failedPlatforms) + action = getActionToPerform( + blacklistedTestData["matchText"], testname[2], + True if failedPlatforms else False, False) + haveAction = True + + if (action == "edit" and blacklistedTestData["partialMatch"]) or action == "delete": + # Update the testname used for both display and editing + # if the user is editing the partial match. + testname = (testname[0], testname[1], + blacklistedTestData["matchText"][1:-1]) + + deletedLines = "" + addedLines = "" + + if failedPlatforms: + # So we have failed platforms. What should be done? + editHelper.paintHeader(testname, blacklistedTestData, failedPlatforms) + if not editHelper.determineEditRequired(existingBlacklistItems.copy(), + editHelper.generateNewBlacklist(blacklistedTestData, + failedPlatforms)): + print(f"\nBlacklist for {testname[2]} is already up-to-date.") + if args.interactive: + usrinput = prompt( + [{ + "type": 'confirm', + 'message': "Force editing?", + "name": "force", + "default": False + }] + ) + if not usrinput['force']: + return # Skip this test. + else: + return # Skip this test. + + if not haveAction: + # So the test wasn't up to date and needs editing? ask what to do. + if args.interactive: + action = getActionToPerform( + blacklistedTestData["matchText"], testname[2], True, False) + else: + if blacklistedTestData["partialMatch"]: + # Never overwrite, replace, or edit partial matches in automatic mode. + # Just log it and let the user know what was skipped. + appendPartialMatchSkipped(testname, blacklistedTestData) + return + else: + action = 'edit' + haveAction = True + + # Initialize the add/delete lists + linesToAdd = list() + linesToDelete = list() + if action in ['edit', 'replace', 'add']: + if args.interactive: + linesToAdd = editHelper.getEdits( + testname, blacklistedTestData, failedPlatforms, existingBlacklistItems, action) + else: + linesToAdd = editHelper.generateNewBlacklist( + blacklistedTestData) + elif action == "abort": + if blacklistedTestData["partialMatch"]: + # Just log the partial match and let the user know what was skipped. + appendPartialMatchSkipped(testname, blacklistedTestData) + print("\nNothing modified...") + if args.interactive: + input("Press Return to continue...") + clear() # Clear the screen after each test + return + + if (not blacklistedTestData["notFound"] + or blacklistedTestData["partialMatch"]) and action != "delete": + # Dry run the delete to see what we're deleting. It will be executed later. + linesToDelete = editHelper.deleteLines( + blacklistedTestData, True, True) + # Execute the edits. + result = editHelper.writeNewEntry( + blacklistedTestData, linesToAdd, linesToDelete) + addedLines = result['addedLines'] + deletedLines = result['deletedLines'] + + # Occurs when a user wants to edit a partial match with no failed platforms. + elif args.interactive and action == 'edit': + linesToAdd = editHelper.getEdits( + testname, blacklistedTestData, failedPlatforms, existingBlacklistItems, action) + linesToDelete = editHelper.deleteLines(blacklistedTestData, True, True) + result = editHelper.writeNewEntry( + blacklistedTestData, linesToAdd, linesToDelete) + addedLines = result.get['addedLines'] + deletedLines = result['deletedLines'] + + # Delete partial matches too, since the database doesn't track individual test cases, + # just function names and if the test function is reported as only pass, the specific test + # cases must have also passed. Don't delete it if the user selected edit in interactive mode. + elif not blacklistedTestData["notFound"] or blacklistedTestData["partialMatch"]: + action = "delete" + + if action == "delete": + print( + f"\nRemoving blacklisted test {testname[2]}\ +{'...' if failedPlatforms else ' with 0 failing configurations...'}") + deletedLines = editHelper.deleteLines( + blacklistedTestData, False, False) + + # Display a table with the changes. + if addedLines or deletedLines: + if not args.interactive: + editHelper.displayModifiedTable(deletedLines, addedLines) + + # Save the file path since we modified it. + modifiedFiles.add(blacklistedTestData["filePath"]) + + if os.path.exists(blacklistedTestData["filePath"]): + with open(blacklistedTestData["filePath"], newline='') as blacklist: + print( + f"\nNew Blacklist file:\n=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=\n\ +{blacklist.read()}=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=\n") + + else: + print("\nNothing modified...") + if args.interactive: + input("Press Return to continue...") + clear() # Clear the screen after each test + + +def getInfluxClient() -> InfluxDBClient: + client = InfluxDBClient( + host=INFLUX_DB_URL, + port=INFLUX_DB_PORT, + ssl=True, + verify_ssl=True, + username=os.environ.get("INFLUX_DB_USER") if os.environ.get( + "INFLUX_DB_USER") else "", + password=os.environ.get("INFLUX_DB_PASSWORD") if os.environ.get( + "INFLUX_DB_PASSWORD") else "", + database="coin" + ) + client._InfluxDBClient__baseurl = "{0}://{1}:{2}/{3}".format( + client._scheme, + client._host, + client._port, + "influxdb" + ) + + return client + + +def doQuery(module: str) -> dict: + """Query the database and put together a dictionary of blacklisted + tests which had at least one failure in the last 60 days.""" + client = getInfluxClient() + + def setClientProp(prop: str) -> None: + """Set the username or password with which to connect to the database""" + if prop == 'username': + os.environ["INFLUX_DB_USER"] = prompt( + [{ + "type": "input", + 'message': f"Influx DB Username:", + "name": "username", + 'default': client._username + }] + )['username'] + client._username = os.environ.get("INFLUX_DB_USER") + else: + os.environ["INFLUX_DB_PASSWORD"] = prompt( + [{ + "type": "password", + 'message': f"Influx DB Password:", + "name": "password" + }] + )['password'] + client._password = os.environ.get("INFLUX_DB_PASSWORD") + + def showDBKeysFields(message: str) -> bool: + """Print out the list of field names and tag names in the coin database + This assists with modifying the query if the user is not explicitly + familiar with the database structure.""" + if prompt( + [{ + "type": "confirm", + 'message': f"{message} Show tags/fields before editing?:", + "name": "help", + 'default': False + }] + )['help']: + clear() + newlinesep = '\n' # Workaround for not allowing '\' inside f-strings + print( + f"""TAG KEYS:\n{ + newlinesep.join([point['tagKey'] for point in + client.query('SHOW TAG KEYS from blacklisted_test').get_points()]) + }\n""") + print( + f"""FIELD KEYS:\n{ + newlinesep.join([point['fieldKey'] for point in + client.query('SHOW field KEYS from blacklisted_test').get_points()]) + }\n""") + return True + else: + return False + + if args.interactive: + # Try the pre-set user and password if they're in environment variables + if os.environ.get("INFLUX_DB_USER") and os.environ.get("INFLUX_DB_PASSWORD"): + try: + # Dummy query to check credentials. Verify read permisison. + client.query("SHOW FIELD KEYS FROM integrations") + success = True + print("Username OK...\nPassword OK...") + except exceptions.InfluxDBClientError: + print( + "Environment variable Username or password incorrect. \ +Please re-enter your credentials...") + success = False + else: + success = False + + while not success: + setClientProp('username') + setClientProp('password') + try: + # Dummy query to check credentials. Verify read permisison. + client.query("SHOW FIELD KEYS FROM integrations") + success = True + except exceptions.InfluxDBClientError as e: + print( + "Username or password incorrect. Please re-enter your credentials...") + print(e) + time.sleep(2) + clear() + + # Get the full list of blacklisted tests that had any passes at all in the last 7 days. + + selectString = "SELECT project, testCase, testFunction, id FROM blacklisted_test " + moduleString = f"and project='qt/{module}' " if module != 'qt5' else '' + whereString = f"WHERE result = 'Passed' and branch = 'dev' {moduleString}and time > now() - 7d" + + success = False + + while not success: + if args.interactive: + # Allow for editing the query. + whereString = prompt([{"type": "input", 'message': f"Edit WHERE clause:", + "name": "query", 'default': whereString}])['query'] + print("OK...") + try: + blPoints = client.query(selectString + whereString) + success = True + # The query didn't return anything in the generator. Maybe the query was bad. + if not next(blPoints.get_points(), None): + if showDBKeysFields("Query returned 0 results."): + success = False + else: + success = False + clear() + except exceptions.InfluxDBClientError as e: + showDBKeysFields( + f"\nError while running query: {e}Please modify the query and try again...\n") + success = False + else: + blPoints = client.query(selectString + whereString) + success = True + + # Generate a dictionary of the testnames, each with an empty dict. + tests = {} + for point in blPoints.get_points(): + tests[(point["project"], point["testCase"], point["testFunction"])] = {} + + # Query for all executed configurations that had at least one failure in the last 60 days. + for test in tests: + # The whitespace line below lets us use carriage return and overwrite the current line + # for each test name being processed. Getting the actual console window width is not + # lightweight or pretty, so this works fine without much risk of garbage being displayed. + print("\r \ + ", end="") + print( + f"\rProcessing blacklisted test: \ +\"{os.path.normpath(os.sep.join(test).strip())}\"", end="") + queryForFail = f"SELECT id, host_arch, host_compiler, host_os, host_os_version, \ +target_arch, target_compiler, target_os, target_os_version FROM blacklisted_test WHERE \ +project = '{test[0]}' and testCase = '{test[1]}' and testFunction = '{test[2]}' \ +and branch = 'dev' and result = 'Failed' and time> now()-60d" + + failures = client.query(queryForFail) + + failedPlatforms = {} + # Verify that the query returned at least one point (one failed configuration). + if next(failures.get_points(), None) is not None: + for point in failures.get_points(): + if test not in failedPlatforms: + # Make the test name a set object so it will be unique. + # This seems redundant at the moment because the configurations are + # addressed as tests[testname][testname] in __main__ + # Maybe it can be fixed elegantly. + failedPlatforms[test] = set() + # Add the configuration to the set object. + # If it's an exact duplicate it will be ignored. + failedPlatforms[test].add( + (point["host_arch"], point["host_compiler"], point["host_os"], + point["host_os_version"], point["target_arch"], point["target_compiler"], + point["target_os"], point["target_os_version"]) + ) + tests[test] = failedPlatforms + else: + # The query returned 0 points. + # Set the object in tests to None so we can safely check for it later. + tests[test] = None + + print("\nDone...") + if args.interactive: + # Cosmetic sleep for the UI # Everyone needs their beauty rest! + time.sleep(2) + return tests + + +def getActivePlatforms() -> list: + """Runs a query on the database to gether a list of recently + run targets. This list can be used to understand what platforms + are currently active in the CI.""" + + client = getInfluxClient() + + result = client.query( + "SELECT id, target_os, target_os_version, target_compiler FROM workitem where branch = \ +'dev' and time >= now()-7d GROUP BY target_os, target_os_version, target_compiler") + + activeTargets = set() + # Create a unique set of the recently run platforms + for point in result.get_points(): + activeTargets.add( + (point['target_os'], point['target_os_version'], point['target_compiler'])) + + friendlyTargetNames = set() + ignoredPlatforms = set() + for target in activeTargets: + try: + if target[1] == OS.Windows_10.name: + friendlyTargetNames.add( + f"{OS[target[1]].normalizedValue} {COMPILER[target[2]].value}") + else: + friendlyTargetNames.add(OS[target[1]].normalizedValue) + except KeyError: + # A platform returned by the database that recently reported data + # is "active", but we don't care about it because it's not in our + # enums for platforms that run tests and can be blacklisted. + # This is to be expected for any platform that has tests disabled + # on all configurations, or for a platform that does not report + # itself in a uniquely identifiable way to the blacklister. + ignoredPlatforms.add( + target[1] if not target[1] == OS.Windows_10.name else f'{target[1]} {target[2]}') + + if ignoredPlatforms: + printableIgnoreList = '\n '.join(sorted(ignoredPlatforms)) + print(f""" +WARN: The following platforms are not present in platformEnums.py, + but have recently run workitems in the CI. If these platforms + are not running tests, this message can be safely ignored. + Otherwise, platformEnums.py may need to be updated. + + =-=-=-=-=-=-=-=-=-=-=- + {printableIgnoreList} + =-=-=-=-=-=-=-=-=-=-=- + +""") + if args.interactive: + input("Press return to continue...") + return list(friendlyTargetNames) + + +def validateQt5Dir() -> dict: + args.qt5dir = os.path.normpath(args.qt5dir) + + if not args.qt5dir.endswith(os.sep): + args.qt5dir = args.qt5dir + os.sep + + if not os.path.exists(args.qt5dir): + return({'exists': False, 'module': None}) + else: + module = args.qt5dir.split(os.sep)[-2] + if not module == 'qt5': + # Strip off the module since the test returns it from the database + args.qt5dir = args.qt5dir[:args.qt5dir[:-1].rfind(os.sep) + 1] + return({'exists': True, 'module': module}) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument('--interactive', '-i', action='store_true', dest="interactive", + help="Set --interactive or -i to confirm changes and edit entries.") + parser.add_argument('--qt5dir', dest='qt5dir', type=str, required=True, + help='The full path to a checked-out qt5 supermodule or a single submodule') + parser.add_argument('--fastForward', dest='fastForward', + type=str, help='Test Case name to Fast Forward to.') + args = parser.parse_args() + + if args.fastForward: + fastForward = True + + moduleValidation = validateQt5Dir() + if not moduleValidation['exists']: + print( + f"Path to qt5 or qt5 submodule does not exist. \ +Please verify that the path is correct: {args.qt5dir}") + exit(0) + + # Gather the list of blacklisted tests and their failed configurations from the last 60 days. + tests = doQuery(moduleValidation['module']) + # Query the most recent integration and see which platforms are currently active in COIN. + # This is a global that gets used any time we check a given platform type's failing percentage. + # See editHelper.checkFailingPlatformSaturation() + activePlatforms = getActivePlatforms() + for testname in tests: + clear() + # The test had no failures. See about removing it from the blacklist entirely. + if tests.get(testname) is None: + processItem(testname, None) + print("\n\n") + else: + # Update the blacklist configurations with failed platforms. + processItem(testname, tests[testname][testname]) + print("\n\n") diff --git a/scripts/coin/blacklist_tool/platformEnums.py b/scripts/coin/blacklist_tool/platformEnums.py new file mode 100644 index 00000000..66636704 --- /dev/null +++ b/scripts/coin/blacklist_tool/platformEnums.py @@ -0,0 +1,250 @@ +############################################################################ +## +# Copyright (C) 2019 The Qt Company Ltd. +# Contact: https://www.qt.io/licensing/ +## +# This file is part of the Quality Assurance module of the Qt Toolkit. +## +# $QT_BEGIN_LICENSE:GPL-EXCEPT$ +# Commercial License Usage +# Licensees holding valid commercial Qt licenses may use this file in +# accordance with the commercial license agreement provided with the +# Software or, alternatively, in accordance with the terms contained in +# a written agreement between you and The Qt Company. For licensing terms +# and conditions see https://www.qt.io/terms-conditions. For further +# information use the contact form at https://www.qt.io/contact-us. +## +# GNU General Public License Usage +# Alternatively, this file may be used under the terms of the GNU +# General Public License version 3 as published by the Free Software +# Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +# included in the packaging of this file. Please review the following +# information to ensure the GNU General Public License requirements will +# be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +# $QT_END_LICENSE$ +## +############################################################################# + +from enum import Enum + + +class OS(Enum): + """Defines properties of OS types. + Enumeration names are exact matches for the platform targets reported by + the database.\n + Tuple values are as follows, with explanation:\n + [1] OS name/version pair values that are read and accepted by the blacklist. + This is what is written to BLACKLIST files.\n + [2] The family os OSs the target belongs to. This is used when checking how + many of a given OS family are currently failing.\n + [3] "canBe" list. This list describes which platforms apply to a given OS target. + This is used when determining which oses should be included under platform terms + such as "xcb"\n + [4] The general platform term used to describe the OS, such as "linux", "osx", or "windows" + """ + openSUSE_15_0 = ("opensuse-leap", "suse", + ["*", "linux", "xcb", "wayland", "openwfd", "directfb", "minimal"], "linux") + openSUSE_42_3 = ("opensuse-42.3", "suse", + ["*", "linux", "xcb", "wayland", "openwfd", "directfb", "minimal"], "linux") + SLES_15 = ("sles-15.0", "suse", ["*", "linux", "xcb", + "wayland", "openwfd", "directfb", "minimal"], "linux") + SLED_15 = ("sled-15.0", "suse", + ["*", "linux", "xcb", "wayland", "openwfd", "minimal"], "linux") + Ubuntu_16_04 = ("ubuntu-16.04", "ubuntu", ["*", "linux", "ubuntu", + "xcb", "directfb", "wayland", "openwfd", "minimal"], "linux") + Ubuntu_18_04 = ("ubuntu-18.04", "ubuntu", ["*", "linux", "ubuntu", + "xcb", "directfb", "wayland", "openwfd", "minimal"], "linux") + RHEL_6_6 = ("rhel-6.6", "rhel", ["*", "linux", "rhel", "xcb", + "wayland", "directfb", "openwfd", "minimal"], "linux") + RHEL_7_4 = ("rhel-7.4", "rhel", ["*", "linux", "rhel", "xcb", + "wayland", "directfb", "openwfd", "minimal"], "linux") + RHEL_7_6 = ("rhel-7.6", "rhel", ["*", "linux", "rhel", "xcb", + "wayland", "directfb", "openwfd", "minimal"], "linux") + OSX_10_11 = ("osx-10.11", "osx", + ["*", "osx", "cocoa", "directfb", "minimal", "offscreen"], "osx") + MacOS_10_12 = ("osx-10.12", "osx", + ["*", "osx", "cocoa", "directfb", "minimal", "offscreen"], "osx") + MacOS_10_13 = ("osx-10.13", "osx", + ["*", "osx", "cocoa", "directfb", "minimal", "offscreen"], "osx") + MacOS_10_14 = ("osx-10.14", "osx", + ["*", "osx", "cocoa", "directfb", "minimal", "offscreen"], "osx") + Windows_7 = ("windows-7sp1", "windows-7sp1", + ["*", "windows", "windows-7", "kms", "minimal", "offscreen"], "windows") + Windows_10 = ("windows-10", "windows-10", + ["*", "windows", "windows-10", "kms", "minimal", "offscreen"], "windows") + WinRT_10 = ("winrt", "winrt", [ + "*", "windows", "winrt", "kms", "minimal", "offscreen"], "windows") + Android_ANY = ("android", "android", [ + "*", "linuxfb", "eglfs", "directfb", "openwfd", "minimal"], "android") + QEMU = ("b2qt", "b2qt", ["*", "linuxfb", + "eglfs", "directfb", "minimal"], "b2qt") + + def __init__(self, normalizedValue: str, osFamily: str, canBe: list, isOfType: str): + """Make the tuple values named so the can be retrieved with a + simple accessor like OS.RHEL_7_4.normalizedValue""" + self.normalizedValue = normalizedValue + self.osFamily = osFamily + self.canBe = canBe + self.isOfType = isOfType + + @classmethod + def count(cls, typeRequested: str) -> int: + count = 0 + for entry in cls: + if entry.isOfType == typeRequested or typeRequested in entry.normalizedValue: + count += 1 + + return count + + @classmethod + def getFamily(cls, normalizedValue: str) -> str: + for entry in cls: + if entry.normalizedValue == normalizedValue: + return entry.osFamily + return "" + + @classmethod + def getType(cls, normalizedValue: str) -> str: + for entry in cls: + if entry.normalizedValue == normalizedValue: + return entry.isOfType + return "" + + @classmethod + def getCanBe(cls, normalizedValue: str) -> list: + for entry in cls: + if entry.normalizedValue == normalizedValue: + return entry.canBe + return [] + + @classmethod + def getFamilyMembers(cls, familyName: str) -> list: + return [entry.normalizedValue for entry in cls if (entry.osFamily == familyName or + familyName == '*')] + + @classmethod + def getTypeMembers(cls, typeName: str) -> list: + return [entry.normalizedValue for entry in cls if (entry.isOfType == typeName or + typeName == '*')] + + +class COMPILER(Enum): + """Mainly used when determining MSVC compilers + to blacklist.""" + GCC = "gcc" + Clang = "clang" + Mingw73 = "mingw-7.3" + MSVC2015 = "msvc-2015" + MSVC2017 = "msvc-2017" + MSVC2019 = "msvc-2019" + + @classmethod + def getNormalizedValue(cls, requestName: str) -> str: + for entry in cls: + if entry.name == requestName: + return entry.value + elif entry.value == requestName: + return entry.value + return "" + + @classmethod + def isCompiler(cls, requestName: str) -> bool: + for entry in cls: + if requestName == entry.value: + return True + return False + + +class PLATFORM(Enum): + """Defines properties of PLATFORM types. + Tuple values are as follows, with explanation:\n + [1] Platform name values that are read and accepted by the blacklist. + This is what is written to BLACKLIST files.\n + [2] "canBe" list. This list describes which platforms apply to a given OS target. + This is used when determining which oses should be included under platform terms + such as "xcb"\n + [3] Describes the base OS type if the platform itself describes some version or + distribution of an OS.\n + [4] Denotes if the platform type is a base type that cannot be whitelisted, such as linux, + windows, or osx.\n + General platform names that are acceptable in blacklists can be found at + https://doc.qt.io/qt-5/qguiapplication.html#platformName-prop + \n + The canBe values show relations so the tool can blacklist + platforms with exceptions such as "xcb !ubuntu""" + + ALL = ("*", [], "", True) + ANDROID = ("android", ["eglfs", "linuxfb", "directfb", + "minimal", "offscreen", "linux", "*"], "", False) + COCOA = ("cocoa", ["osx", "directfb", "minimal", + "directfb", "offscreen", "*"], "", False) + # QSysInfo::ProductType() returns "osx" for all macOS systems, + # regardless of Apple naming convention + OSX = ("osx", ["directfb", "minimal", "directfb", + "offscreen", "*"], "osx", True) + DIRECTFB = ("directfb", ["osx", "android", "cocoa", + "qnx", "linux", "rhel", "ubuntu", "*"], "", False) + EGLFS = ("eglfs", ["android", "ios", "qnx", "windows", + "windows_10", "linux", "rhel", "ubuntu", "*"], "", False) + IOS = ("ios", ["*"], "ios", True) + KMS = ("kms", ["windows", "windows-10", "*"], "", False) + LINUXFB = ("linuxfb", ["linux", "rhel", "ubuntu", + "windows", "windows-10", "osx", "*"], "", False) + MINIMAL = ("minimal", ["linux", "rhel", "ubuntu", + "windows", "windows-10", "osx", "*"], "", False) + OFFSCREEN = ("offscreen", ["osx", "android", "cocoa", "ios", "qnx", + "windows", "windows_10", "linux", "rhel", "ubuntu", "*"], "", False) + OPENWFD = ("openwfd", ["osx", "android", "cocoa", "ios", "qnx", + "windows", "windows_10", "linux", "rhel", "ubuntu", "*"], "", False) + QNX = ("qnx", ["*"], "", True) + WINDOWS = ("windows", ["kms", "minimal", "*"], "windows", True) + WINDOWS_10 = ("windows-10", ["kms", "windows", + "minimal", "offscreen", "*"], "windows", False) + WAYLAND = ("wayland", ["linux", "rhel", "ubuntu", "*"], "", False) + XCB = ("xcb", ["linux", "rhel", "ubuntu", "*"], "", False) + LINUX = ("linux", ["*", "eglfs", "directfb", "linuxfb", + "offscreen", "minimal", "xcb"], "linux", True) + RHEL = ("rhel", ["linux", "directfb", "eglfs", "linuxfb", + "minimal", "offscreen", "openwfd", "xcb", "*"], "linux", False) + UBUNTU = ("ubuntu", ["linux", "directfb", "eglfs", "linuxfb", + "minimal", "offscreen", "openwfd", "xcb", "*"], "linux", False) + + def __init__(self, normalizedValue: str, canBe: list, osFamily: str, isRootType: bool): + self.normalizedValue = normalizedValue + self.canBe = canBe + self.osFamily = osFamily + self.isRootType = isRootType + + @classmethod + def getNormalizedValue(cls, requestName: str) -> str: + for entry in cls: + if entry.name == requestName: + return entry.normalizedValue + elif entry.normalizedValue == requestName: + return entry.normalizedValue + return "" + + @classmethod + def getCanBe(cls, normalizedValue: str) -> list: + if normalizedValue == '*': + return [x.normalizedValue for x in cls if x.normalizedValue != '*'] + else: + for entry in cls: + if entry.normalizedValue == normalizedValue: + return entry.canBe + return [] + + @classmethod + def getFamily(cls, normalizedValue: str) -> str: + for entry in cls: + if entry.normalizedValue == normalizedValue: + return entry.osFamily + return "" + + @classmethod + def getIsRootType(cls, normalizedValue: str) -> bool: + for entry in cls: + if entry.normalizedValue == normalizedValue: + return entry.isRootType + return False |