diff options
author | Patrik Teivonen <patrik.teivonen@qt.io> | 2022-07-05 14:19:26 +0300 |
---|---|---|
committer | Patrik Teivonen <patrik.teivonen@qt.io> | 2022-09-11 21:39:07 +0000 |
commit | 6eb9237645e0ad29afa699b714840eaf338266e2 (patch) | |
tree | 43f88073f84da62ef56a4527c77efa3ee2a71de8 | |
parent | 2083a9e4f96f710946c9ca679b79f64570741df4 (diff) |
pylint: Add miscellaneous checksv6.4.0-rc1-packaging
Add comment about disabled checks in pylint hook and enable all default pylint checks except for those.
Address these miscellaneous pylint warnings:
-C0123: Use isinstance() rather than type() for a typecheck. (unidiomatic-typecheck)
-R1714: Consider merging these comparisons with "in" (consider-using-in)
-R1722: Consider using sys.exit() (consider-using-sys-exit)
-C0206: Consider iterating with .items() (consider-using-dict-items)
-R1710: All return statements in a function should return an expression. (inconsistent-return-statements)
-W0707: Consider explicitly re-raising using the 'from' keyword (raise-missing-from)
-W1510: Using subprocess.run without explicitly set `check` is not recommended. (subprocess-run-check)
-W0102: Dangerous default value as argument (dangerous-default-value)
-W1508: os.getenv default type is builtins.bool. Expected str or None. (invalid-envvar-default)
-W0212: Access to a protected member of a client class (protected-access)
-W0631: Using possibly undefined loop variable (undefined-loop-variable)
-W1113: Keyword argument before variable positional arguments list in the definition of __init__ function (keyword-arg-before-vararg)
-W0201: Attribute defined outside __init__ (attribute-defined-outside-init)
-W0221: Number of parameters in overridden method differ (arguments-differ)
-C0202: Class method should have 'cls' as first argument (bad-classmethod-argument)
-W1505: Using deprecated method currentThread() (deprecated-method)
-R1707: Disallow trailing comma tuple (trailing-comma-tuple)
-W0603: Using the global statement (global-statement)
-W0602: Using global for 'OUTPUT_STATES' but no assignment is done (global-variable-not-assigned)
-C2801: Unnecessarily calls dunder method __contains__. Use in keyword.
Change-Id: Ia1c6ff3f6c328a94b40a1c0f11c83151abc91423
Reviewed-by: Iikka Eklund <iikka.eklund@qt.io>
-rw-r--r-- | .pre-commit-config.yaml | 10 | ||||
-rw-r--r-- | packaging-tools/bld_openssl.py | 4 | ||||
-rw-r--r-- | packaging-tools/bld_utils.py | 33 | ||||
-rw-r--r-- | packaging-tools/bldinstallercommon.py | 33 | ||||
-rw-r--r-- | packaging-tools/build_clang.py | 3 | ||||
-rw-r--r-- | packaging-tools/build_clang_qdoc.py | 2 | ||||
-rw-r--r-- | packaging-tools/build_wrapper.py | 22 | ||||
-rw-r--r-- | packaging-tools/create_installer.py | 2 | ||||
-rw-r--r-- | packaging-tools/install_qt.py | 3 | ||||
-rw-r--r-- | packaging-tools/libclang_training/run_batch_files.py | 5 | ||||
-rwxr-xr-x | packaging-tools/read_remote_config.py | 10 | ||||
-rwxr-xr-x | packaging-tools/release_repo_updater.py | 4 | ||||
-rwxr-xr-x | packaging-tools/remote_uploader.py | 2 | ||||
-rwxr-xr-x | packaging-tools/runner.py | 3 | ||||
-rw-r--r-- | packaging-tools/sdkcomponent.py | 3 | ||||
-rwxr-xr-x | packaging-tools/sign_windows_installer.py | 4 | ||||
-rw-r--r-- | packaging-tools/tests/test_run_command.py | 10 | ||||
-rw-r--r-- | packaging-tools/tests/testhelpers.py | 2 | ||||
-rw-r--r-- | packaging-tools/threadedwork.py | 11 |
19 files changed, 92 insertions, 74 deletions
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fcc223f8e..c2f52f47c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,15 @@ repos: fail_fast: true - id: pylint name: Analyze code (pylint) - entry: bash -c 'for x in "$@"; do pipenv run python3 -m pylint --errors-only --enable=C0209,W1202,W1203,C0325,R1705,R1720,R1721,R1723,R1724,W0104,W0105,W0107,W0612,W0613,R0205,R1711,R1727,R1719,W1503,W0402,W1514,R1732,W0621,W0622,C0103 "$x"; done' + # Disabled: + # Missing docstrings: C0114,C0115,C0116 + # Line too long: C0301 + # Need refactoring: C0302,R0201,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0916,R1702 + # Duplicate code: R0801 + # TODO comments: W0511 + # Too general exceptions: W0703 + # Checkers removed in some newer versions of pylint: R0022 + entry: bash -c 'for x in "$@"; do pipenv run python3 -m pylint --disable=C0114,C0115,C0116,C0301,C0302,R0201,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0916,R1702,W0511,W0703,R0022 "$x"; done' language: system types: [python] fail_fast: true diff --git a/packaging-tools/bld_openssl.py b/packaging-tools/bld_openssl.py index 0dc6d9132..95b77b781 100644 --- a/packaging-tools/bld_openssl.py +++ b/packaging-tools/bld_openssl.py @@ -65,7 +65,7 @@ def check_environment(): def check_cmd(cmd): if subprocess.call(cmd, stdout=fnull, stderr=fnull) != 0: print(f"*** Cannot execute {cmd[0]}") - exit(1) + sys.exit(1) check_cmd(['nasm', '-h']) check_cmd(['nmake', '/?']) check_cmd(['7z']) @@ -96,7 +96,7 @@ def main() -> None: parser = setup_argument_parser() args = parser.parse_args() - if args.toolset != 'VC-WIN32' and args.toolset != 'VC-WIN64A': + if args.toolset not in ('VC-WIN32', 'VC-WIN64A'): print('*** --toolset must be either VC-WIN32 or VC-WIN64A') sys.exit(1) diff --git a/packaging-tools/bld_utils.py b/packaging-tools/bld_utils.py index 994bd1a16..23bea606c 100644 --- a/packaging-tools/bld_utils.py +++ b/packaging-tools/bld_utils.py @@ -38,7 +38,7 @@ from copy import deepcopy from socket import setdefaulttimeout from subprocess import PIPE, STDOUT, Popen from sys import platform -from threading import currentThread +from threading import current_thread from time import sleep from urllib.error import HTTPError from urllib.parse import urljoin, urlparse @@ -181,7 +181,7 @@ def download(url, target, read_block_size=1048576): if received_size != int(total_size): raise Exception(f"Broken download, got a wrong size after download from '{url}'(total size: {total_size}, but {received_size} received).") except HTTPError as error: - raise Exception(f"Can not download '{url}' to '{target}' as target(error code: '{error.code}').") + raise Exception(f"Can not download '{url}' to '{target}' as target(error code: '{error.code}').") from error renamed = False try_rename_counter = 0 @@ -203,7 +203,7 @@ def download(url, target, read_block_size=1048576): sleep(2) continue if not os.path.lexists(target): - raise Exception(f"Could not rename {savefile_tmp} to {target}{os.linesep}Error: {str(error)}") + raise Exception(f"Could not rename {savefile_tmp} to {target}{os.linesep}Error: {str(error)}") from error finally: # this is done before the except code is called try: os.remove(savefile_tmp) @@ -239,13 +239,14 @@ def get_environment(extra_environment=None): @deep_copy_arguments -def run_command(command, cwd, extra_environment=None, only_error_case_output=False, expected_exit_codes=[0]): - if type(expected_exit_codes) is not list: +def run_command(command, cwd, extra_environment=None, only_error_case_output=False, expected_exit_codes=None): + expected_exit_codes = expected_exit_codes or [0] + if not isinstance(expected_exit_codes, list): raise TypeError(f"expected_exit_codes({type(expected_exit_codes)}) is not {list}") - if type(only_error_case_output) is not bool: + if not isinstance(only_error_case_output, bool): raise TypeError(f"only_error_case_output({type(only_error_case_output)}) is not {bool}") - if type(command) is list: + if isinstance(command, list): command_as_list = command else: command_as_list = command[:].split(' ') @@ -280,7 +281,7 @@ def run_command(command, cwd, extra_environment=None, only_error_case_output=Fal use_shell = is_windows() last_stdout_lines = [] last_stderr_lines = [] - if currentThread().name == "MainThread" and not only_error_case_output: + if current_thread().name == "MainThread" and not only_error_case_output: process = Popen( command_as_list, shell=use_shell, cwd=cwd, bufsize=-1, env=environment @@ -306,14 +307,14 @@ def run_command(command, cwd, extra_environment=None, only_error_case_output=Fal for line in stdout.readlines(): line = line.decode() last_stdout_lines.append(line) - if currentThread().name != "MainThread": + if current_thread().name != "MainThread": sys.stdout.write(line) # Show what we received from standard error. for line in stderr.readlines(): line = line.decode() last_stderr_lines.append(line) - if currentThread().name != "MainThread": + if current_thread().name != "MainThread": sys.stdout.write(line) # Sleep a bit before polling the readers again. @@ -337,7 +338,7 @@ def run_command(command, cwd, extra_environment=None, only_error_case_output=Fal if exit_code not in expected_exit_codes: last_output = "" exit_type = "" - if currentThread().name != "MainThread" or only_error_case_output: + if current_thread().name != "MainThread" or only_error_case_output: if len(last_stderr_lines) != 0: last_output += "".join(str(last_stderr_lines)) exit_type = "error " @@ -355,7 +356,8 @@ def run_command(command, cwd, extra_environment=None, only_error_case_output=Fal @deep_copy_arguments -def run_install_command(arguments=['install'], cwd=None, caller_arguments=None, extra_environment=None, only_error_case_output=False): +def run_install_command(arguments=None, cwd=None, caller_arguments=None, extra_environment=None, only_error_case_output=False): + arguments = arguments or ['install'] if hasattr(caller_arguments, 'installcommand') and caller_arguments.installcommand: installcommand = caller_arguments.installcommand.split() else: @@ -367,18 +369,19 @@ def run_install_command(arguments=['install'], cwd=None, caller_arguments=None, extra_environment["MAKEFLAGS"] = "-j1" if arguments: - installcommand.extend(arguments if type(arguments) is list else arguments.split()) + installcommand.extend(arguments if isinstance(arguments, list) else arguments.split()) return run_command(installcommand, cwd, extra_environment, only_error_case_output=only_error_case_output) @deep_copy_arguments -def run_build_command(arguments=None, cwd=None, caller_arguments=None, extra_environment=None, only_error_case_output=False, expected_exit_codes=[0]): +def run_build_command(arguments=None, cwd=None, caller_arguments=None, extra_environment=None, only_error_case_output=False, expected_exit_codes=None): + expected_exit_codes = expected_exit_codes or [0] buildcommand = ['make'] if hasattr(caller_arguments, 'buildcommand') and caller_arguments.buildcommand: buildcommand = caller_arguments.buildcommand.split() if arguments: - buildcommand.extend(arguments if type(arguments) is list else arguments.split()) + buildcommand.extend(arguments if isinstance(arguments, list) else arguments.split()) return run_command(buildcommand, cwd, extra_environment, only_error_case_output=only_error_case_output, expected_exit_codes=expected_exit_codes) diff --git a/packaging-tools/bldinstallercommon.py b/packaging-tools/bldinstallercommon.py index 70ae75212..693b09e0a 100644 --- a/packaging-tools/bldinstallercommon.py +++ b/packaging-tools/bldinstallercommon.py @@ -40,7 +40,7 @@ from pathlib import Path from subprocess import PIPE, STDOUT, Popen, check_call from tempfile import mkdtemp from traceback import print_exc -from typing import Callable, List, Union +from typing import Callable, List, Optional, Union from urllib.parse import urlparse from urllib.request import urlcleanup, urlopen, urlretrieve @@ -77,11 +77,8 @@ def is_content_url_valid(url): ############################### # function ############################### -current_download_percent = 0 # pylint: disable=C0103 - - def dl_progress(count, block_size, total_size): - global current_download_percent # pylint: disable=C0103 + current_download_percent = 0 percent = int(count * block_size * 100 / total_size) # produce only reasonable amount of prints into stdout if percent > current_download_percent: @@ -102,14 +99,14 @@ def retrieve_url(url, savefile): urlcleanup() urlretrieve(url, savefile_tmp, reporthook=dl_progress) shutil.move(savefile_tmp, savefile) - except Exception: + except Exception as err: exc = sys.exc_info()[0] print(exc) try: os.remove(savefile_tmp) except Exception: # swallow, do not shadow actual error pass - raise exc + raise exc from err ############################### @@ -300,7 +297,8 @@ def locate_executable(search_dir: Union[str, Path], patterns: List[str]) -> str: # Function ############################### def locate_path(search_dir: Union[str, Path], patterns: List[str], - filters: List[Callable[[Path], bool]] = []) -> str: + filters: Optional[List[Callable[[Path], bool]]] = None) -> str: + filters = filters or [] matches = locate_paths(search_dir, patterns, filters) if len(matches) != 1: raise PackagingError(f"Expected one result in '{search_dir}' matching '{patterns}'" @@ -312,7 +310,8 @@ def locate_path(search_dir: Union[str, Path], patterns: List[str], # Function ############################### def locate_paths(search_dir: Union[str, Path], patterns: List[str], - filters: List[Callable[[Path], bool]] = []) -> List[str]: + filters: Optional[List[Callable[[Path], bool]]] = None) -> List[str]: + filters = filters or [] patterns = patterns if patterns else ["*"] paths = [p for p in Path(search_dir).rglob("*") if any(p.match(ptn) for ptn in patterns)] return [str(p) for p in paths if all(f(p) for f in filters)] @@ -388,7 +387,8 @@ def sanity_check_rpath_max_length(file_path, new_rpath): ############################### # Function ############################### -def pathsplit(path, rest=[]): +def pathsplit(path, rest=None): + rest = rest or [] (head, tail) = os.path.split(path) if len(head) < 1: return [tail] + rest @@ -397,7 +397,8 @@ def pathsplit(path, rest=[]): return pathsplit(head, [tail] + rest) -def commonpath(list1, list2, common=[]): +def commonpath(list1, list2, common=None): + common = common or [] if len(list1) < 1: return (common, list1, list2) if len(list2) < 1: @@ -641,9 +642,9 @@ def create_extract_function(file_path, target_path): def create_download_and_extract_tasks(url, target_path, temp_path): filename = os.path.basename(urlparse(url).path) sevenzip_file = os.path.join(temp_path, filename) - download_task = Task(f"download '{url}' to '{sevenzip_file}'") + download_task = Task(f"download '{url}' to '{sevenzip_file}'", function=None) download_task.add_function(download, url, sevenzip_file) - extract_task = Task(f"extract '{sevenzip_file}' to '{target_path}'") + extract_task = Task(f"extract '{sevenzip_file}' to '{target_path}'", function=None) extract_task.add_function(create_extract_function(sevenzip_file, target_path)) return (download_task, extract_task) @@ -654,7 +655,7 @@ def create_download_and_extract_tasks(url, target_path, temp_path): def create_download_extract_task(url, target_path, temp_path): filename = os.path.basename(urlparse(url).path) sevenzip_file = os.path.join(temp_path, filename) - download_extract_task = Task(f"download {url} to {sevenzip_file} and extract it to {target_path}") + download_extract_task = Task(f"download {url} to {sevenzip_file} and extract it to {target_path}", function=None) download_extract_task.add_function(download, url, sevenzip_file) download_extract_task.add_function(create_extract_function(sevenzip_file, target_path)) return download_extract_task @@ -664,9 +665,9 @@ def create_download_extract_task(url, target_path, temp_path): # function ############################### def create_qt_download_task(module_urls, target_qt5_path, temp_path, caller_arguments): - qt_task = Task(f'download and extract Qt to "{target_qt5_path}"') + qt_task = Task(f'download and extract Qt to "{target_qt5_path}"', function=None) download_work = ThreadedWork(f'download Qt packages to "{temp_path}"') - unzip_task = Task(f'extracting packages to "{target_qt5_path}"') + unzip_task = Task(f'extracting packages to "{target_qt5_path}"', function=None) # add Qt modules for module_url in module_urls: if is_content_url_valid(module_url): diff --git a/packaging-tools/build_clang.py b/packaging-tools/build_clang.py index ad808a05a..88f16d662 100644 --- a/packaging-tools/build_clang.py +++ b/packaging-tools/build_clang.py @@ -100,7 +100,7 @@ def msvc_environment(bitness): if not os.path.exists(program_files): program_files = os.path.join('C:', '/Program Files') msvc_ver = msvc_version() - if msvc_ver == '14.1' or msvc_ver == '14.2': + if msvc_ver in ('14.1', '14.2'): vcvarsall = os.path.join(program_files, 'Microsoft Visual Studio', msvc_year(), 'Professional', 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat') arg = 'x64' if bitness == 64 else 'x86' else: @@ -447,6 +447,7 @@ def upload_clang(file_path, remote_path): def profile_data(toolchain): if is_windows() and is_mingw_toolchain(toolchain): return os.getenv('PROFILE_DATA_URL') + return None def main(): diff --git a/packaging-tools/build_clang_qdoc.py b/packaging-tools/build_clang_qdoc.py index b24984d60..dfbb7d4a7 100644 --- a/packaging-tools/build_clang_qdoc.py +++ b/packaging-tools/build_clang_qdoc.py @@ -89,7 +89,7 @@ def msvc_environment(bitness): if not os.path.exists(program_files): program_files = os.path.join('C:', '/Program Files') msvc_ver = msvc_version() - if msvc_ver == '14.1' or msvc_ver == '14.2': + if msvc_ver in ('14.1', '14.2'): vcvarsall = os.path.join(program_files, 'Microsoft Visual Studio', msvc_year(), 'Professional', 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat') arg = 'x64' if bitness == 64 else 'x86' else: diff --git a/packaging-tools/build_wrapper.py b/packaging-tools/build_wrapper.py index 990f33762..2e77e63e0 100644 --- a/packaging-tools/build_wrapper.py +++ b/packaging-tools/build_wrapper.py @@ -200,7 +200,7 @@ def create_download_documentation_task(base_url, download_path): run_command(['7z', 'a', '-mx1', '-mmt2', '-md32m', '-ms=1g', target_filepath, dest_doc_path], dest_doc_path) - download_task = Task(f"downloading documentation from {base_url}") + download_task = Task(f"downloading documentation from {base_url}", function=None) for item in file_list: url = base_url + '/doc/' + item download_filepath = os.path.join(download_path, item) @@ -208,7 +208,7 @@ def create_download_documentation_task(base_url, download_path): download_task.add_function(create_extract_function(download_filepath, extract_path)) download_task.add_function(create_remove_one_dir_level_function(os.path.join(extract_path, item.rstrip(".zip")))) - repackage_task = Task(f"repackaging documentation as {target_filepath}") + repackage_task = Task(f"repackaging documentation as {target_filepath}", function=None) repackage_task.add_function(repackage) return (download_task, repackage_task, file_url(target_filepath)) @@ -240,9 +240,9 @@ def create_download_openssl_task(url, download_path): run_command(['7z', 'a', '-mmt2', target_filepath, pattern], source_path) - download_task = Task(f"downloading openssl from {url}") + download_task = Task(f"downloading openssl from {url}", function=None) download_task.add_function(download, url, download_filepath) - repackage_task = Task(f"repackaging openssl as {target_filepath}") + repackage_task = Task(f"repackaging openssl as {target_filepath}", function=None) repackage_task.add_function(create_extract_function(download_filepath, extract_path)) repackage_task.add_function(repackage) return (download_task, repackage_task, file_url(target_filepath)) @@ -314,12 +314,14 @@ class BuildLog: with open(self.log_filepath, 'r', encoding="utf-8") as handle: print(handle.read()) return True # re-raise + return None # writes output of process to log_filepath # on error it dumps the log file to stdout as well -def check_call_log(args, execution_path, extra_env=dict(os.environ), +def check_call_log(args, execution_path, extra_env=None, log_filepath=None, log_overwrite=False): + extra_env = extra_env or os.environ.copy() if not log_filepath: do_execute_sub_process(args, execution_path, extra_env=extra_env) else: @@ -626,7 +628,7 @@ def handle_qt_creator_build(option_dict, qtcreator_plugins): gammaray_url = (pkg_base_path + '/' + gammaray_url + '/' + target_env_dir + '/qt5_gammaray.7z') download_work = ThreadedWork('Download packages') - extract_work = Task('Extract packages') + extract_work = Task('Extract packages', function=None) def add_download_extract(url, target_path): (dl_task, extract) = create_download_and_extract_tasks( @@ -678,7 +680,7 @@ def handle_qt_creator_build(option_dict, qtcreator_plugins): download_work.add_task_object(dl_task) extract_work.add_function(repackage.do_task) - download_packages_work = Task('Get and extract all needed packages') + download_packages_work = Task('Get and extract all needed packages', function=None) download_packages_work.add_function(download_work.run) download_packages_work.add_function(extract_work.do_task) download_packages_work.do_task() @@ -743,7 +745,7 @@ def handle_qt_creator_build(option_dict, qtcreator_plugins): try: value = get_pkg_value(key) # for python2 this is unicode which is not accepted as environment - if value and type(value) is not str: + if value and not isinstance(value, str): value = value.encode('UTF-8') if value: build_environment[key] = value @@ -788,14 +790,14 @@ def handle_qt_creator_build(option_dict, qtcreator_plugins): additional_plugins.extend([make_qtcplugin('appmanagerintegration', 'pcore-plugin-appman', qtcreator_version, modules=qt_module_local_urls, dependencies=plugin_dependencies, - additional_arguments=['--with-docs'])]), + additional_arguments=['--with-docs'])]) plugin_telemetry_args = [] if usp_server_url and usp_auth_key: plugin_telemetry_args = ['--add-config=-DUSP_SERVER_URL=' + option_dict['USP_SERVER_URL'], '--add-config=-DUSP_AUTH_KEY=' + option_dict['USP_AUTH_KEY']] additional_plugins.extend([make_qtcplugin('plugin-telemetry', 'plugin-telemetry', qtcreator_version, modules=qt_module_local_urls, - additional_arguments=plugin_telemetry_args)]), + additional_arguments=plugin_telemetry_args)]) # Build Qt Creator plugins icu_local_url = file_url(os.path.join(qt_temp, os.path.basename(icu_libs))) if is_linux() else None diff --git a/packaging-tools/create_installer.py b/packaging-tools/create_installer.py index 7f62f4f5d..c62d4f6d4 100644 --- a/packaging-tools/create_installer.py +++ b/packaging-tools/create_installer.py @@ -706,7 +706,7 @@ def cleanup_docs(install_dir): if filename.endswith(('.qdocconf', '.sha1', '.html')): full_filename = os.path.join(submodule, filename) if os.path.isfile(full_filename): - log.info("Cleaning up -> deleting file: %s", item) + log.info("Cleaning up -> deleting file: %s", full_filename) os.remove(full_filename) diff --git a/packaging-tools/install_qt.py b/packaging-tools/install_qt.py index 0dcc85a97..03b260510 100644 --- a/packaging-tools/install_qt.py +++ b/packaging-tools/install_qt.py @@ -31,6 +31,7 @@ import argparse import os +import sys from bldinstallercommon import create_qt_download_task, patch_qt from threadedwork import ThreadedWork @@ -65,7 +66,7 @@ def get_arguments(): if not args.qt_modules or args.base_url or args.module_name: if not args.base_url or not args.module_name: print('either --qt-module or --base-url and module_name(s) are required') - exit(1) + sys.exit(1) args.qt_modules = args.qt_modules if args.qt_modules else [] # ensure list args.qt_modules += [args.base_url + '/' + module + '/' + module + args.base_url_postfix diff --git a/packaging-tools/libclang_training/run_batch_files.py b/packaging-tools/libclang_training/run_batch_files.py index d0a3bfe18..afccc3989 100644 --- a/packaging-tools/libclang_training/run_batch_files.py +++ b/packaging-tools/libclang_training/run_batch_files.py @@ -146,6 +146,7 @@ class DebugView: def __init__(self, log_file_path): self.log_file_path = log_file_path self.executable = 'dbgview.exe' + self.proc = None def start_async(self): args = [self.executable, '/accepteula', '/l', self.log_file_path] @@ -315,8 +316,8 @@ def merge_generated_csv_files(run_records): new_value.append(run_record) batch_file_id_2_run_record[run_record.batch_file_id] = new_value - for batch_file_id in batch_file_id_2_run_record: - csv_file_paths = [run_record.csv_file_path for run_record in batch_file_id_2_run_record[batch_file_id]] + for batch_file_id, runrecord_list in batch_file_id_2_run_record.items(): + csv_file_paths = [run_record.csv_file_path for run_record in runrecord_list] merge_file_path = os.path.join(Config.LogDir, batch_file_id + ".csv") merge_csv_files.merge_files(merge_file_path, csv_file_paths) diff --git a/packaging-tools/read_remote_config.py b/packaging-tools/read_remote_config.py index ff7c18616..91414e59c 100755 --- a/packaging-tools/read_remote_config.py +++ b/packaging-tools/read_remote_config.py @@ -36,8 +36,6 @@ from configparser import ConfigParser from io import StringIO from urllib.request import urlopen -_pkg_remote_settings = None # pylint: disable=invalid-name - class RemotePkgConfigError(Exception): pass @@ -56,12 +54,12 @@ def parse_packaging_keys_config(config): def get_pkg_value(key, section="packaging", url=os.getenv("PACKAGING_KEYS_CONFIG_URL")): - global _pkg_remote_settings # pylint: disable=invalid-name - if not _pkg_remote_settings: + if getattr(get_pkg_value, 'pkg_remote_settings', None) is None: if not url: raise RemotePkgConfigError("Remote config URL not specified") - _pkg_remote_settings = parse_packaging_keys_config(read_packaging_keys_config_url(url)) - return _pkg_remote_settings.get(section, key) + config = read_packaging_keys_config_url(url) + get_pkg_value.pkg_remote_settings = parse_packaging_keys_config(config) + return get_pkg_value.pkg_remote_settings.get(section, key) def main() -> None: diff --git a/packaging-tools/release_repo_updater.py b/packaging-tools/release_repo_updater.py index e44a3e706..5d508c31e 100755 --- a/packaging-tools/release_repo_updater.py +++ b/packaging-tools/release_repo_updater.py @@ -850,9 +850,9 @@ def main() -> None: parser.add_argument("--offline-installer-build-id", dest="offline_installer_id", type=str, default=os.getenv('BUILD_NUMBER', timestamp), help="Add unique id for the offline installer") - parser.add_argument("--update-staging", dest="update_staging", action='store_true', default=os.getenv("DO_UPDATE_STAGING_REPOSITORY", False), + parser.add_argument("--update-staging", dest="update_staging", action='store_true', default=bool(os.getenv("DO_UPDATE_STAGING_REPOSITORY", "")), help="Should the staging repository be updated?") - parser.add_argument("--update-production", dest="update_production", action='store_true', default=os.getenv("DO_UPDATE_PRODUCTION_REPOSITORY", False), + parser.add_argument("--update-production", dest="update_production", action='store_true', default=bool(os.getenv("DO_UPDATE_PRODUCTION_REPOSITORY", "")), help="Should the production repository be updated?") parser.add_argument("--update-source", dest="update_source_type", type=str, choices=[RepoSource.PENDING.value, RepoSource.STAGING.value], diff --git a/packaging-tools/remote_uploader.py b/packaging-tools/remote_uploader.py index 26e03e446..65a0f9fdf 100755 --- a/packaging-tools/remote_uploader.py +++ b/packaging-tools/remote_uploader.py @@ -47,7 +47,9 @@ class RemoteUploader: def __init__(self, dry_run, remote_server, remote_server_username, remote_base_path): self.dry_run = dry_run self.set_tools(remote_server, remote_server_username) + self.remote_latest_link = "" self.remote_login = remote_server_username + '@' + remote_server + self.remote_target_dir = "" self.remote_target_base_dir = remote_base_path self.init_finished = False diff --git a/packaging-tools/runner.py b/packaging-tools/runner.py index 9f0e2383e..682133bd6 100755 --- a/packaging-tools/runner.py +++ b/packaging-tools/runner.py @@ -74,7 +74,8 @@ async def async_exec_cmd(cmd: List[str], timeout: int = 60 * 60, env: Dict[str, def do_execute_sub_process(args, execution_path, abort_on_fail=True, get_output=False, - extra_env=dict(os.environ), redirect_output=None, args_log=None): + extra_env=None, redirect_output=None, args_log=None): + extra_env = extra_env or os.environ.copy() _args_log = args_log or ' '.join([str(i) for i in args]) print(' --------------------------------------------------------------------') print(f' Executing: [{_args_log}]') diff --git a/packaging-tools/sdkcomponent.py b/packaging-tools/sdkcomponent.py index 0c32efaf6..0594736f5 100644 --- a/packaging-tools/sdkcomponent.py +++ b/packaging-tools/sdkcomponent.py @@ -82,6 +82,7 @@ class SdkComponent: return '*** Archive check fail! ***\n*** Unable to locate archive: ' + self.archive_uri elif not os.path.isfile(self.archive_uri): return '*** Archive check fail! ***\n*** Unable to locate archive: ' + self.archive_uri + return None def path_leaf(self, path): head, tail = ntpath.split(path) @@ -134,7 +135,7 @@ class SdkComponent: self.version = self.version.replace(item[0], item[1]) def is_root_component(self): - if self.root_component == 'yes' or self.root_component == 'true': + if self.root_component in ('yes', 'true'): return True return False diff --git a/packaging-tools/sign_windows_installer.py b/packaging-tools/sign_windows_installer.py index e691583e9..ccd02ed6a 100755 --- a/packaging-tools/sign_windows_installer.py +++ b/packaging-tools/sign_windows_installer.py @@ -89,13 +89,13 @@ def _handle_signing(file_path: str): log_entry[9] = "****" log_entry[11] = "****" log.info("Calling: %s", ' '.join(log_entry)) - sign_result = subprocess.run(cmd_args_sign, stdout=DEVNULL, stderr=DEVNULL) + sign_result = subprocess.run(cmd_args_sign, stdout=DEVNULL, stderr=DEVNULL, check=False) if sign_result.returncode != 0: raise PackagingError(f"Package {file_path} signing with error {sign_result.returncode}") log.info("Successfully signed: %s", file_path) signtool = os.path.basename(os.environ["WINDOWS_SIGNTOOL_X64_PATH"]) cmd_args_verify: List[str] = [signtool, "verify", "-pa", file_path] - verify_result = subprocess.run(cmd_args_verify, stdout=DEVNULL, stderr=DEVNULL) + verify_result = subprocess.run(cmd_args_verify, stdout=DEVNULL, stderr=DEVNULL, check=False) if verify_result.returncode != 0: raise PackagingError(f"Failed to verify {file_path} with error {verify_result.returncode}") log.info("Successfully verified: %s", file_path) diff --git a/packaging-tools/tests/test_run_command.py b/packaging-tools/tests/test_run_command.py index 57cbfee07..1adb18dc8 100644 --- a/packaging-tools/tests/test_run_command.py +++ b/packaging-tools/tests/test_run_command.py @@ -107,7 +107,7 @@ class TestRunCommand(unittest.TestCase): self.assertEqual(expected_message_start, message_start) expected_message_end = "9 printed line" message_end = str(context_manager.exception).splitlines()[-1] - self.assertTrue(message_end.__contains__(expected_message_end)) + self.assertTrue(expected_message_end in message_end) def test_different_exit_code_only_error_case_output(self): self.assertEqual( @@ -123,7 +123,7 @@ class TestRunCommand(unittest.TestCase): ) def test_with_threadedwork(self): - current_method_name = sys._getframe().f_code.co_name + current_method_name = sys._getframe().f_code.co_name # pylint: disable=W0212 test_work = ThreadedWork(f"{current_method_name} - run some command threaded") task_string_list = [] task_string_list.append("--sleep 1 --print_lines 10") @@ -136,7 +136,7 @@ class TestRunCommand(unittest.TestCase): test_work.run() def test_with_threadedwork_unexpected_exit_code(self): - current_method_name = sys._getframe().f_code.co_name + current_method_name = sys._getframe().f_code.co_name # pylint: disable=W0212 test_work = ThreadedWork(f"{current_method_name} - run some command threaded") # this exchange the current os._exit(-1) implementation only for this testing case separator_line = f"{os.linesep}>>>>>>>>>>>>>>>>>>>>>>>>>>>>>{os.linesep}" @@ -153,7 +153,7 @@ class TestRunCommand(unittest.TestCase): test_work.run() def test_with_threadedwork_crash(self): - current_method_name = sys._getframe().f_code.co_name + current_method_name = sys._getframe().f_code.co_name # pylint: disable=W0212 test_work = ThreadedWork(f"{current_method_name} - run some command threaded") # this exchange the current os._exit(-1) implementation only for this testing case separator_line = f"{os.linesep}>>>>>>>>>>>>>>>>>>>>>>>>>>>>>{os.linesep}" @@ -191,7 +191,7 @@ if __name__ == '__main__': sys.__stderr__.flush() crash() if caller_arguments.exit_code: - os._exit(caller_arguments.exit_code) + os._exit(caller_arguments.exit_code) # pylint: disable=W0212 if caller_arguments.testMethod: # python test_run_command.py --testMethod test_crash_only_error_case_output TestRunCommand(methodName=caller_arguments.testMethod).debug() diff --git a/packaging-tools/tests/testhelpers.py b/packaging-tools/tests/testhelpers.py index 91844b49d..ad7cfa696 100644 --- a/packaging-tools/tests/testhelpers.py +++ b/packaging-tools/tests/testhelpers.py @@ -67,7 +67,7 @@ def is_internal_file_server_reachable() -> bool: try: package_server = get_pkg_value("PACKAGE_STORAGE_SERVER") ping = sh.which("ping") - ret = subprocess.run(args=[ping, "-c", "1", package_server], timeout=5, stdout=PIPE, stderr=PIPE) + ret = subprocess.run(args=[ping, "-c", "1", package_server], timeout=5, stdout=PIPE, stderr=PIPE, check=False) return ret.returncode == 0 except Exception: pass diff --git a/packaging-tools/threadedwork.py b/packaging-tools/threadedwork.py index 75746245b..87c7ec258 100644 --- a/packaging-tools/threadedwork.py +++ b/packaging-tools/threadedwork.py @@ -51,8 +51,6 @@ class StdOutHook: stripped_text = text.strip() if stripped_text == "": return - global output_states # pylint: disable=invalid-name - global output_format_string # pylint: disable=invalid-name local_progress_indicator = None if len(stripped_text) > 6: local_progress_indicator = next_progress_indicator() @@ -99,8 +97,8 @@ org_sterr = sys.stderr def enable_threaded_print(enable=True, thread_count=cpu_count()): if enable: - global output_states # pylint: disable=invalid-name - global output_format_string # pylint: disable=invalid-name + global output_states # pylint: disable=W0603,C0103 + global output_format_string # pylint: disable=W0603,C0103 output_states = [""] * (thread_count) output_format_string = "" for xthread in range(thread_count): @@ -133,7 +131,7 @@ class TaskFunction(): class Task(): - def __init__(self, description, function=None, *arguments): + def __init__(self, description, function, *arguments): self.task_number = 0 # will be set from outside self.description = description self.list_of_functions = [] @@ -174,6 +172,7 @@ class ThreadedWork(): self.legend = [] self.task_number = 0 self.exit_function = None + self.exit_function_arguments = None def set_exit_fail_function(self, function, *arguments): self.exit_function = function @@ -230,7 +229,7 @@ class Consumer(threading.Thread): self.worker_thread_id = worker_thread_id threading.Thread.__init__(self) - def run(self, stable_run_indicator=True): + def run(self, stable_run_indicator=True): # pylint: disable=W0221 if stable_run_indicator: thread_data.progress_indicator = itertools.cycle(['..']) else: |