diff options
author | Patrik Teivonen <patrik.teivonen@qt.io> | 2022-09-01 11:03:19 +0300 |
---|---|---|
committer | Patrik Teivonen <patrik.teivonen@qt.io> | 2022-09-07 21:11:57 +0000 |
commit | 03ca8c3c145fa8a01235d15f9532714fa09d5b4b (patch) | |
tree | 117854bc1c5583be6eb99d132365398703f971ba | |
parent | a35e03258a371b6ff83497186a9ba3266bc45cf4 (diff) |
pep8-naming: N802 function name should be lowercase (function names)
Enable this check and rename functions.
Change-Id: Ifc6c3248f6c6b90a240d66bb2d20cd716d887729
Reviewed-by: Iikka Eklund <iikka.eklund@qt.io>
25 files changed, 377 insertions, 378 deletions
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 716471578..184651749 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,10 +24,9 @@ repos: # Disable E203 for compatibility with blackformatter, and W503 as it goes against PEP8 # Disable checks that are not relevant to this patch, they will be introduced later # Disable pep8-naming checks that will be introduced later: - # - N802 function name should be lowercase (function names) # - N803 argument name should be lowercase (function arguments) # - N806 variable in function should be lowercase - entry: bash -c 'pipenv run python3 -m flake8 --max-line-length=99 --ignore=E203,E501,W503,N802,N803,N806 "$@"' + entry: bash -c 'pipenv run python3 -m flake8 --max-line-length=99 --ignore=E203,E501,W503,N803,N806 "$@"' language: system types: [python] fail_fast: true diff --git a/packaging-tools/bld_ifw_tools.py b/packaging-tools/bld_ifw_tools.py index 3c1eff560..0efad273e 100755 --- a/packaging-tools/bld_ifw_tools.py +++ b/packaging-tools/bld_ifw_tools.py @@ -429,7 +429,7 @@ def prepare_installer_framework(options): prepare_compressed_package(options.qt_installer_framework_uri, options.qt_installer_framework_uri_saveas, options.installer_framework_source_dir) -def start_IFW_build(options, cmd_args, installer_framework_build_dir): +def start_ifw_build(options, cmd_args, installer_framework_build_dir): print(f"cmd_args: {list_as_string(cmd_args)}") do_execute_sub_process(cmd_args, installer_framework_build_dir) cmd_args = options.make_cmd @@ -459,7 +459,7 @@ def build_installer_framework(options): cmd_args = [qmake_bin] cmd_args += options.qt_installer_framework_qmake_args cmd_args += [options.installer_framework_source_dir] - start_IFW_build(options, cmd_args, options.installer_framework_build_dir) + start_ifw_build(options, cmd_args, options.installer_framework_build_dir) def build_installer_framework_examples(options): diff --git a/packaging-tools/bld_lib.py b/packaging-tools/bld_lib.py index 2dd78dd2e..3c3833632 100644 --- a/packaging-tools/bld_lib.py +++ b/packaging-tools/bld_lib.py @@ -63,14 +63,14 @@ handler.setFormatter(formatter) log.addHandler(handler) -def findFile(searchPath: str, fileName: str) -> str: +def find_file(searchPath: str, fileName: str) -> str: for root, _, files in os.walk(searchPath): if fileName in files: return os.path.join(root, fileName) assert False, f"Unable to find: {fileName} from: {searchPath}" -def collectLibs(searchPath: str) -> List[str]: +def collect_libs(searchPath: str) -> List[str]: for root, dirs, _ in os.walk(searchPath): for dir_name in dirs: if dir_name == "lib": @@ -78,7 +78,7 @@ def collectLibs(searchPath: str) -> List[str]: assert False, f"Unable to find: 'lib' from: {searchPath}" -def parseQtVersion(downloadUrlPath: str) -> str: +def parse_qt_version(downloadUrlPath: str) -> str: regex = re.compile(r'([\d.]+)') for item in downloadUrlPath.split("/"): m = regex.search(item) @@ -87,10 +87,10 @@ def parseQtVersion(downloadUrlPath: str) -> str: assert False, f"Could not parse Qt version number from: {downloadUrlPath}" -def downloadQtPkg(args: argparse.Namespace, currentDir: str) -> Tuple[str, str]: +def download_qt_pkg(args: argparse.Namespace, currentDir: str) -> Tuple[str, str]: urlRes = urlparse(args.qtpkg) assert urlRes.scheme and urlRes.netloc and urlRes.path, f"Invalid URL: {args.qtpkg}" - qtVersion = parseQtVersion(urlRes.path) + qtVersion = parse_qt_version(urlRes.path) saveAs = os.path.join(currentDir, os.path.basename(urlRes.path)) if os.path.exists(saveAs): @@ -102,7 +102,7 @@ def downloadQtPkg(args: argparse.Namespace, currentDir: str) -> Tuple[str, str]: return saveAs, qtVersion -def extractArchive(saveAs: str, currentDir: str) -> str: +def extract_archive(saveAs: str, currentDir: str) -> str: qtDestDir = os.path.join(currentDir, "qt_pkg") if not os.path.exists(qtDestDir): os.makedirs(qtDestDir) @@ -130,7 +130,7 @@ def build(args: argparse.Namespace, qtDestDir: str, currentDir: str) -> str: qmakeToolName = "qmake" makeToolName = "make" - qmakeTool = findFile(qtDestDir, qmakeToolName) + qmakeTool = find_file(qtDestDir, qmakeToolName) assert qmakeTool, f"Could not find: {qmakeToolName} from: {qtDestDir}" # patch @@ -176,7 +176,7 @@ def archive(args: argparse.Namespace, installRootDir: str, currentDir: str) -> s archivePath = os.path.join(installRootDir, srcPath.lstrip(os.path.sep)) log.info("Archiving from: %s", archivePath) - libs = collectLibs(installRootDir) + libs = collect_libs(installRootDir) for lib in libs: shutil.copy2(lib, archivePath) @@ -196,18 +196,18 @@ def archive(args: argparse.Namespace, installRootDir: str, currentDir: str) -> s return artifactsFilePath -def handleBuild(args: argparse.Namespace) -> None: +def handle_build(args: argparse.Namespace) -> None: currentDir = os.getcwd() - saveAs, qtVersion = downloadQtPkg(args, currentDir) - qtDestDir = extractArchive(saveAs, currentDir) + saveAs, qtVersion = download_qt_pkg(args, currentDir) + qtDestDir = extract_archive(saveAs, currentDir) installRootDir = build(args, qtDestDir, currentDir) artifactsFilePath = archive(args, installRootDir, currentDir) remoteUploader = RemoteUploader(False, args.remote_server, args.username, args.remote_base_path) remoteUploader.init_snapshot_upload_path(args.project_name, qtVersion, args.build_id) - remoteUploader.copyToRemote(artifactsFilePath) - remoteUploader.updateLatestSymlink() + remoteUploader.copy_to_remote(artifactsFilePath) + remoteUploader.update_latest_symlink() def main() -> None: @@ -232,7 +232,7 @@ def main() -> None: log.error("Could not find '7z' from the system. This tool is needed for notarization. Aborting..") sys.exit(1) - handleBuild(args) + handle_build(args) if __name__ == "__main__": diff --git a/packaging-tools/bld_module.py b/packaging-tools/bld_module.py index ca5a378b8..c80854d23 100755 --- a/packaging-tools/bld_module.py +++ b/packaging-tools/bld_module.py @@ -42,10 +42,10 @@ from bld_utils import ( is_linux, is_macos, is_windows, - runBuildCommand, - runCommand, - runInstallCommand, - stripVars, + run_build_command, + run_command, + run_install_command, + strip_vars, ) from bldinstallercommon import ( clone_repository, @@ -182,7 +182,7 @@ def main() -> None: callerArguments = parser.parse_args() # cleanup some values inside the callerArguments object - stripVars(callerArguments, "\"") + strip_vars(callerArguments, "\"") if callerArguments.qt5path != os.path.abspath(callerArguments.qt5path): print(f"changing the value of --qt5path from {callerArguments.qt5path} to {os.path.abspath(callerArguments.qt5path)}") callerArguments.qt5path = os.path.abspath(callerArguments.qt5path) @@ -200,7 +200,7 @@ def main() -> None: elif callerArguments.module7z != '': Path(MODULE_SRC_DIR).mkdir(parents=True, exist_ok=True) myGetQtModule = ThreadedWork("get and extract module src") - myGetQtModule.addTaskObject(create_download_and_extract_tasks(callerArguments.module7z, MODULE_SRC_DIR, tempPath)) + myGetQtModule.add_task_object(create_download_and_extract_tasks(callerArguments.module7z, MODULE_SRC_DIR, tempPath)) myGetQtModule.run() qtModuleSourceDirectory = MODULE_SRC_DIR else: @@ -234,7 +234,7 @@ def main() -> None: if not os.path.lexists(callerArguments.qt5path): # get Qt myGetQtBinaryWork = ThreadedWork("get and extract Qt 5 binary") - myGetQtBinaryWork.addTaskObject( + myGetQtBinaryWork.add_task_object( create_qt_download_task( callerArguments.qt5_module_urls, callerArguments.qt5path, tempPath, callerArguments @@ -307,13 +307,13 @@ def main() -> None: generateCommand.append(os.environ["EXTRA_QMAKE_ARGS"]) generateCommand.append(qtModuleProFile) - runCommand(generateCommand, currentWorkingDirectory=qtModuleBuildDirectory, extra_environment=environment) + run_command(generateCommand, currentWorkingDirectory=qtModuleBuildDirectory, extra_environment=environment) - ret = runBuildCommand(currentWorkingDirectory=qtModuleBuildDirectory, callerArguments=callerArguments) + ret = run_build_command(currentWorkingDirectory=qtModuleBuildDirectory, callerArguments=callerArguments) if ret: raise RuntimeError(f"Failure running the last command: {ret}") - ret = runInstallCommand( + ret = run_install_command( ['install', 'INSTALL_ROOT=' + qtModuleInstallDirectory], currentWorkingDirectory=qtModuleBuildDirectory, callerArguments=callerArguments, extra_environment=environment @@ -332,14 +332,14 @@ def main() -> None: # enginio etc. docs creation if callerArguments.makeDocs: # build docs first - ret = runInstallCommand( + ret = run_install_command( 'docs', currentWorkingDirectory=qtModuleBuildDirectory, callerArguments=callerArguments, extra_environment=environment ) if ret: raise RuntimeError(f"Failure running the last command: {ret}") # then make install those - ret = runInstallCommand( + ret = run_install_command( ['install_docs', 'INSTALL_ROOT=' + qtModuleInstallDirectory], currentWorkingDirectory=qtModuleBuildDirectory, callerArguments=callerArguments, extra_environment=environment @@ -349,7 +349,7 @@ def main() -> None: # make separate "doc.7z" for later use if needed doc_dir = locate_path(qtModuleInstallDirectory, ["doc"], filters=[os.path.isdir]) archive_name = callerArguments.module_name + '-' + os.environ['LICENSE'] + '-doc-' + os.environ['MODULE_VERSION'] + '.7z' - ret = runCommand( + ret = run_command( ['7z', 'a', os.path.join('doc_archives', archive_name), doc_dir], currentWorkingDirectory=os.path.dirname(os.path.realpath(__file__)) ) @@ -379,7 +379,7 @@ def main() -> None: archive_cmd.append(os.path.join(dir_to_archive, '*')) else: archive_cmd.append(dir_to_archive) - ret = runCommand(archive_cmd, currentWorkingDirectory=os.path.dirname(os.path.realpath(__file__))) + ret = run_command(archive_cmd, currentWorkingDirectory=os.path.dirname(os.path.realpath(__file__))) if ret: raise RuntimeError(f"Failure running the last command: {ret}") diff --git a/packaging-tools/bld_utils.py b/packaging-tools/bld_utils.py index b7ea6e0a4..233148c9b 100644 --- a/packaging-tools/bld_utils.py +++ b/packaging-tools/bld_utils.py @@ -99,14 +99,14 @@ def compress(path, directoryName, sevenZipTarget): sevenZipTarget = sevenZipTarget + sevenZipExtension sevenZipFileName = os.path.split(sevenZipTarget)[1] with DirRenamer(path, directoryName): - runCommand(' '.join(('7z a -mx9', sevenZipFileName, directoryName)), parentDirectoryPath) + run_command(' '.join(('7z a -mx9', sevenZipFileName, directoryName)), parentDirectoryPath) currentSevenZipPath = os.path.join(parentDirectoryPath, sevenZipFileName) if currentSevenZipPath != sevenZipTarget: shutil.move(currentSevenZipPath, sevenZipTarget) -def stripVars(sobject, chars): +def strip_vars(sobject, chars): for key, value in vars(sobject).items(): if isinstance(value, str): setattr(sobject, key, value.strip(chars)) @@ -142,7 +142,7 @@ def download(url, target, read_block_size=1048576): if os.path.lexists(target): raise Exception(f"Can not download '{url}' to '{target}' as target. The file already exists.") - def localDownload(localFilePath, targtFilePath): + def local_download(localFilePath, targtFilePath): if os.path.isfile(localFilePath): print(f"copying file from '{localFilePath}' to '{targtFilePath}'") try: @@ -155,14 +155,14 @@ def download(url, target, read_block_size=1048576): if os.path.lexists(url[len("file:///"):]): # because scheme of a absolute windows path is the drive letter in python 2, # we need to use file:// as a work around in urls - localDownload(url[len("file:///"):], target) + local_download(url[len("file:///"):], target) return # there is code which only have two slashes - protocol://host/path <- localhost can be omitted if os.path.lexists(url[len("file://"):]): - localDownload(url[len("file://"):], target) + local_download(url[len("file://"):], target) return if os.path.lexists(url): - localDownload(url, target) + local_download(url, target) return savefile_tmp = os.extsep.join((target, 'tmp')) @@ -211,7 +211,7 @@ def download(url, target, read_block_size=1048576): pass -def setValueOnEnvironmentDict(environment, key, value): +def set_value_on_environment_dict(environment, key, value): if key in environment: # if the data already contains the value stop here if value in environment[key].split(os.pathsep): @@ -222,7 +222,7 @@ def setValueOnEnvironmentDict(environment, key, value): @deep_copy_arguments -def getEnvironment(extra_environment=None): +def get_environment(extra_environment=None): # first take the one from the system and use the plain dictionary data for that environment = dict(os.environ) @@ -232,14 +232,14 @@ def getEnvironment(extra_environment=None): for key in extra_environment.keys(): keyUpper = key.upper() if any((keyUpper == 'PATH', keyUpper == 'INCLUDE', keyUpper == 'LIB')): - setValueOnEnvironmentDict(environment, key, extra_environment[key]) + set_value_on_environment_dict(environment, key, extra_environment[key]) else: environment[key] = extra_environment[key] return environment @deep_copy_arguments -def runCommand(command, currentWorkingDirectory, extra_environment=None, onlyErrorCaseOutput=False, expectedExitCodes=[0]): +def run_command(command, currentWorkingDirectory, extra_environment=None, onlyErrorCaseOutput=False, expectedExitCodes=[0]): if type(expectedExitCodes) is not list: raise TypeError(f"expectedExitCodes({type(expectedExitCodes)}) is not {list}") if type(onlyErrorCaseOutput) is not bool: @@ -250,7 +250,7 @@ def runCommand(command, currentWorkingDirectory, extra_environment=None, onlyErr else: commandAsList = command[:].split(' ') - environment = getEnvironment(extra_environment) + environment = get_environment(extra_environment) # if we can not find the command, just check the current working dir if (not os.path.lexists(commandAsList[0]) and currentWorkingDirectory @@ -355,7 +355,7 @@ def runCommand(command, currentWorkingDirectory, extra_environment=None, onlyErr @deep_copy_arguments -def runInstallCommand(arguments=['install'], currentWorkingDirectory=None, callerArguments=None, extra_environment=None, onlyErrorCaseOutput=False): +def run_install_command(arguments=['install'], currentWorkingDirectory=None, callerArguments=None, extra_environment=None, onlyErrorCaseOutput=False): if hasattr(callerArguments, 'installcommand') and callerArguments.installcommand: installcommand = callerArguments.installcommand.split() else: @@ -368,39 +368,39 @@ def runInstallCommand(arguments=['install'], currentWorkingDirectory=None, calle if arguments: installcommand.extend(arguments if type(arguments) is list else arguments.split()) - return runCommand(installcommand, currentWorkingDirectory, extra_environment, onlyErrorCaseOutput=onlyErrorCaseOutput) + return run_command(installcommand, currentWorkingDirectory, extra_environment, onlyErrorCaseOutput=onlyErrorCaseOutput) @deep_copy_arguments -def runBuildCommand(arguments=None, currentWorkingDirectory=None, callerArguments=None, extra_environment=None, onlyErrorCaseOutput=False, expectedExitCodes=[0]): +def run_build_command(arguments=None, currentWorkingDirectory=None, callerArguments=None, extra_environment=None, onlyErrorCaseOutput=False, expectedExitCodes=[0]): buildcommand = ['make'] if hasattr(callerArguments, 'buildcommand') and callerArguments.buildcommand: buildcommand = callerArguments.buildcommand.split() if arguments: buildcommand.extend(arguments if type(arguments) is list else arguments.split()) - return runCommand(buildcommand, currentWorkingDirectory, extra_environment, onlyErrorCaseOutput=onlyErrorCaseOutput, expectedExitCodes=expectedExitCodes) + return run_command(buildcommand, currentWorkingDirectory, extra_environment, onlyErrorCaseOutput=onlyErrorCaseOutput, expectedExitCodes=expectedExitCodes) @deep_copy_arguments -def getReturnValue(command, currentWorkingDirectory=None, extra_environment=None): +def get_return_value(command, currentWorkingDirectory=None, extra_environment=None): commandAsList = command[:].split(' ') return Popen( commandAsList, stdout=PIPE, stderr=STDOUT, - cwd=currentWorkingDirectory, env=getEnvironment(extra_environment) + cwd=currentWorkingDirectory, env=get_environment(extra_environment) ).communicate()[0].strip() -def gitSHA(path): +def git_sha(path): gitBinary = "git" - if isGitDirectory(path): - return getReturnValue(gitBinary + " rev-list -n1 HEAD", currentWorkingDirectory=path).strip() + if is_git_directory(path): + return get_return_value(gitBinary + " rev-list -n1 HEAD", currentWorkingDirectory=path).strip() return '' # get commit SHA either directly from git, or from a .tag file in the source directory -def get_commit_SHA(source_path): - buildGitSHA = gitSHA(source_path) +def get_commit_sha(source_path): + buildGitSHA = git_sha(source_path) if not buildGitSHA: tagfile = os.path.join(source_path, '.tag') if os.path.exists(tagfile): @@ -409,7 +409,7 @@ def get_commit_SHA(source_path): return buildGitSHA -def isGitDirectory(repository_path): +def is_git_directory(repository_path): if not repository_path: return False gitConfigDir = os.path.abspath(os.path.join(repository_path, '.git')) diff --git a/packaging-tools/bldinstallercommon.py b/packaging-tools/bldinstallercommon.py index f125ed739..82cd4fdbd 100644 --- a/packaging-tools/bldinstallercommon.py +++ b/packaging-tools/bldinstallercommon.py @@ -44,7 +44,7 @@ from typing import Callable, List, Union from urllib.parse import urlparse from urllib.request import urlcleanup, urlopen, urlretrieve -from bld_utils import download, is_linux, is_macos, is_windows, runCommand +from bld_utils import download, is_linux, is_macos, is_windows, run_command from installer_utils import PackagingError from runner import do_execute_sub_process from threadedwork import Task, ThreadedWork @@ -80,7 +80,7 @@ def is_content_url_valid(url): CURRENT_DOWNLOAD_PERCENT = 0 -def dlProgress(count, blockSize, totalSize): +def dl_progress(count, blockSize, totalSize): global CURRENT_DOWNLOAD_PERCENT percent = int(count * blockSize * 100 / totalSize) # produce only reasonable amount of prints into stdout @@ -100,7 +100,7 @@ def retrieve_url(url, savefile): try: savefile_tmp = savefile + '.tmp' urlcleanup() - urlretrieve(url, savefile_tmp, reporthook=dlProgress) + urlretrieve(url, savefile_tmp, reporthook=dl_progress) shutil.move(savefile_tmp, savefile) except Exception: exc = sys.exc_info()[0] @@ -214,12 +214,12 @@ def remove_tree(path): path = win32api.GetShortPathName(path.replace('/', '\\')) # a funny thing is that rmdir does not set an exitcode it is just using the last set one try: - runCommand(['rmdir', path, '/S', '/Q'], os.getcwd(), onlyErrorCaseOutput=True) + run_command(['rmdir', path, '/S', '/Q'], os.getcwd(), onlyErrorCaseOutput=True) except Exception: print_exc() else: # shutil.rmtree(path) - runCommand(['rm', '-rf', path], os.getcwd(), onlyErrorCaseOutput=True) + run_command(['rm', '-rf', path], os.getcwd(), onlyErrorCaseOutput=True) return not os.path.exists(path) @@ -589,7 +589,7 @@ def extract_file(path, to_directory='.'): print(f'Did not extract the file! Not archived or no appropriate extractor was found: {path}') return False - ret = runCommand(cmd_args, currentWorkingDirectory=to_directory, onlyErrorCaseOutput=True) + ret = run_command(cmd_args, currentWorkingDirectory=to_directory, onlyErrorCaseOutput=True) if ret: raise RuntimeError(f"Failure running the last command: {ret}") return True @@ -635,8 +635,8 @@ def create_extract_function(file_path, target_path): Path(target_path).mkdir(parents=True, exist_ok=True) working_dir = os.path.dirname(file_path) if file_path.endswith('.tar.gz'): - return lambda: runCommand(['tar', 'zxf', file_path, '-C', target_path], working_dir) - return lambda: runCommand(['7z', 'x', '-y', file_path, '-o' + target_path], working_dir) + return lambda: run_command(['tar', 'zxf', file_path, '-C', target_path], working_dir) + return lambda: run_command(['7z', 'x', '-y', file_path, '-o' + target_path], working_dir) ############################### @@ -646,9 +646,9 @@ def create_download_and_extract_tasks(url, target_path, temp_path): filename = os.path.basename(urlparse(url).path) sevenzip_file = os.path.join(temp_path, filename) download_task = Task(f"download '{url}' to '{sevenzip_file}'") - download_task.addFunction(download, url, sevenzip_file) + download_task.add_function(download, url, sevenzip_file) extract_task = Task(f"extract '{sevenzip_file}' to '{target_path}'") - extract_task.addFunction(create_extract_function(sevenzip_file, target_path)) + extract_task.add_function(create_extract_function(sevenzip_file, target_path)) return (download_task, extract_task) @@ -659,8 +659,8 @@ def create_download_extract_task(url, target_path, temp_path): filename = os.path.basename(urlparse(url).path) sevenzip_file = os.path.join(temp_path, filename) download_extract_task = Task(f"download {url} to {sevenzip_file} and extract it to {target_path}") - download_extract_task.addFunction(download, url, sevenzip_file) - download_extract_task.addFunction(create_extract_function(sevenzip_file, target_path)) + download_extract_task.add_function(download, url, sevenzip_file) + download_extract_task.add_function(create_extract_function(sevenzip_file, target_path)) return download_extract_task @@ -677,8 +677,8 @@ def create_qt_download_task(module_urls, target_qt5_path, temp_path, caller_argu (download_task, extract_task) = create_download_and_extract_tasks( module_url, target_qt5_path, temp_path ) - download_work.addTaskObject(download_task) - unzip_task.addFunction(extract_task.do) + download_work.add_task_object(download_task) + unzip_task.add_function(extract_task.do) else: print(f"warning: could not find '{module_url}' for download") # add icu, d3dcompiler, opengl32, openssl @@ -687,29 +687,29 @@ def create_qt_download_task(module_urls, target_qt5_path, temp_path, caller_argu (download_task, extract_task) = create_download_and_extract_tasks( caller_arguments.icu7z, target_path, temp_path ) - download_work.addTaskObject(download_task) - unzip_task.addFunction(extract_task.do) + download_work.add_task_object(download_task) + unzip_task.add_function(extract_task.do) if is_windows(): if hasattr(caller_arguments, 'd3dcompiler7z') and caller_arguments.d3dcompiler7z: (download_task, extract_task) = create_download_and_extract_tasks( caller_arguments.d3dcompiler7z, target_path, temp_path ) - download_work.addTaskObject(download_task) - unzip_task.addFunction(extract_task.do) + download_work.add_task_object(download_task) + unzip_task.add_function(extract_task.do) if hasattr(caller_arguments, 'opengl32sw7z') and caller_arguments.opengl32sw7z: (download_task, extract_task) = create_download_and_extract_tasks( caller_arguments.opengl32sw7z, target_path, temp_path ) - download_work.addTaskObject(download_task) - unzip_task.addFunction(extract_task.do) + download_work.add_task_object(download_task) + unzip_task.add_function(extract_task.do) if hasattr(caller_arguments, 'openssl7z') and caller_arguments.openssl7z: (download_task, extract_task) = create_download_and_extract_tasks( caller_arguments.openssl7z, target_path, temp_path ) - download_work.addTaskObject(download_task) - unzip_task.addFunction(extract_task.do) - qt_task.addFunction(download_work.run) - qt_task.addFunction(unzip_task.do) + download_work.add_task_object(download_task) + unzip_task.add_function(extract_task.do) + qt_task.add_function(download_work.run) + qt_task.add_function(unzip_task.do) return qt_task @@ -724,4 +724,4 @@ def patch_qt(qt5_path): if is_linux(): handle_component_rpath(qt5_path, 'lib') print("##### patch Qt ##### ... done") - runCommand(qmake_binary + " -query", qt5_path) + run_command(qmake_binary + " -query", qt5_path) diff --git a/packaging-tools/build_clang.py b/packaging-tools/build_clang.py index cff938dfd..cbcd1de8d 100644 --- a/packaging-tools/build_clang.py +++ b/packaging-tools/build_clang.py @@ -33,7 +33,7 @@ import os from shutil import rmtree import environmentfrombatchfile -from bld_utils import is_linux, is_macos, is_windows, runCommand +from bld_utils import is_linux, is_macos, is_windows, run_command from bldinstallercommon import create_download_extract_task, create_qt_download_task from read_remote_config import get_pkg_value from runner import do_execute_sub_process @@ -41,13 +41,13 @@ from threadedwork import ThreadedWork def git_clone_and_checkout(base_path, remote_repository_url, directory, revision): - runCommand(['git', 'clone', - '--depth', '1', - '--config', 'core.eol=lf', - '--config', 'core.autocrlf=input', - '--branch', revision, - '--recursive', - remote_repository_url, directory], base_path) + run_command(['git', 'clone', + '--depth', '1', + '--config', 'core.eol=lf', + '--config', 'core.autocrlf=input', + '--branch', revision, + '--recursive', + remote_repository_url, directory], base_path) def get_clang(base_path, llvm_repository_url, llvm_revision): @@ -190,10 +190,10 @@ def mingw_training(base_path, qtcreator_path, environment, bitness): qt_temp = os.path.join(base_path, 'qt_download') qt_mingw_temp = os.path.join(base_path, 'qt_download_mingw') download_packages_work = ThreadedWork("get and extract Qt") - download_packages_work.addTaskObject(create_qt_download_task(qt_module_urls, qt_dir, qt_temp, None)) - download_packages_work.addTaskObject(create_qt_download_task(qt_mingw_module_urls, qt_mingw_dir, qt_mingw_temp, None)) + download_packages_work.add_task_object(create_qt_download_task(qt_module_urls, qt_dir, qt_temp, None)) + download_packages_work.add_task_object(create_qt_download_task(qt_mingw_module_urls, qt_mingw_dir, qt_mingw_temp, None)) - download_packages_work.addTaskObject(create_download_extract_task( + download_packages_work.add_task_object(create_download_extract_task( 'https://download.sysinternals.com/files/DebugView.zip', debugview_dir, base_path)) @@ -202,7 +202,7 @@ def mingw_training(base_path, qtcreator_path, environment, bitness): cmake_arch_suffix = 'win64-x64' if bitness == 64 else 'win32-x86' cmake_base_url = 'http://' + pkg_server + '/packages/jenkins/cmake/' \ + cmake_version() + '/cmake-' + cmake_version() + '-' + cmake_arch_suffix + '.zip' - download_packages_work.addTaskObject(create_download_extract_task( + download_packages_work.add_task_object(create_download_extract_task( cmake_base_url, cmake_dir, base_path)) download_packages_work.run() @@ -243,10 +243,10 @@ def mingw_training(base_path, qtcreator_path, environment, bitness): '-S' + qtcreator_path, '-B' + creator_build_dir] - runCommand(qtc_cmake, creator_build_dir, environment) - runCommand([cmake_command, '--build', creator_build_dir], creator_build_dir, environment) - runCommand([cmake_command, '--install', creator_build_dir, '--prefix', creator_install_dir], creator_build_dir, environment) - runCommand([cmake_command, '--install', creator_build_dir, '--prefix', creator_install_dir, '--component', 'Dependencies'], creator_build_dir, environment) + run_command(qtc_cmake, creator_build_dir, environment) + run_command([cmake_command, '--build', creator_build_dir], creator_build_dir, environment) + run_command([cmake_command, '--install', creator_build_dir, '--prefix', creator_install_dir], creator_build_dir, environment) + run_command([cmake_command, '--install', creator_build_dir, '--prefix', creator_install_dir, '--component', 'Dependencies'], creator_build_dir, environment) # Remove the regular libclang.dll which got deployed via 'Dependencies' qtcreator install target os.remove(os.path.join(creator_install_dir, 'bin', 'libclang.dll')) @@ -254,7 +254,7 @@ def mingw_training(base_path, qtcreator_path, environment, bitness): # Train mingw libclang library with build QtCreator # First time open the project, then close it. This will generate initial settings and .user files. Second time do the actual training. for batchFile in ['qtc.openProject.batch', 'qtc.fileTextEditorCpp.batch']: - runCommand( + run_command( [os.path.join(training_dir, 'runBatchFiles.bat'), msvc_version(), 'x64' if bitness == 64 else 'x86', batchFile], base_path, extra_environment=None, onlyErrorCaseOutput=False, expectedExitCodes=[0, 1] ) @@ -434,14 +434,14 @@ def check_clang(toolchain, build_path, environment): def package_clang(install_path, result_file_path): (basepath, dirname) = os.path.split(install_path) zip_command = ['7z', 'a', '-mmt4', result_file_path, dirname] - runCommand(zip_command, basepath) + run_command(zip_command, basepath) def upload_clang(file_path, remote_path): (path, filename) = os.path.split(file_path) scp_bin = '%SCP%' if is_windows() else 'scp' scp_command = [scp_bin, filename, remote_path] - runCommand(scp_command, path) + run_command(scp_command, path) def profile_data(toolchain): diff --git a/packaging-tools/build_clang_qdoc.py b/packaging-tools/build_clang_qdoc.py index abcf529b8..b24984d60 100644 --- a/packaging-tools/build_clang_qdoc.py +++ b/packaging-tools/build_clang_qdoc.py @@ -32,18 +32,18 @@ import os import environmentfrombatchfile -from bld_utils import is_linux, is_windows, runCommand +from bld_utils import is_linux, is_windows, run_command from read_remote_config import get_pkg_value from runner import do_execute_sub_process def git_clone_and_checkout(base_path, remote_repository_url, directory, revision): - runCommand(['git', 'clone', - '--config', 'core.eol=lf', - '--config', 'core.autocrlf=input', - '--branch', revision, - '--recursive', - remote_repository_url, directory], base_path) + run_command(['git', 'clone', + '--config', 'core.eol=lf', + '--config', 'core.autocrlf=input', + '--branch', revision, + '--recursive', + remote_repository_url, directory], base_path) def get_clang(base_path, llvm_revision): @@ -237,14 +237,14 @@ def check_clang(toolchain, build_path, environment): def package_clang(install_path, result_file_path): (basepath, dirname) = os.path.split(install_path) zip_command = ['cmake', '-E', 'tar', 'cvf', result_file_path, '--format=7zip', dirname] - runCommand(zip_command, basepath) + run_command(zip_command, basepath) def upload_clang(file_path, remote_path): (path, filename) = os.path.split(file_path) scp_bin = '%SCP%' if is_windows() else 'scp' scp_command = [scp_bin, filename, remote_path] - runCommand(scp_command, path) + run_command(scp_command, path) def main(): diff --git a/packaging-tools/build_wrapper.py b/packaging-tools/build_wrapper.py index 4a041124e..4297036e4 100644 --- a/packaging-tools/build_wrapper.py +++ b/packaging-tools/build_wrapper.py @@ -50,11 +50,11 @@ from bld_sdktool import build_sdktool, zip_sdktool from bld_utils import ( download, file_url, - get_commit_SHA, + get_commit_sha, is_linux, is_macos, is_windows, - runCommand, + run_command, ) from bldinstallercommon import ( clone_repository, @@ -65,7 +65,7 @@ from bldinstallercommon import ( git_archive_repo, safe_config_key_fetch, ) -from optionparser import getPkgOptions +from optionparser import get_pkg_options from read_remote_config import get_pkg_value from runner import do_execute_sub_process from threadedwork import Task, ThreadedWork @@ -197,19 +197,19 @@ def create_download_documentation_task(base_url, download_path): dest_doc_path = os.path.join(download_path, 'doc') os.rename(source_path, dest_doc_path) # limit compression to 2 cores to limit memory footprint for 32bit Windows - runCommand(['7z', 'a', '-mx1', '-mmt2', '-md32m', '-ms=1g', target_filepath, dest_doc_path], - dest_doc_path) + run_command(['7z', 'a', '-mx1', '-mmt2', '-md32m', '-ms=1g', target_filepath, dest_doc_path], + dest_doc_path) download_task = Task(f"downloading documentation from {base_url}") for item in file_list: url = base_url + '/doc/' + item download_filepath = os.path.join(download_path, item) - download_task.addFunction(download, url, download_filepath) - download_task.addFunction(create_extract_function(download_filepath, extract_path)) - download_task.addFunction(create_remove_one_dir_level_function(os.path.join(extract_path, item.rstrip(".zip")))) + download_task.add_function(download, url, download_filepath) + download_task.add_function(create_extract_function(download_filepath, extract_path)) + download_task.add_function(create_remove_one_dir_level_function(os.path.join(extract_path, item.rstrip(".zip")))) repackage_task = Task(f"repackaging documentation as {target_filepath}") - repackage_task.addFunction(repackage) + repackage_task.add_function(repackage) return (download_task, repackage_task, file_url(target_filepath)) @@ -237,36 +237,36 @@ def create_download_openssl_task(url, download_path): else: source_path = linuxdir pattern = '*.so*' - runCommand(['7z', 'a', '-mmt2', target_filepath, pattern], - source_path) + run_command(['7z', 'a', '-mmt2', target_filepath, pattern], + source_path) download_task = Task(f"downloading openssl from {url}") - download_task.addFunction(download, url, download_filepath) + download_task.add_function(download, url, download_filepath) repackage_task = Task(f"repackaging openssl as {target_filepath}") - repackage_task.addFunction(create_extract_function(download_filepath, extract_path)) - repackage_task.addFunction(repackage) + repackage_task.add_function(create_extract_function(download_filepath, extract_path)) + repackage_task.add_function(repackage) return (download_task, repackage_task, file_url(target_filepath)) PluginConf = namedtuple('PluginConf', ['git_url', 'branch_or_tag', 'checkout_dir']) -def parseQtCreatorPlugins(pkgConfFile): +def parse_qtcreator_plugins(pkgConfFile): """Parse available Qt Creator plugins from configuration file""" pluginList = [] if not pkgConfFile: return pluginList - pluginOptions = getPkgOptions(pkgConfFile) + pluginOptions = get_pkg_options(pkgConfFile) sectionName = "QtCreator.Build.Plugin" keyName = "plugins" - if not pluginOptions.optionExists(sectionName, keyName): + if not pluginOptions.option_exists(sectionName, keyName): return pluginList - pluginConfList = pluginOptions.configSectionMap(sectionName)[keyName] + pluginConfList = pluginOptions.config_section_map(sectionName)[keyName] for pluginName in pluginConfList.replace(" ", "").replace("\n", "").split(","): section = "QtCreator.Build.Plugin." + pluginName - pluginUrl = pluginOptions.configSectionMap(section)["QTC_PLUGIN_GIT_URL"] - branchOrTag = pluginOptions.configSectionMap(section)["QTC_PLUGIN_GIT_BRANCH_OR_TAG"] - checkoutDirName = pluginOptions.configSectionMap(section)["QTC_PLUGIN_CHECKOUT_DIR_NAME"] + pluginUrl = pluginOptions.config_section_map(section)["QTC_PLUGIN_GIT_URL"] + branchOrTag = pluginOptions.config_section_map(section)["QTC_PLUGIN_GIT_BRANCH_OR_TAG"] + checkoutDirName = pluginOptions.config_section_map(section)["QTC_PLUGIN_CHECKOUT_DIR_NAME"] plugin = PluginConf(git_url=pluginUrl, branch_or_tag=branchOrTag, checkout_dir=checkoutDirName) pluginList.extend([plugin]) return pluginList @@ -282,7 +282,7 @@ QtcPlugin = namedtuple('QtcPlugin', ['name', 'package_commercial']) -def make_QtcPlugin(name, path, version, dependencies=None, modules=None, +def make_qtcplugin(name, path, version, dependencies=None, modules=None, additional_arguments=None, build=True, package_commercial=False): return QtcPlugin(name=name, path=path, version=version, @@ -426,7 +426,7 @@ def get_qtcreator_version(path_to_qtcreator_src, optionDict): return None -def make_QtcPlugin_from_json(plugin_json): +def make_qtcplugin_from_json(plugin_json): return QtcPlugin(name=plugin_json['Name'], path=plugin_json['Path'], version=plugin_json.get('Version'), @@ -457,7 +457,7 @@ def parse_qt_creator_plugin_conf(plugin_conf_file_path, optionDict): plugin = plugin._replace(modules=[module % optionDict for module in plugin.modules]) plugin = plugin._replace(additional_arguments=[arg % optionDict for arg in plugin.additional_arguments]) return plugin - return [fixup_plugin(make_QtcPlugin_from_json(plugin)) for plugin in plugins_json if valid_for_platform(plugin)] + return [fixup_plugin(make_qtcplugin_from_json(plugin)) for plugin in plugins_json if valid_for_platform(plugin)] def collect_qt_creator_plugin_sha1s(optionDict, plugins): @@ -631,8 +631,8 @@ def handle_qt_creator_build(optionDict, qtCreatorPlugins): def add_download_extract(url, target_path): (dl_task, extract) = create_download_and_extract_tasks( url, target_path, download_temp) - download_work.addTaskObject(dl_task) - extract_work.addFunction(extract.do) + download_work.add_task_object(dl_task) + extract_work.add_function(extract.do) # clang package use_optimized_libclang = False @@ -670,17 +670,17 @@ def handle_qt_creator_build(optionDict, qtCreatorPlugins): # We have to download, unpack, and repack renaming the toplevel directory. (dl_task, repackage, documentation_local_url) = create_download_documentation_task( pkg_base_path + '/' + qt_base_path, os.path.join(download_temp, 'qtdocumentation')) - download_work.addTaskObject(dl_task) - extract_work.addFunction(repackage.do) + download_work.add_task_object(dl_task) + extract_work.add_function(repackage.do) if openssl_libs: (dl_task, repackage, openssl_local_url) = create_download_openssl_task(openssl_libs, os.path.join(download_temp, 'openssl')) - download_work.addTaskObject(dl_task) - extract_work.addFunction(repackage.do) + download_work.add_task_object(dl_task) + extract_work.add_function(repackage.do) download_packages_work = Task('Get and extract all needed packages') - download_packages_work.addFunction(download_work.run) - download_packages_work.addFunction(extract_work.do) + download_packages_work.add_function(download_work.run) + download_packages_work.add_function(extract_work.do) download_packages_work.do() # copy optimized clang package @@ -770,22 +770,22 @@ def handle_qt_creator_build(optionDict, qtCreatorPlugins): if os.path.isdir(os.path.join(work_dir, "licensechecker")): add_args = ['--add-path', os.path.join(work_dir, 'license-managing')] - additional_plugins.extend([make_QtcPlugin('licensechecker', 'licensechecker', qtcreator_version, + additional_plugins.extend([make_qtcplugin('licensechecker', 'licensechecker', qtcreator_version, modules=qt_module_local_urls, additional_arguments=add_args, package_commercial=True)]) plugin_dependencies = ['licensechecker'] - additional_plugins.extend([make_QtcPlugin('vxworks-qtcreator-plugin', 'vxworks-qtcreator-plugin', qtcreator_version, + additional_plugins.extend([make_qtcplugin('vxworks-qtcreator-plugin', 'vxworks-qtcreator-plugin', qtcreator_version, modules=qt_module_local_urls, dependencies=plugin_dependencies, package_commercial=True)]) - additional_plugins.extend([make_QtcPlugin('isoiconbrowser', 'qtquickdesigner', qtcreator_version, + additional_plugins.extend([make_qtcplugin('isoiconbrowser', 'qtquickdesigner', qtcreator_version, modules=qt_module_local_urls, dependencies=plugin_dependencies, package_commercial=True)]) - additional_plugins.extend([make_QtcPlugin('gammarayintegration', 'gammarayintegration', qtcreator_version, + additional_plugins.extend([make_qtcplugin('gammarayintegration', 'gammarayintegration', qtcreator_version, modules=qt_module_local_urls + [kdsme_url, gammaray_url] + module_urls(['qt3d', 'qtgamepad']), dependencies=plugin_dependencies, additional_arguments=['--deploy'])]) - additional_plugins.extend([make_QtcPlugin('appmanagerintegration', 'pcore-plugin-appman', qtcreator_version, + additional_plugins.extend([make_qtcplugin('appmanagerintegration', 'pcore-plugin-appman', qtcreator_version, modules=qt_module_local_urls, dependencies=plugin_dependencies, additional_arguments=['--with-docs'])]), @@ -793,7 +793,7 @@ def handle_qt_creator_build(optionDict, qtCreatorPlugins): if usp_server_url and usp_auth_key: plugin_telemetry_args = ['--add-config=-DUSP_SERVER_URL=' + optionDict['USP_SERVER_URL'], '--add-config=-DUSP_AUTH_KEY=' + optionDict['USP_AUTH_KEY']] - additional_plugins.extend([make_QtcPlugin('plugin-telemetry', 'plugin-telemetry', qtcreator_version, + additional_plugins.extend([make_qtcplugin('plugin-telemetry', 'plugin-telemetry', qtcreator_version, modules=qt_module_local_urls, additional_arguments=plugin_telemetry_args)]), @@ -809,7 +809,7 @@ def handle_qt_creator_build(optionDict, qtCreatorPlugins): openssl_url=openssl_local_url, additional_config=qtc_additional_config, log_filepath=log_filepath) - qtcreator_sha = get_commit_SHA(qtcreator_source) + qtcreator_sha = get_commit_sha(qtcreator_source) with open(os.path.join(work_dir, 'QTC_SHA1'), 'w', encoding="utf-8") as f: f.write(qtcreator_sha + '\n') @@ -818,7 +818,7 @@ def handle_qt_creator_build(optionDict, qtCreatorPlugins): sha1s = collect_qt_creator_plugin_sha1s(optionDict, additional_plugins) licensemanaging_source = os.path.join(work_dir, 'license-managing') if os.path.exists(licensemanaging_source): - sha1s.append('license-managing: ' + get_commit_SHA(licensemanaging_source)) + sha1s.append('license-managing: ' + get_commit_sha(licensemanaging_source)) sha1s.append('qt-creator: ' + qtcreator_sha) with open(os.path.join(work_dir, 'SHA1'), 'w', encoding="utf-8") as f: f.writelines([sha + '\n' for sha in sha1s]) @@ -864,7 +864,7 @@ def handle_qt_creator_build(optionDict, qtCreatorPlugins): # notarize if is_macos() and notarize: - notarizeDmg(os.path.join(work_dir, 'qt-creator_build', 'qt-creator.dmg'), 'Qt Creator') + notarize_dmg(os.path.join(work_dir, 'qt-creator_build', 'qt-creator.dmg'), 'Qt Creator') # Upload file_upload_list = [] # pairs (source, dest), source relative to WORK_DIR, dest relative to server + dir_path @@ -990,7 +990,7 @@ def handle_sdktool_build(optionDict): python_url = optionDict.get('PYTHON_URL') if python_url: python_path = os.path.join(download_temp, 'python') - download_packages_work.addTaskObject(create_download_extract_task( + download_packages_work.add_task_object(create_download_extract_task( python_url, python_path, download_temp)) cmd_args.extend(['--python-path', python_path]) @@ -1004,7 +1004,7 @@ def handle_sdktool_build(optionDict): update_job_link(unversioned_base_path, base_path, optionDict) -def notarizeDmg(dmgPath, installer_name_base): +def notarize_dmg(dmgPath, installer_name_base): # bundle-id is just a unique identifier without any special meaning, used to track the notarization progress bundleId = installer_name_base + "-" + strftime('%Y-%m-%d', gmtime()) bundleId = bundleId.replace('_', '-').replace(' ', '') # replace illegal characters for bundleId @@ -1046,14 +1046,14 @@ def do_git_archive_repo(optionDict, repo_and_ref): do_execute_sub_process(cmd_args, SCRIPT_ROOT_DIR) -def initPkgOptions(args): - def mergeTwoDicts(x, y): +def init_pkg_options(args): + def merge_two_dicts(x, y): """Given two dicts, merge them into a new dict as a shallow copy.""" z = x.copy() z.update(y) return z - def getDefaultTargetEnv(): + def get_default_target_env(): """For local builds define default build target""" if is_windows(): return "win-msvc2015-Windows10-x64" @@ -1066,9 +1066,9 @@ def initPkgOptions(args): optionDict = {} # Are we using local conf file for pkg options? if args.pkg_conf_file: - options = getPkgOptions(args.pkg_conf_file) - optionDict = mergeTwoDicts(optionDict, options.configMap()) - optionDict['TARGET_ENV'] = args.target_env if args.target_env else getDefaultTargetEnv() + options = get_pkg_options(args.pkg_conf_file) + optionDict = merge_two_dicts(optionDict, options.config_map()) + optionDict['TARGET_ENV'] = args.target_env if args.target_env else get_default_target_env() optionDict['BUILD_NUMBER'] = str(strftime('%Y%m%d%H%M%S', gmtime())) optionDict['PACKAGE_STORAGE_SERVER_ADDR'] = optionDict['PACKAGE_STORAGE_SERVER_USER'] + '@' + optionDict['PACKAGE_STORAGE_SERVER'] else: @@ -1170,12 +1170,12 @@ def main() -> None: args = parser.parse_args(sys.argv[1:]) # Init configuration options first - optionDict = initPkgOptions(args) + optionDict = init_pkg_options(args) # Execute given command # QtCreator specific if args.command == bld_qtcreator: - handle_qt_creator_build(optionDict, parseQtCreatorPlugins(args.pkg_conf_file)) + handle_qt_creator_build(optionDict, parse_qtcreator_plugins(args.pkg_conf_file)) # sdktool elif args.command == bld_qtc_sdktool: handle_sdktool_build(optionDict) diff --git a/packaging-tools/create_installer.py b/packaging-tools/create_installer.py index cb36da8fc..8498dca8a 100644 --- a/packaging-tools/create_installer.py +++ b/packaging-tools/create_installer.py @@ -62,7 +62,7 @@ from bldinstallercommon import ( safe_config_key_fetch, ) from installer_utils import PackagingError -from patch_qt import patchFiles, patchQtEdition +from patch_qt import patch_files, patch_qt_edition from pkg_constants import INSTALLER_OUTPUT_DIR_NAME from runner import do_execute_sub_process from sdkcomponent import SdkComponent @@ -250,7 +250,7 @@ def parse_component_data(task, configuration_file, configurations_base_path): sdk_component = SdkComponent(section, configuration, task.packages_dir_name_list, task.archive_location_resolver, task.substitution_list, task.offline_installer) if task.dry_run: - sdk_component.setArchiveSkip(True) + sdk_component.set_archive_skip(True) # validate component sdk_component.validate() if sdk_component.is_valid(): @@ -404,7 +404,7 @@ def get_component_data(task, sdk_component, archive, install_dir, data_dir_dest, except PackagingError: pass if 'patch_qt' in archive.package_finalize_items: - patchFiles(install_dir, product='qt_framework') + patch_files(install_dir, product='qt_framework') if 'set_executable' in archive.package_finalize_items: handle_set_executable(install_dir, archive.package_finalize_items) if 'set_licheck' in archive.package_finalize_items: @@ -449,7 +449,7 @@ def get_component_data(task, sdk_component, archive, install_dir, data_dir_dest, def handle_set_executable(baseDir, packageFinalizeItems): - for item in parsePackageFinalizeItems(packageFinalizeItems, 'set_executable'): + for item in parse_package_finalize_items(packageFinalizeItems, 'set_executable'): expectedPath = os.path.join(baseDir, item) if not os.path.exists(expectedPath): raise CreateInstallerError(f'Can not set executable bit as path not found: "{expectedPath}"') @@ -458,16 +458,16 @@ def handle_set_executable(baseDir, packageFinalizeItems): def handle_set_licheck(task, baseDir, packageFinalizeItems): - for licheckFileName in parsePackageFinalizeItems(packageFinalizeItems, 'set_licheck'): + for licheckFileName in parse_package_finalize_items(packageFinalizeItems, 'set_licheck'): licheckFilePath = os.path.join(baseDir, licheckFileName) if not os.path.exists(licheckFilePath): raise CreateInstallerError(f'Can not set licheck as path not found: "{licheckFilePath}"') - patchQtEdition(baseDir, licheckFileName, task.build_timestamp) + patch_qt_edition(baseDir, licheckFileName, task.build_timestamp) log.info("Licheck set for: %s", licheckFilePath) break -def parsePackageFinalizeItems(packageFinalizeItems, itemCategory): +def parse_package_finalize_items(packageFinalizeItems, itemCategory): for item in packageFinalizeItems.split(","): if itemCategory not in item: continue @@ -615,13 +615,13 @@ def create_target_components(task): if is_windows(): install_dir = win32api.GetShortPathName(install_dir) data_dir_dest = win32api.GetShortPathName(data_dir_dest) - getComponentDataWork.addTask(f"adding {archive.archive_name} to {sdk_component.package_name}", - get_component_data, task, sdk_component, archive, install_dir, data_dir_dest, compress_content_dir) + getComponentDataWork.add_task(f"adding {archive.archive_name} to {sdk_component.package_name}", + get_component_data, task, sdk_component, archive, install_dir, data_dir_dest, compress_content_dir) # handle component sha1 uri if sdk_component.component_sha1_uri: sha1_file_dest = os.path.normpath(dest_base + 'SHA1') - getComponentDataWork.addTask(f"getting component sha1 file for {sdk_component.package_name}", - get_component_sha1_file, sdk_component, sha1_file_dest) + getComponentDataWork.add_task(f"getting component sha1 file for {sdk_component.package_name}", + get_component_sha1_file, sdk_component, sha1_file_dest) # maybe there is some static data data_content_source_root = os.path.normpath(sdk_component.pkg_template_dir + os.sep + 'data') diff --git a/packaging-tools/install_qt.py b/packaging-tools/install_qt.py index d581581fc..4f1f10935 100644 --- a/packaging-tools/install_qt.py +++ b/packaging-tools/install_qt.py @@ -78,7 +78,7 @@ def install_qt(args): download_packages_work = ThreadedWork('get and extract Qt 5 binaries') need_to_install_qt = not os.path.lexists(args.qt_path) if need_to_install_qt: - download_packages_work.addTaskObject(create_qt_download_task( + download_packages_work.add_task_object(create_qt_download_task( args.qt_modules, args.qt_path, args.temp_path, args)) # run task if needed diff --git a/packaging-tools/libclang_training/libclangtimings2csv.py b/packaging-tools/libclang_training/libclangtimings2csv.py index d406698f1..42ec1a86d 100644 --- a/packaging-tools/libclang_training/libclangtimings2csv.py +++ b/packaging-tools/libclang_training/libclangtimings2csv.py @@ -40,7 +40,7 @@ import re import sys -def constructRecordMatcher(): +def construct_record_matcher(): regex = ( '( Parsing)' + '|( Precompiling preamble)' @@ -52,7 +52,7 @@ def constructRecordMatcher(): return re.compile(regex) -def constructTimeNeededMatcher(): +def construct_time_needed_matcher(): # An output line looks like: # : 2.5625 (100.0%) 0.1563 (100.0%) 2.7188 (100.0%) 2.7813 (100.0%) # Note: There is always at least the wall clock time at the utmost right, @@ -66,13 +66,13 @@ def constructTimeNeededMatcher(): return re.compile(regex) -def csvLine(values): +def csv_line(values): return ','.join(values) + '\n' -def extractRecords(fileContent): - recordMatcher = constructRecordMatcher() - timeNeededMatcher = constructTimeNeededMatcher() +def extract_records(fileContent): + recordMatcher = construct_record_matcher() + timeNeededMatcher = construct_time_needed_matcher() records = [] previousTimeMatchEnd = -1 @@ -98,10 +98,10 @@ def extractRecords(fileContent): return records -def recordsToString(records): +def records_to_string(records): string = "" for record in records: - string += csvLine(record) + string += csv_line(record) return string @@ -110,12 +110,12 @@ def convert(inputFile, columnLabel=None): if not columnLabel: columnLabel = os.path.basename(inputFile) with open(inputFile, 'r', encoding="utf-8") as fileContent: - records = [[columnLabel, columnLabel]] + extractRecords(fileContent.read()) + records = [[columnLabel, columnLabel]] + extract_records(fileContent.read()) - return recordsToString(records) + return records_to_string(records) -def printUsageAndExit(): +def print_usage_and_exit(): print(__doc__) sys.exit(0) @@ -132,11 +132,11 @@ def main(): # process options for o, _ in opts: if o in ("-h", "--help"): - printUsageAndExit() + print_usage_and_exit() # process arguments if not args: - printUsageAndExit() + print_usage_and_exit() for arg in args: print(convert(arg)) diff --git a/packaging-tools/libclang_training/mergeCsvFiles.py b/packaging-tools/libclang_training/mergeCsvFiles.py index 0dd3ac32c..746bb5b06 100644 --- a/packaging-tools/libclang_training/mergeCsvFiles.py +++ b/packaging-tools/libclang_training/mergeCsvFiles.py @@ -47,7 +47,7 @@ class FileWithValues: self.values = values -def readCsv(filePath, delimiter): +def read_csv(filePath, delimiter): lines = [] with open(filePath, 'rt', encoding="utf-8") as f: lines = f.readlines() @@ -62,7 +62,7 @@ def readCsv(filePath, delimiter): return records -def readCsvFiles(filePaths): +def read_csv_files(filePaths): files = [] for filePath in filePaths: @@ -82,7 +82,7 @@ def readCsvFiles(filePaths): return files -def checkConsistency(files): +def check_consistency(files): referenceEntry = files[0] referenceEntrySize = len(referenceEntry.values) referenceEntryIdentifiers = [v[0] for v in referenceEntry.values] @@ -103,7 +103,7 @@ def checkConsistency(files): return referenceEntryIdentifiers -def mergeFilesHelper(outputFilePath, referenceIdentifiers, files): +def merge_files_helper(outputFilePath, referenceIdentifiers, files): with open(outputFilePath, 'wt', encoding="utf-8") as csvfile: writer = csv.writer(csvfile, delimiter=Global.Delimiter, quotechar='"', quoting=csv.QUOTE_MINIMAL) @@ -118,13 +118,13 @@ def mergeFilesHelper(outputFilePath, referenceIdentifiers, files): writer.writerow(row) -def mergeFiles(outputFilePath, filesToMerge): - files = readCsvFiles(filesToMerge) - referenceIdentifiers = checkConsistency(files) - mergeFilesHelper(outputFilePath, referenceIdentifiers, files) +def merge_files(outputFilePath, filesToMerge): + files = read_csv_files(filesToMerge) + referenceIdentifiers = check_consistency(files) + merge_files_helper(outputFilePath, referenceIdentifiers, files) -def printHelpAndExit(): +def print_help_and_exit(): print(__doc__) sys.exit(0) @@ -139,13 +139,13 @@ def main(): for o, _ in opts: if o in ("-h", "--help"): - printHelpAndExit() + print_help_and_exit() if len(args) <= 2: - printHelpAndExit() + print_help_and_exit() outputFile = args[0] filesToMerge = args[1:] - mergeFiles(outputFile, filesToMerge) + merge_files(outputFile, filesToMerge) if __name__ == "__main__": diff --git a/packaging-tools/libclang_training/runBatchFiles.py b/packaging-tools/libclang_training/runBatchFiles.py index c15de9321..7d38c6e2c 100644 --- a/packaging-tools/libclang_training/runBatchFiles.py +++ b/packaging-tools/libclang_training/runBatchFiles.py @@ -66,18 +66,18 @@ import libclangtimings2csv import mergeCsvFiles -def verboseStart(args): +def verbose_start(args): if Config.Verbose: print(f"info: starting {args}") -def checkExistenceOrDie(filePath): +def check_existence_or_die(filePath): if not os.path.exists(filePath): print(f"error: file path does not exist: {filePath}", file=sys.stderr) sys.exit(1) -def checkExitCodeOrDie(exitCode, args): +def check_exit_code_or_die(exitCode, args): if exitCode != 0: print(f"error: exit code is, {exitCode} for", ' '.join(args), file=sys.stderr) sys.exit(1) @@ -95,12 +95,12 @@ class Config: BatchFiles: List[str] = [] @staticmethod - def initializeFromEnvironment(): + def initialize_from_environment(): Config.LogDir = os.environ['QTC_CLANG_BATCH_CONFIG_LOG_DIR'] - checkExistenceOrDie(Config.LogDir) + check_existence_or_die(Config.LogDir) Config.QtCreatorSettingsDir = os.environ['QTC_CLANG_BATCH_CONFIG_SETTINGS_DIR'] - checkExistenceOrDie(Config.QtCreatorSettingsDir) + check_existence_or_die(Config.QtCreatorSettingsDir) Config.TargetLibClangDll = os.environ['QTC_CLANG_BATCH_CONFIG_TARGET_LIBCLANG'] @@ -108,13 +108,13 @@ class Config: Config.LibClangDlls = libClangDlls.split(os.pathsep) assert len(Config.LibClangDlls) >= 1 for dll in Config.LibClangDlls: - checkExistenceOrDie(dll) + check_existence_or_die(dll) batchFiles = os.environ['QTC_CLANG_BATCH_CONFIG_FILES'] Config.BatchFiles = batchFiles.split(os.pathsep) assert len(Config.BatchFiles) >= 1 for b in Config.BatchFiles: - checkExistenceOrDie(b) + check_existence_or_die(b) # TODO: Check for format @staticmethod @@ -147,9 +147,9 @@ class DebugView: self.logFilePath = logFilePath self.executable = 'dbgview.exe' - def startAsync(self): + def start_async(self): args = [self.executable, '/accepteula', '/l', self.logFilePath] - verboseStart(args) + verbose_start(args) self.proc = Popen(args, shell=False) # pylint: disable=R1732 sleep(2) @@ -161,7 +161,7 @@ class DebugView: self.proc.wait() -def createEnvironment(batchFilePath): +def create_environment(batchFilePath): env = os.environ.copy() env['LIBCLANG_TIMING'] = '1' env['QT_LOGGING_RULES'] = 'qtc.clangcodemodel.batch=true' @@ -174,28 +174,28 @@ def createEnvironment(batchFilePath): return env -def runSyncAndLogOutputWindows(args, batchFilePath, logFilePath): +def run_sync_and_log_output_windows(args, batchFilePath, logFilePath): debugView = DebugView(logFilePath) - debugView.startAsync() + debugView.start_async() - verboseStart(args) - with Popen(args, env=createEnvironment(batchFilePath)) as p: + verbose_start(args) + with Popen(args, env=create_environment(batchFilePath)) as p: p.communicate() debugView.stop() - checkExitCodeOrDie(p.returncode, args) + check_exit_code_or_die(p.returncode, args) -def runSyncAndLogOutputUnix(args, batchFilePath, logFilePath): +def run_sync_and_log_output_unix(args, batchFilePath, logFilePath): with open(logFilePath, "w", encoding="utf-8") as logFile: - verboseStart(args) - with Popen(args, stdout=logFile, stderr=STDOUT, env=createEnvironment(batchFilePath)) as p: + verbose_start(args) + with Popen(args, stdout=logFile, stderr=STDOUT, env=create_environment(batchFilePath)) as p: p.communicate() - checkExitCodeOrDie(p.returncode, args) + check_exit_code_or_die(p.returncode, args) -def runQtCreatorWithBatchFile(batchFilePath, logFilePath): +def run_qtcreator_with_batch_file(batchFilePath, logFilePath): args = [ 'qtcreator', '-noload', 'all', @@ -208,12 +208,12 @@ def runQtCreatorWithBatchFile(batchFilePath, logFilePath): ] if sys.platform == "win32": - runSyncAndLogOutputWindows(args, batchFilePath, logFilePath) + run_sync_and_log_output_windows(args, batchFilePath, logFilePath) else: - runSyncAndLogOutputUnix(args, batchFilePath, logFilePath) + run_sync_and_log_output_unix(args, batchFilePath, logFilePath) -def convertLogFileToCsvFile(logFilePath, columnLabel): +def convert_log_file_to_csv_file(logFilePath, columnLabel): output = libclangtimings2csv.convert(logFilePath, columnLabel) csvFilePath = logFilePath + '.csv' @@ -223,18 +223,18 @@ def convertLogFileToCsvFile(logFilePath, columnLabel): return csvFilePath -def logFileFromId(logFileId): +def log_file_from_id(logFileId): return logFileId + ".log" -def createDir(dirPath): +def create_dir(dirPath): if not os.path.exists(dirPath): if Config.Verbose: print(f"info: creating not existent {dirPath}") os.makedirs(dirPath) -def createBackupFile(filePath): +def create_backup_file(filePath): if os.path.exists(filePath): backupPath = filePath[:-4] + ".backup_" + str(time()) + ".log" if Config.Verbose: @@ -242,71 +242,71 @@ def createBackupFile(filePath): copyfile(filePath, backupPath) -def printDuration(s): +def print_duration(s): hours, remainder = divmod(s, 3600) minutes, seconds = divmod(remainder, 60) print(f"...needed {hours}:{minutes}:{seconds}") -def processBatchFileTimed(libClangId, batchFilePath): +def process_batch_file_timed(libClangId, batchFilePath): timeStarted = time() print(f"processing {batchFilePath}", end=' ') - runRecord = processBatchFile(libClangId, batchFilePath) + runRecord = process_batch_file(libClangId, batchFilePath) - printDuration(time() - timeStarted) + print_duration(time() - timeStarted) return runRecord -def processBatchFile(libClangId, batchFilePath): +def process_batch_file(libClangId, batchFilePath): runRecord = RunRecord(libClangId, batchFilePath) logFilePath = os.path.join(Config.LogDir, runRecord.logFilePath) - createDir(Config.LogDir) - createBackupFile(logFilePath) + create_dir(Config.LogDir) + create_backup_file(logFilePath) - runQtCreatorWithBatchFile(batchFilePath, logFilePath) + run_qtcreator_with_batch_file(batchFilePath, logFilePath) - csvFilePath = convertLogFileToCsvFile(logFilePath, runRecord.libClangId) + csvFilePath = convert_log_file_to_csv_file(logFilePath, runRecord.libClangId) runRecord.csvFilePath = csvFilePath return runRecord -def getLibClangId(libClangDll): +def get_libclang_id(libClangDll): fileName = os.path.basename(libClangDll) parts = fileName.split('.') identifier = '.'.join(parts[0:-1]) return identifier -def switchLibClang(libClangDll): +def switch_libclang(libClangDll): print(f"copying '{libClangDll}' -> '{Config.TargetLibClangDll}'") copyfile(libClangDll, Config.TargetLibClangDll) -def runQtCreatorWithLibClang(libClangDll): +def run_qtcreator_with_libclang(libClangDll): print("") - switchLibClang(libClangDll) + switch_libclang(libClangDll) runRecords = [] - libClangId = getLibClangId(libClangDll) + libClangId = get_libclang_id(libClangDll) for batchFile in Config.BatchFiles: - runRecord = processBatchFileTimed(libClangId, batchFile) + runRecord = process_batch_file_timed(libClangId, batchFile) runRecords.append(runRecord) return runRecords -def logIdPartFromLibClangDll(libClangDll): +def log_id_part_from_libclang_dll(libClangDll): fileName = os.path.basename(libClangDll) parts = fileName.split('.') fileName = '.'.join(parts[1:-1]) return fileName -def mergeGeneratedCsvFiles(runRecords): +def merge_generated_csv_files(runRecords): batchFileId2RunRecord = {} for rr in runRecords: newValue = [rr] @@ -319,20 +319,20 @@ def mergeGeneratedCsvFiles(runRecords): csvFilePaths = [rr.csvFilePath for rr in batchFileId2RunRecord[batchFileId]] mergeFilePath = os.path.join(Config.LogDir, batchFileId + ".csv") - mergeCsvFiles.mergeFiles(mergeFilePath, csvFilePaths) + mergeCsvFiles.merge_files(mergeFilePath, csvFilePaths) print(f"generated: {mergeFilePath}") def main(): - Config.initializeFromEnvironment() + Config.initialize_from_environment() Config.dump() runRecords = [] for libClangDll in Config.LibClangDlls: - runRecords += runQtCreatorWithLibClang(libClangDll) + runRecords += run_qtcreator_with_libclang(libClangDll) print() - mergeGeneratedCsvFiles(runRecords) + merge_generated_csv_files(runRecords) if __name__ == "__main__": diff --git a/packaging-tools/notarize.py b/packaging-tools/notarize.py index 0ff908600..8692dcc87 100755 --- a/packaging-tools/notarize.py +++ b/packaging-tools/notarize.py @@ -58,14 +58,14 @@ class NotarizationError(Exception): pass -def parseValueFromData(key, data): +def parse_value_from_data(key, data): for line in data.split("\n"): if line.strip().startswith(key): return line.split(key)[-1].strip() return "" -async def requestCmd(args, cmd): +async def request_cmd(args, cmd): p = await asyncio.create_subprocess_exec(*cmd, stdout=asyncio.subprocess.PIPE, stderr=STDOUT) attempts = 3 @@ -89,26 +89,26 @@ async def requestCmd(args, cmd): return data[0].decode('utf-8') -async def requestNotarization(args): +async def request_notarization(args): # long lasting command, it uploads the binary to Apple server cmd = ['xcrun', 'altool', '-u', args.user, '-p', args.passwd, '--notarize-app', '-t', 'osx'] cmd += ['--primary-bundle-id', args.bundle_id, '-f', args.dmg] - data = await requestCmd(args, cmd) - requestUUID = parseValueFromData("RequestUUID", data) + data = await request_cmd(args, cmd) + requestUUID = parse_value_from_data("RequestUUID", data) if not requestUUID: raise NotarizationError(f"Failed to notarize app:\n\n{data}") return requestUUID.split("=")[-1].strip() -async def pollNotarizationCompleted(args, uuid): +async def poll_notarization_completed(args, uuid): cmd = ['xcrun', 'altool', '-u', args.user, '-p', args.passwd, '--notarization-info', uuid] attempts = 180 pollInterval = 60 # attempts * pollInterval = 3h while attempts: - data = await requestCmd(args, cmd) - statusCode = parseValueFromData("Status Code:", data) + data = await request_cmd(args, cmd) + statusCode = parse_value_from_data("Status Code:", data) if statusCode == "0": log.info("Notarization succeeded for: %s", args.dmg) @@ -128,15 +128,15 @@ async def pollNotarizationCompleted(args, uuid): return False -async def embedNotarization(args): +async def embed_notarization(args): # Embed the notarization in the dmg package cmd = ['xcrun', 'stapler', 'staple', args.dmg] retry_count = 10 delay = 60 while retry_count: retry_count -= 1 - data = await requestCmd(args, cmd) - status = parseValueFromData("The staple and validate action", data) + data = await request_cmd(args, cmd) + status = parse_value_from_data("The staple and validate action", data) if status.lower().startswith("worked"): log.info("The [%s] was notirized successfully!", args.dmg) @@ -154,10 +154,10 @@ async def embedNotarization(args): async def notarize(args): - uuid = await requestNotarization(args) - if not await pollNotarizationCompleted(args, uuid): + uuid = await request_notarization(args) + if not await poll_notarization_completed(args, uuid): raise NotarizationError(f"Notarization failed for: {args.dmg}") - await embedNotarization(args) + await embed_notarization(args) def main() -> None: diff --git a/packaging-tools/optionparser.py b/packaging-tools/optionparser.py index a03eb8864..6aad9fe9f 100644 --- a/packaging-tools/optionparser.py +++ b/packaging-tools/optionparser.py @@ -43,7 +43,7 @@ class PackagingOptions: self.config.optionxform = str self.config.read(confFile) - def configSectionMap(self, section): + def config_section_map(self, section): dict1 = {} options = self.config.options(section) for option in options: @@ -56,16 +56,16 @@ class PackagingOptions: dict1[option] = None return dict1 - def sectionExists(self, section): + def section_exists(self, section): return self.config.has_section(section) - def optionExists(self, section, option): - return self.sectionExists(section) and self.config.has_option(section, option) + def option_exists(self, section, option): + return self.section_exists(section) and self.config.has_option(section, option) - def configMap(self): + def config_map(self): dict1 = {} for section in self.config.sections(): - dict1.update(self.configSectionMap(section)) + dict1.update(self.config_section_map(section)) return dict1 def verbose(self): @@ -77,7 +77,7 @@ class PackagingOptions: print() -def getPkgOptions(confFilePath): +def get_pkg_options(confFilePath): return PackagingOptions(confFilePath) @@ -88,9 +88,9 @@ def main() -> None: help="Absolute path pointing into configuration file which contains all required options for packaging.") args = parser.parse_args(sys.argv[1:]) # Print out all options - options = getPkgOptions(args.conf_file) - configMap = options.configMap() - print(configMap) + options = get_pkg_options(args.conf_file) + config_map = options.config_map() + print(config_map) if __name__ == '__main__': diff --git a/packaging-tools/patch_qt.py b/packaging-tools/patch_qt.py index 59046dd7d..113d9db56 100755 --- a/packaging-tools/patch_qt.py +++ b/packaging-tools/patch_qt.py @@ -34,37 +34,37 @@ import re from fileinput import FileInput -def _fileIterator(artifactsDir): +def _file_iterator(artifactsDir): print(f"Patching build time paths from: {artifactsDir}") for root, _, files in os.walk(artifactsDir): for fileName in files: yield os.path.join(os.path.join(root, fileName)) -def _getPatchers(product): +def _get_patchers(product): if product == 'qt_framework': - return [patchAbsoluteLibPathsFromFile, eraseQmakePrlBuildDir, patchQConfigPri] + return [patch_absolute_lib_paths_from_file, erase_qmake_prl_build_dir, patch_qconfig_pri] # default - return [patchAbsoluteLibPathsFromFile, eraseQmakePrlBuildDir] + return [patch_absolute_lib_paths_from_file, erase_qmake_prl_build_dir] -def patchFiles(artifactsDir, product): +def patch_files(artifactsDir, product): print(f"Patching files from: {artifactsDir}") - patchers = _getPatchers(product) - for filePath in _fileIterator(artifactsDir): + patchers = _get_patchers(product) + for filePath in _file_iterator(artifactsDir): for patcher in patchers: patcher(filePath) -def patchQtEdition(artifactsDir, licheckFileName, releaseDate): +def patch_qt_edition(artifactsDir, licheckFileName, releaseDate): for root, _, files in os.walk(artifactsDir): for fileName in files: if fileName == 'qconfig.pri': - _patchQtEdition(os.path.join(root, fileName), licheckFileName, releaseDate) + _patch_qt_edition(os.path.join(root, fileName), licheckFileName, releaseDate) return -def _patchQtEdition(filePath, licheckFileName, releaseDate): +def _patch_qt_edition(filePath, licheckFileName, releaseDate): for line in FileInput(filePath, inplace=True): if 'QT_EDITION' in line: edition_line = 'QT_EDITION = Enterprise' @@ -77,13 +77,13 @@ def _patchQtEdition(filePath, licheckFileName, releaseDate): print(line.rstrip('\n')) -def patchQConfigPri(filePath): +def patch_qconfig_pri(filePath): for line in FileInput(filePath, inplace=True): - patchedLine = patchQConfigPriFromLine(line) + patchedLine = patch_qconfig_pri_from_line(line) print(patchedLine.rstrip('\n')) -def patchQConfigPriFromLine(line): +def patch_qconfig_pri_from_line(line): if 'QMAKE_DEFAULT_LIBDIRS' in line: return line.split('=')[0].strip() + ' =' if 'QMAKE_DEFAULT_INCDIRS' in line: @@ -91,24 +91,24 @@ def patchQConfigPriFromLine(line): return line -def eraseQmakePrlBuildDir(filePath): +def erase_qmake_prl_build_dir(filePath): # Erase lines starting with 'QMAKE_PRL_BUILD_DIR' from .prl files for line in FileInput(filePath, inplace=True): - patchedLine = patchQmakePrlBuildDirFromLine(line) + patchedLine = patch_qmake_prl_build_dir_from_line(line) print(patchedLine.rstrip('\n')) -def patchQmakePrlBuildDirFromLine(line): +def patch_qmake_prl_build_dir_from_line(line): return '' if line.startswith('QMAKE_PRL_BUILD_DIR') else line -def patchAbsoluteLibPathsFromFile(filePath): +def patch_absolute_lib_paths_from_file(filePath): for line in FileInput(filePath, inplace=True): - patchedLine = patchAbsoluteLibPathsFromLine(line, filePath.split(".")[-1]) + patchedLine = patch_absolute_lib_paths_from_line(line, filePath.split(".")[-1]) print(patchedLine.rstrip('\n')) -def patchAbsoluteLibPathsFromLine(line, fileExtension): +def patch_absolute_lib_paths_from_line(line, fileExtension): r""" Captures XXX in e.g. /usr/lib/libXXX.so, /usr/lib64/libXXX.a, and C:\XXX.lib Paths are not allowed to contain whitespace though @@ -119,7 +119,7 @@ def patchAbsoluteLibPathsFromLine(line, fileExtension): (\.[0-9]+)? - capture group for for versioned libraries """ - def _removeWhiteSpace(line): + def _remove_whitespace(line): """Remove white space from paths if found inside quoted blocks.""" eraseEnabled = False result = "" @@ -143,7 +143,7 @@ def patchAbsoluteLibPathsFromLine(line, fileExtension): re.compile(r'[^\s\"]+[\\/]([a-zA-Z0-9\_\-\.\+]+)\.(lib)(\.[0-9]+)?\b') ] - def _substituteLib(match): + def _substitute_lib(match): if match.group(0).startswith("$$[QT_"): return match.group(0) result = "" if fileExtension == "cmake" else "-l" # .pri, .prl, .la, .pc @@ -153,8 +153,8 @@ def patchAbsoluteLibPathsFromLine(line, fileExtension): for regex in expressions: # check if there are any matches? if re.search(regex, line): - line = _removeWhiteSpace(line) - line = regex.sub(_substituteLib, line) + line = _remove_whitespace(line) + line = regex.sub(_substitute_lib, line) break return line diff --git a/packaging-tools/remote_uploader.py b/packaging-tools/remote_uploader.py index 462df65c1..09d5f5bbe 100755 --- a/packaging-tools/remote_uploader.py +++ b/packaging-tools/remote_uploader.py @@ -68,15 +68,15 @@ class RemoteUploader: self.remoteTargetDir = self.remoteTargetBaseDir + "/" + projectName + "/" + version + "/" + snapshotId self.remoteLatestLink = self.remoteTargetBaseDir + "/" + projectName + "/" + version + "/latest" self.init_finished = True - self.ensureRemoteDir(self.remoteTargetDir) + self.ensure_remote_dir(self.remoteTargetDir) def init_upload_path(self, remotePath): assert not self.init_finished, f"Already initialized as: {self.remoteTargetDir}" self.remoteTargetDir = self.remoteTargetBaseDir + "/" + remotePath self.init_finished = True - self.ensureRemoteDir(self.remoteTargetDir) + self.ensure_remote_dir(self.remoteTargetDir) - def ensureRemoteDir(self, remoteDir): + def ensure_remote_dir(self, remoteDir): assert self.init_finished, "RemoteUploader not initialized!" print(f"Creating remote directory: {remoteDir}") cmd = self.ssh_cmd + ['mkdir', '-p', remoteDir] @@ -84,26 +84,26 @@ class RemoteUploader: if not self.dryRun: check_call(cmd, timeout=60) # give it 60s - def _copyToRemote(self, fileName, destDirName): + def _copy_to_remote(self, fileName, destDirName): """Copy the given file to destDirName which is relative to remoteBasePath.""" assert self.init_finished, "RemoteUploader not initialized!" remoteDestination = self.remoteLogin + ':' + self.remoteTargetDir if destDirName: remoteDestination = remoteDestination + '/' + destDirName + '/' if "windows" in platform.system().lower(): - self.ensureRemoteDir(self.remoteTargetDir + '/' + destDirName + '/') + self.ensure_remote_dir(self.remoteTargetDir + '/' + destDirName + '/') print(f"Copying [{fileName}] to [{remoteDestination}]") cmd = self.copy_cmd + [fileName, remoteDestination] print("Executing: ", ' '.join(cmd)) if not self.dryRun: check_call(cmd, timeout=60 * 10) # give it 10 mins - def copyToRemote(self, path: str, destDirName=""): + def copy_to_remote(self, path: str, destDirName=""): items = [path] if os.path.isfile(path) else [os.path.join(path, x) for x in os.listdir(path)] for item in items: - self._copyToRemote(item, destDirName) + self._copy_to_remote(item, destDirName) - def updateLatestSymlink(self, forceUpdate=True): + def update_latest_symlink(self, forceUpdate=True): assert self.init_finished, "RemoteUploader not initialized!" print(f"Creating remote symlink: [{self.remoteLatestLink}] -> [{self.remoteTargetDir}]") options = ["-sfn"] if forceUpdate else ["-sn"] @@ -134,8 +134,8 @@ def main() -> None: uploader = RemoteUploader(args.dry_run, args.remote_server, args.remote_server_user, args.remote_server_base_path) uploader.init_snapshot_upload_path(args.project_name, args.project_version, args.project_snapshot_id) - uploader.copyToRemote(args.source, args.subdir_name) - uploader.updateLatestSymlink() + uploader.copy_to_remote(args.source, args.subdir_name) + uploader.update_latest_symlink() if __name__ == "__main__": diff --git a/packaging-tools/sdkcomponent.py b/packaging-tools/sdkcomponent.py index 542500285..1a846bf8d 100644 --- a/packaging-tools/sdkcomponent.py +++ b/packaging-tools/sdkcomponent.py @@ -138,7 +138,7 @@ class SdkComponent: return True return False - def setArchiveSkip(self, doSkip): + def set_archive_skip(self, doSkip): self.archive_skip = doSkip def validate(self): diff --git a/packaging-tools/tests/test_installer_utils.py b/packaging-tools/tests/test_installer_utils.py index d22a174ac..3eef30da9 100644 --- a/packaging-tools/tests/test_installer_utils.py +++ b/packaging-tools/tests/test_installer_utils.py @@ -47,7 +47,7 @@ from read_remote_config import get_pkg_value from tests.testhelpers import ( asyncio_test, asyncio_test_parallel_data, - isInternalFileServerReachable, + is_internal_file_server_reachable, ) @@ -111,7 +111,7 @@ class TestInstallerUtils(unittest.TestCase): await extract_archive(tarArchivePath, destDir) self.assertTrue(os.path.isfile(os.path.join(destDir, tempPath, "foobar.txt"))) - @unittest.skipUnless(isInternalFileServerReachable(), + @unittest.skipUnless(is_internal_file_server_reachable(), "Skipping because file server is not accessible") @asyncio_test async def test_download_archive(self) -> None: diff --git a/packaging-tools/tests/test_packaging.py b/packaging-tools/tests/test_packaging.py index 5fba2b303..541ff6de4 100755 --- a/packaging-tools/tests/test_packaging.py +++ b/packaging-tools/tests/test_packaging.py @@ -37,19 +37,19 @@ from fileinput import FileInput from shutil import rmtree from tempfile import mkdtemp -from create_installer import parsePackageFinalizeItems +from create_installer import parse_package_finalize_items from patch_qt import ( - patchAbsoluteLibPathsFromLine, - patchQConfigPriFromLine, - patchQmakePrlBuildDirFromLine, - patchQtEdition, + patch_absolute_lib_paths_from_line, + patch_qconfig_pri_from_line, + patch_qmake_prl_build_dir_from_line, + patch_qt_edition, ) from runner import do_execute_sub_process class TestPackaging(unittest.TestCase): - def test_patchAbsoluteLibPaths(self): + def test_patch_absolute_lib_paths(self): testData = (("QMAKE_LIBS_ZLIB = /opt/android/android-ndk-r18b/platforms/android-21/arch-arm64/usr/lib/libz.so", "QMAKE_LIBS_ZLIB = -lz", "pri"), @@ -85,40 +85,40 @@ class TestPackaging(unittest.TestCase): "prl")) for data in testData: - result = patchAbsoluteLibPathsFromLine(data[0], data[2]) + result = patch_absolute_lib_paths_from_line(data[0], data[2]) self.assertEqual(result, data[1], f"Failed to patch: [{data[0]}] as: [{data[1]}]") - def test_patchQmakePrlBuildDirFromLine(self): + def test_patch_qmake_prl_build_dir_from_line(self): testData = (("QMAKE_PRL_BUILD_DIR = /foo/bar", ""), ("QMAKE_PRL_BUILD_DIR= /foo/bar", ""), ("foo bar = /foo/bar", "foo bar = /foo/bar")) for data in testData: - result = patchQmakePrlBuildDirFromLine(data[0]) + result = patch_qmake_prl_build_dir_from_line(data[0]) self.assertEqual(result, data[1], f"Failed to patch: [{data[0]}] as: [{data[1]}]") - def test_patchQConfigPriFromLine(self): + def test_patch_qconfig_pri_from_line(self): testData = (("QMAKE_DEFAULT_LIBDIRS = /foo/bar", "QMAKE_DEFAULT_LIBDIRS ="), ("QMAKE_DEFAULT_INCDIRS = /foo/bar", "QMAKE_DEFAULT_INCDIRS ="), ("foo bar = /foo/bar", "foo bar = /foo/bar")) for data in testData: - result = patchQConfigPriFromLine(data[0]) + result = patch_qconfig_pri_from_line(data[0]) self.assertEqual(result, data[1], f"Failed to patch: [{data[0]}] as: [{data[1]}]. Got: [{result}]") - def test_parsePackageFinalizeItems(self): + def test_parse_package_finalize_items(self): testData = (("set_executable=licheck64, foo=bar, set_executable=something", "set_executable", ["licheck64", "something"]), ("set_executable=licheck64,foo=bar, set_executable = something", "set_executable", ["licheck64", "something"]), ("set_executable=licheck64", "set_executable", ["licheck64"])) for data in testData: matchCount = 0 - for item in parsePackageFinalizeItems(data[0], data[1]): + for item in parse_package_finalize_items(data[0], data[1]): self.assertIn(item, data[2]) matchCount += 1 self.assertEqual(matchCount, len(data[2])) - def test_patchQtEdition(self): + def test_patch_qt_edition(self): tempDir = mkdtemp(dir=os.getcwd()) tempFile = os.path.join(tempDir, "qconfig.pri") @@ -130,7 +130,7 @@ class TestPackaging(unittest.TestCase): licheckName = "licheck_foo" releaseTimeStamp = "11223344" - patchQtEdition(tempDir, licheckName, releaseTimeStamp) + patch_qt_edition(tempDir, licheckName, releaseTimeStamp) expectedData = [] expectedData.append("something foo") @@ -150,7 +150,7 @@ class TestPackaging(unittest.TestCase): @unittest.skipUnless(os.environ.get("PKG_TEST_QT_CONFIG_BASE_PATH"), "Skipping because 'PKG_TEST_QT_CONFIG_BASE_PATH' is not set") @unittest.skipUnless(os.environ.get("PKG_TEST_QT_ARTIFACTS_URL"), "Skipping because 'PKG_TEST_QT_CONFIG_BASE_PATH' is not set") @unittest.skipUnless(os.environ.get("PKG_TEST_QT_IFW_TOOL_URL"), "Skipping because 'PKG_TEST_QT_IFW_TOOL_URL' is not set") - def test_createInstaller(self): + def test_create_installer(self): extension = '.run' if platform.system().lower().startswith('linux') else '' testsDir = os.path.dirname(os.path.abspath(__file__)) path = os.path.join(os.environ.get("PKG_TEST_QT_CONFIG_BASE_PATH"), "offline_installer_jobs", "5.9.3") diff --git a/packaging-tools/tests/test_release_repo_updater.py b/packaging-tools/tests/test_release_repo_updater.py index 4ddd0ec8f..48534229d 100755 --- a/packaging-tools/tests/test_release_repo_updater.py +++ b/packaging-tools/tests/test_release_repo_updater.py @@ -59,7 +59,7 @@ from release_task_reader import parse_data from tests.testhelpers import ( asyncio_test, asyncio_test_parallel_data, - isInternalFileServerReachable, + is_internal_file_server_reachable, ) @@ -128,7 +128,7 @@ class TestReleaseRepoUpdater(unittest.TestCase): self.assertTrue(remote_file_exists(self.server, os.path.abspath(__file__))) self.assertFalse(remote_file_exists(self.server, "/some/bogus/directory/foo.txt")) - @unittest.skipUnless(isInternalFileServerReachable(), "Skipping because file server is not accessible") + @unittest.skipUnless(is_internal_file_server_reachable(), "Skipping because file server is not accessible") @asyncio_test async def test_upload_ifw_to_remote(self) -> None: repogen = "" diff --git a/packaging-tools/tests/test_runCommand.py b/packaging-tools/tests/test_runCommand.py index 84c90d61a..50b6cd882 100644 --- a/packaging-tools/tests/test_runCommand.py +++ b/packaging-tools/tests/test_runCommand.py @@ -36,7 +36,7 @@ import sys import unittest from time import sleep -from bld_utils import runCommand +from bld_utils import run_command from threadedwork import ThreadedWork if sys.platform.startswith("win"): @@ -50,7 +50,7 @@ if sys.platform.startswith("win"): subprocess_flags = 0x8000000 # win32con.CREATE_NO_WINDOW? -def baseCommand(): +def base_command(): return " ".join([sys.executable, os.path.abspath(__file__)]) @@ -66,22 +66,22 @@ def crash(): c += 1 -def printLines(count): +def print_lines(count): for lineNumber in range(count): print(f"{lineNumber} printed line") -def useRunCommand(testArguments, *arguments): - return runCommand(f"{baseCommand()} {testArguments}", *arguments) +def use_run_command(testArguments, *arguments): + return run_command(f"{base_command()} {testArguments}", *arguments) class TestRunCommand(unittest.TestCase): - def test_ExitValue_0(self): - self.assertEqual(useRunCommand("--printLines 10", os.getcwd()), 0) + def test_exit_value_0(self): + self.assertEqual(use_run_command("--print_lines 10", os.getcwd()), 0) - def test_Crash(self): + def test_crash(self): with self.assertRaises(Exception) as contextManager: - useRunCommand("--printLines 10 --crash", os.getcwd()) + use_run_command("--print_lines 10 --crash", os.getcwd()) self.assertIsNotNone(contextManager) self.assertIsNotNone(contextManager.exception) @@ -89,10 +89,10 @@ class TestRunCommand(unittest.TestCase): messageStart = str(contextManager.exception)[:len(expectedMessageStart)] self.assertEqual(expectedMessageStart, messageStart) - def test_Crash_onlyErrorCaseOutput(self): + def test_crash_only_error_case_output(self): with self.assertRaises(Exception) as contextManager: - useRunCommand( - "--printLines 10 --crash", os.getcwd(), + use_run_command( + "--print_lines 10 --crash", os.getcwd(), # extra_environment= None, # onlyErrorCaseOutput= @@ -109,10 +109,10 @@ class TestRunCommand(unittest.TestCase): messageEnd = str(contextManager.exception).splitlines()[-1] self.assertTrue(messageEnd.__contains__(expectedMessageEnd)) - def test_differentExitCode_onlyErrorCaseOutput(self): + def test_different_exit_code_only_error_case_output(self): self.assertEqual( - useRunCommand( - "--printLines 10 --exitCode 5", os.getcwd(), + use_run_command( + "--print_lines 10 --exitCode 5", os.getcwd(), # extra_environment= None, # onlyErrorCaseOutput= @@ -122,51 +122,51 @@ class TestRunCommand(unittest.TestCase): ), 5 ) - def test_withThreadedWork(self): + def test_with_threadedwork(self): currentMethodName = sys._getframe().f_code.co_name testWork = ThreadedWork(f"{currentMethodName} - run some command threaded") taskStringList = [] - taskStringList.append("--sleep 1 --printLines 10") - taskStringList.append("--sleep 2 --printLines 30") - taskStringList.append("--sleep 1 --printLines 40") - taskStringList.append("--sleep 2 --printLines 50") - taskStringList.append("--sleep 1 --printLines 100") + taskStringList.append("--sleep 1 --print_lines 10") + taskStringList.append("--sleep 2 --print_lines 30") + taskStringList.append("--sleep 1 --print_lines 40") + taskStringList.append("--sleep 2 --print_lines 50") + taskStringList.append("--sleep 1 --print_lines 100") for taskString in taskStringList: - testWork.addTask(taskString, useRunCommand, taskString, os.getcwd()) + testWork.add_task(taskString, use_run_command, taskString, os.getcwd()) testWork.run() - def test_withThreadedWork_unexpected_exitCode(self): + def test_with_threadedwork_unexpected_exit_code(self): currentMethodName = sys._getframe().f_code.co_name testWork = ThreadedWork(f"{currentMethodName} - run some command threaded") # this exchange the current os._exit(-1) implementation only for this testing case separatorLine = f"{os.linesep}>>>>>>>>>>>>>>>>>>>>>>>>>>>>>{os.linesep}" separatorText = "the complete python application would stop here" - testWork.setExitFailFunction(sys.__stdout__.write, ''.join([separatorLine, separatorText, separatorLine])) + testWork.set_exit_fail_function(sys.__stdout__.write, ''.join([separatorLine, separatorText, separatorLine])) taskStringList = [] - taskStringList.append("--sleep 1 --printLines 10") - taskStringList.append("--sleep 2 --printLines 30") - taskStringList.append("--sleep 1 --printLines 40") - taskStringList.append("--sleep 2 --printLines 3 --exitCode 5") - taskStringList.append("--sleep 1 --printLines 100") + taskStringList.append("--sleep 1 --print_lines 10") + taskStringList.append("--sleep 2 --print_lines 30") + taskStringList.append("--sleep 1 --print_lines 40") + taskStringList.append("--sleep 2 --print_lines 3 --exitCode 5") + taskStringList.append("--sleep 1 --print_lines 100") for taskString in taskStringList: - testWork.addTask(taskString, useRunCommand, taskString, os.getcwd()) + testWork.add_task(taskString, use_run_command, taskString, os.getcwd()) testWork.run() - def test_withThreadedWork_crash(self): + def test_with_threadedwork_crash(self): currentMethodName = sys._getframe().f_code.co_name testWork = ThreadedWork(f"{currentMethodName} - run some command threaded") # this exchange the current os._exit(-1) implementation only for this testing case separatorLine = f"{os.linesep}>>>>>>>>>>>>>>>>>>>>>>>>>>>>>{os.linesep}" separatorText = "the complete python application would stop here" - testWork.setExitFailFunction(sys.__stdout__.write, ''.join([separatorLine, separatorText, separatorLine])) + testWork.set_exit_fail_function(sys.__stdout__.write, ''.join([separatorLine, separatorText, separatorLine])) taskStringList = [] - taskStringList.append("--sleep 1 --printLines 10") - taskStringList.append("--sleep 2 --printLines 30") - taskStringList.append("--sleep 1 --printLines 40") - taskStringList.append("--sleep 2 --printLines 3 --crash") - taskStringList.append("--sleep 1 --printLines 100") + taskStringList.append("--sleep 1 --print_lines 10") + taskStringList.append("--sleep 2 --print_lines 30") + taskStringList.append("--sleep 1 --print_lines 40") + taskStringList.append("--sleep 2 --print_lines 3 --crash") + taskStringList.append("--sleep 1 --print_lines 100") for taskString in taskStringList: - testWork.addTask(taskString, useRunCommand, taskString, os.getcwd()) + testWork.add_task(taskString, use_run_command, taskString, os.getcwd()) testWork.run() @@ -177,15 +177,15 @@ if __name__ == '__main__': else: parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0])) parser.add_argument('--sleep', type=int) - parser.add_argument('--printLines', type=int) + parser.add_argument('--print_lines', type=int) parser.add_argument('--crash', action='store_true', default=False) parser.add_argument('--exitCode', type=int) parser.add_argument('--testMethod') caller_arguments = parser.parse_args() if caller_arguments.sleep: sleep(caller_arguments.sleep) - if caller_arguments.printLines: - printLines(caller_arguments.printLines) + if caller_arguments.print_lines: + print_lines(caller_arguments.print_lines) if caller_arguments.crash: sys.__stdout__.flush() sys.__stderr__.flush() @@ -193,5 +193,5 @@ if __name__ == '__main__': if caller_arguments.exitCode: os._exit(caller_arguments.exitCode) if caller_arguments.testMethod: - # python test_runCommand.py --testMethod test_Crash_onlyErrorCaseOutput + # python test_runCommand.py --testMethod test_crash_only_error_case_output TestRunCommand(methodName=caller_arguments.testMethod).debug() diff --git a/packaging-tools/tests/testhelpers.py b/packaging-tools/tests/testhelpers.py index 1ba17d040..438325ac1 100644 --- a/packaging-tools/tests/testhelpers.py +++ b/packaging-tools/tests/testhelpers.py @@ -63,7 +63,7 @@ def asyncio_test_parallel_data(*data_args, unpack=True): return decorator -def isInternalFileServerReachable() -> bool: +def is_internal_file_server_reachable() -> bool: try: packageServer = get_pkg_value("PACKAGE_STORAGE_SERVER") ping = sh.which("ping") diff --git a/packaging-tools/threadedwork.py b/packaging-tools/threadedwork.py index 2c270029b..60dbd73d7 100644 --- a/packaging-tools/threadedwork.py +++ b/packaging-tools/threadedwork.py @@ -38,7 +38,7 @@ from queue import Queue from time import sleep from traceback import format_exc -# we are using RLock, because threadedPrint is using the same lock +# we are using RLock, because threaded_print is using the same lock output_lock = threading.RLock() output_states = None output_format_string = '' @@ -55,7 +55,7 @@ class StdOutHook: global output_format_string localProgressIndicator = None if len(strippedText) > 6: - localProgressIndicator = nextProgressIndicator() + localProgressIndicator = next_progress_indicator() else: localProgressIndicator = strippedText @@ -86,7 +86,7 @@ class StdErrHook: # builtin print() isn't threadsafe, lets make it threadsafe -def threadedPrint(*a, **b): +def threaded_print(*a, **b): with output_lock: org_print(*a, **b) @@ -97,7 +97,7 @@ org_stdout = sys.stdout org_sterr = sys.stderr -def enableThreadedPrint(enable=True, threadCount=cpu_count()): +def enable_threaded_print(enable=True, threadCount=cpu_count()): if enable: global output_states global output_format_string @@ -107,7 +107,7 @@ def enableThreadedPrint(enable=True, threadCount=cpu_count()): output_format_string = output_format_string + "{" + str(x) + ":10}" sys.stdout = StdOutHook() sys.stderr = StdErrHook() - builtins.print = threadedPrint + builtins.print = threaded_print else: sys.stdout = org_stdout sys.stderr = org_sterr @@ -117,7 +117,7 @@ def enableThreadedPrint(enable=True, threadCount=cpu_count()): thread_data = threading.local() -def nextProgressIndicator(): +def next_progress_indicator(): return next(thread_data.progressIndicator) @@ -144,7 +144,7 @@ class Task(): self.exitFunction = os._exit self.exitFunctionArguments = [-1] - def addFunction(self, function, *arguments): + def add_function(self, function, *arguments): aFunction = TaskFunction(function, *arguments) self.listOfFunctions.append(aFunction) @@ -175,14 +175,14 @@ class ThreadedWork(): self.taskNumber = 0 self.exitFunction = None - def setExitFailFunction(self, function, *arguments): + def set_exit_fail_function(self, function, *arguments): self.exitFunction = function self.exitFunctionArguments = arguments - def addTask(self, description, function, *arguments): - self.addTaskObject(Task(description, function, *arguments)) + def add_task(self, description, function, *arguments): + self.add_task_object(Task(description, function, *arguments)) - def addTaskObject(self, task): + def add_task_object(self, task): task.taskNumber = self.taskNumber if self.exitFunction: task.exitFunction = self.exitFunction @@ -198,7 +198,7 @@ class ThreadedWork(): print(os.linesep.join(self.legend)) if maxThreads > 1: - enableThreadedPrint(True, maxThreads) + enable_threaded_print(True, maxThreads) listOfConsumers = [] for i in range(maxThreads): # every Consumer needs a stop/none item @@ -219,7 +219,7 @@ class ThreadedWork(): sys.exit(0) # self.queue.join() <- this ignoring the KeyboardInterrupt if maxThreads > 1: - enableThreadedPrint(False) + enable_threaded_print(False) print(f"\n{self.description} ... done") |