From 6099b89f35499c60eb0ed4be10e40966326ea678 Mon Sep 17 00:00:00 2001 From: Patrik Teivonen Date: Thu, 16 Jun 2022 14:39:04 +0300 Subject: PEP8: Fix indentation Enable flake8 indentation related checks and fix indentation: - E123 Closing bracket does not match indentation of opening bracket's line - E126 Continuation line over-indented for hanging indent - E128 Continuation line under-indented for visual indent - E125 Continuation line with same indent as next logical line - E117 Over-indented - E124 Closing bracket does not match visual indentation - E127 Continuation line over-indented for visual indent - E111 Indentation is not a multiple of four - E121 Continuation line under-indented for hanging indent Change-Id: I2f25ed06d229b8fdaf94d69fa080dc1178ff3a2d Reviewed-by: Akseli Salovaara Reviewed-by: Iikka Eklund --- .pre-commit-config.yaml | 2 +- packaging-tools/bld_ifw_tools.py | 47 +++++++------ packaging-tools/bld_module.py | 56 ++++++++++------ packaging-tools/bld_openssl.py | 8 ++- packaging-tools/bld_sdktool.py | 32 +++++---- packaging-tools/bld_utils.py | 22 +++--- packaging-tools/bldinstallercommon.py | 42 +++++++----- packaging-tools/build_clang.py | 77 +++++++++++---------- packaging-tools/build_clang_qdoc.py | 5 +- packaging-tools/build_wrapper.py | 9 ++- packaging-tools/dump_debug_infos.py | 54 ++++++++++----- packaging-tools/environmentfrombatchfile.py | 5 +- .../libclang_training/libclangtimings2csv.py | 16 ++--- packaging-tools/release_repo_updater.py | 10 +-- .../tests/test_release_repo_meta_update.py | 78 +++++++++++----------- packaging-tools/tests/test_release_repo_updater.py | 45 +++++++------ packaging-tools/tests/test_runCommand.py | 62 +++++++++-------- 17 files changed, 321 insertions(+), 249 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bcc7d9f49..a35a47865 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: name: Check PEP8 compliance (flake8) # Disable E203 for compatibility with blackformatter, and W503 as it goes against PEP8 # Disable checks that are not relevant to this patch, they will be introduced later - entry: bash -c 'pipenv run python3 -m flake8 --max-line-length=99 --ignore=E111,E117,E121,E123,E124,E125,E126,E127,E128,E203,E501,W503 "$@"' + entry: bash -c 'pipenv run python3 -m flake8 --max-line-length=99 --ignore=E203,E501,W503 "$@"' language: system types: [python] fail_fast: true diff --git a/packaging-tools/bld_ifw_tools.py b/packaging-tools/bld_ifw_tools.py index 01eebb250..be539542d 100755 --- a/packaging-tools/bld_ifw_tools.py +++ b/packaging-tools/bld_ifw_tools.py @@ -497,8 +497,10 @@ def build_ifw_docs(options): cmd_args = options.make_doc_cmd + ' docs' env = dict(os.environ) env['LD_LIBRARY_PATH'] = os.path.normpath(os.path.join(os.path.dirname(qmake_bin), '..', 'lib')) - do_execute_sub_process(args=cmd_args.split(' '), execution_path=options.installer_framework_build_dir, - abort_on_fail=True, get_output=False, extra_env=env) + do_execute_sub_process( + args=cmd_args.split(' '), execution_path=options.installer_framework_build_dir, + abort_on_fail=True, get_output=False, extra_env=env + ) ################################################################ @@ -763,11 +765,13 @@ def get_platform_suffix(): # Setup argument parser ############################### def setup_argument_parser(): - parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]), - add_help=True, description="Build Qt Installer-Framework", - epilog="Builds Qt Installer Framework against static Qt libraries and archives the build artifacts for further usage. \n" - "To create an installer which installs the built Installer Framework libraries and tools use \"--create_installer\" option.", - formatter_class=argparse.RawTextHelpFormatter) + parser = argparse.ArgumentParser( + prog=os.path.basename(sys.argv[0]), + add_help=True, description="Build Qt Installer-Framework", + epilog="Builds Qt Installer Framework against static Qt libraries and archives the build artifacts for further usage. \n" + "To create an installer which installs the built Installer Framework libraries and tools use \"--create_installer\" option.", + formatter_class=argparse.RawTextHelpFormatter + ) parser.add_argument('--qt_archive_uri', help="Qt source package for Qt Installer-Framework", required=False) parser.add_argument('--qt_configure_options', help="Configure options for (static) Qt build", required=False) @@ -808,20 +812,21 @@ if __name__ == "__main__": qt_conf_args = qt_conf_args.replace('-release', '-debug') ifw_qmake_args = ifw_qmake_args.replace('-config release', '-config debug') # create options object - OPTIONS = IfwOptions(qt_src, - qt_configure_options, - CARGS.ifw_uri, - ifw_branch, - ifw_qmake_args, - CARGS.openssl_dir, - CARGS.product_key_checker_pri, - CARGS.qt_binaries_static, - CARGS.qt_binaries_dynamic, - signserver, - signpwd, - CARGS.incremental, - CARGS.archive_qt - ) + OPTIONS = IfwOptions( + qt_src, + qt_configure_options, + CARGS.ifw_uri, + ifw_branch, + ifw_qmake_args, + CARGS.openssl_dir, + CARGS.product_key_checker_pri, + CARGS.qt_binaries_static, + CARGS.qt_binaries_dynamic, + signserver, + signpwd, + CARGS.incremental, + CARGS.archive_qt + ) # build ifw tools if CARGS.archive_qt: build_and_archive_qt(OPTIONS) diff --git a/packaging-tools/bld_module.py b/packaging-tools/bld_module.py index 7383328c0..07d244ca1 100755 --- a/packaging-tools/bld_module.py +++ b/packaging-tools/bld_module.py @@ -107,8 +107,11 @@ def patch_build_time_paths(search_path, search_strings, qt_install_prefix): # install an argument parser -parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]), - add_help=True, description="build Qt 5 based Qt Module", formatter_class=argparse.RawTextHelpFormatter) +parser = argparse.ArgumentParser( + prog=os.path.basename(sys.argv[0]), + add_help=True, description="build Qt 5 based Qt Module", + formatter_class=argparse.RawTextHelpFormatter +) if is_windows(): parser.epilog = "example on windows: " + os.linesep + "\tpython {0} --clean " \ "--buildcommand C:\\bin\\ibjom.cmd" \ @@ -147,7 +150,7 @@ parser.add_argument('--buildcommand', help="this means usually make", default="m parser.add_argument('--installcommand', help="this means usually make", default="make") parser.add_argument('--debug', help="use debug builds", action='store_true', default=False) parser.add_argument('--qt5_module_url', help="a file or url where to get a qt5 module's binary content that is needed for the build as 7z", - dest='qt5_module_urls', action='append') + dest='qt5_module_urls', action='append') parser.add_argument('--module-name', help='name of the module, used for the resulting 7zip file') parser.add_argument('--module_url', help="Git URL for Qt Module", required=False, default='') parser.add_argument('--module_branch', help="Git branch for Qt Module", required=False, default='') @@ -170,7 +173,7 @@ callerArguments = parser.parse_args() stripVars(callerArguments, "\"") if callerArguments.qt5path != os.path.abspath(callerArguments.qt5path): print("changing the value of --qt5path from {0} to {1}".format(callerArguments.qt5path, - os.path.abspath(callerArguments.qt5path))) + os.path.abspath(callerArguments.qt5path))) callerArguments.qt5path = os.path.abspath(callerArguments.qt5path) if not callerArguments.module_name: @@ -214,7 +217,7 @@ if callerArguments.clean: if not os.path.lexists(callerArguments.qt5path) and not callerArguments.qt5_module_urls: parser.print_help() print(("error: Please add the missing qt5_module_url arguments if the {0} does not exist" - + os.linesep + os.linesep).format(callerArguments.qt5path)) + + os.linesep + os.linesep).format(callerArguments.qt5path)) raise RuntimeError() qmakeBinary = os.path.abspath(os.path.join(callerArguments.qt5path, 'bin', 'qmake')) @@ -222,8 +225,12 @@ qmakeBinary = os.path.abspath(os.path.join(callerArguments.qt5path, 'bin', 'qmak if not os.path.lexists(callerArguments.qt5path): # get Qt myGetQtBinaryWork = ThreadedWork("get and extract Qt 5 binary") - myGetQtBinaryWork.addTaskObject(bldinstallercommon.create_qt_download_task(callerArguments.qt5_module_urls, - callerArguments.qt5path, tempPath, callerArguments)) + myGetQtBinaryWork.addTaskObject( + bldinstallercommon.create_qt_download_task( + callerArguments.qt5_module_urls, + callerArguments.qt5path, tempPath, callerArguments + ) + ) myGetQtBinaryWork.run() # Save QT_INSTALL_PREFIX @@ -245,8 +252,10 @@ pathKeyList.append(pythonExecutablePath) environment = {'PATH': os.pathsep.join(pathKeyList)} if is_linux(): - environment["LD_LIBRARY_PATH"] = os.pathsep.join([os.path.join(callerArguments.qt5path, 'lib')] -+ os.environ.get("LD_LIBRARY_PATH", "").split(os.pathsep)) + environment["LD_LIBRARY_PATH"] = os.pathsep.join( + [os.path.join(callerArguments.qt5path, 'lib')] + + os.environ.get("LD_LIBRARY_PATH", "").split(os.pathsep) + ) environment["QMAKESPEC"] = "linux-g++" if is_macos(): @@ -296,9 +305,11 @@ ret = runBuildCommand(currentWorkingDirectory=qtModuleBuildDirectory, callerArgu if ret: raise RuntimeError('Failure running the last command: %i' % ret) -ret = runInstallCommand(['install', 'INSTALL_ROOT=' + qtModuleInstallDirectory], - currentWorkingDirectory=qtModuleBuildDirectory, - callerArguments=callerArguments, extra_environment=environment) +ret = runInstallCommand( + ['install', 'INSTALL_ROOT=' + qtModuleInstallDirectory], + currentWorkingDirectory=qtModuleBuildDirectory, + callerArguments=callerArguments, extra_environment=environment +) if ret: raise RuntimeError('Failure running the last command: %i' % ret) @@ -317,22 +328,27 @@ if callerArguments.collectDocs: # enginio etc. docs creation if callerArguments.makeDocs: # build docs first - ret = runInstallCommand('docs', - currentWorkingDirectory=qtModuleBuildDirectory, - callerArguments=callerArguments, extra_environment=environment) + ret = runInstallCommand( + 'docs', currentWorkingDirectory=qtModuleBuildDirectory, + callerArguments=callerArguments, extra_environment=environment + ) if ret: raise RuntimeError('Failure running the last command: %i' % ret) # then make install those - ret = runInstallCommand(['install_docs', 'INSTALL_ROOT=' + qtModuleInstallDirectory], - currentWorkingDirectory=qtModuleBuildDirectory, - callerArguments=callerArguments, extra_environment=environment) + ret = runInstallCommand( + ['install_docs', 'INSTALL_ROOT=' + qtModuleInstallDirectory], + currentWorkingDirectory=qtModuleBuildDirectory, + callerArguments=callerArguments, extra_environment=environment + ) if ret: raise RuntimeError('Failure running the last command: %i' % ret) # make separate "doc.7z" for later use if needed doc_dir = locate_path(qtModuleInstallDirectory, ["doc"], filters=[os.path.isdir]) archive_name = callerArguments.module_name + '-' + os.environ['LICENSE'] + '-doc-' + os.environ['MODULE_VERSION'] + '.7z' - ret = runCommand(['7z', 'a', os.path.join('doc_archives', archive_name), doc_dir], - currentWorkingDirectory=os.path.dirname(os.path.realpath(__file__))) + ret = runCommand( + ['7z', 'a', os.path.join('doc_archives', archive_name), doc_dir], + currentWorkingDirectory=os.path.dirname(os.path.realpath(__file__)) + ) if ret: raise RuntimeError('Failure running the last command: %i' % ret) diff --git a/packaging-tools/bld_openssl.py b/packaging-tools/bld_openssl.py index c4c3d6f33..74e4e4628 100644 --- a/packaging-tools/bld_openssl.py +++ b/packaging-tools/bld_openssl.py @@ -74,9 +74,11 @@ def check_environment(): def setup_argument_parser(): - parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]), - add_help=True, description='Build openssl from sources', - formatter_class=argparse.RawTextHelpFormatter) + parser = argparse.ArgumentParser( + prog=os.path.basename(sys.argv[0]), + add_help=True, description='Build openssl from sources', + formatter_class=argparse.RawTextHelpFormatter + ) parser.add_argument('--sourcedir', help='Source directory', required=True) parser.add_argument('--installdir', help=r'Target directory (should be on C:\)', required=False, default=r'C:\usr\local\openssl') diff --git a/packaging-tools/bld_sdktool.py b/packaging-tools/bld_sdktool.py index 7ba6c20bc..4689685bf 100644 --- a/packaging-tools/bld_sdktool.py +++ b/packaging-tools/bld_sdktool.py @@ -94,8 +94,10 @@ def get_and_extract_qt_src(url, temp, path): def configure_qt(params, src, build): Path(build).mkdir(parents=True, exist_ok=True) configure = os.path.join(src, 'configure') - do_execute_sub_process([configure, '-prefix', build] + qt_static_configure_options(), - build, redirect_output=params.redirect_output) + do_execute_sub_process( + [configure, '-prefix', build] + qt_static_configure_options(), + build, redirect_output=params.redirect_output + ) def build_qt(params, build): @@ -113,16 +115,17 @@ def build_sdktool_impl(params, qt_build_path): if is_windows(): cmake_args += ['-DCMAKE_C_COMPILER=cl', '-DCMAKE_CXX_COMPILER=cl'] - do_execute_sub_process(cmake_args + ['-G', 'Ninja', params.src_path], - params.build_path, - redirect_output=params.redirect_output) - do_execute_sub_process(['cmake', '--build', '.'], params.build_path, - redirect_output=params.redirect_output) - do_execute_sub_process(['cmake', - '--install', '.', - '--prefix', params.target_path], - params.build_path, - redirect_output=params.redirect_output) + do_execute_sub_process( + cmake_args + ['-G', 'Ninja', params.src_path], + params.build_path, redirect_output=params.redirect_output + ) + do_execute_sub_process( + ['cmake', '--build', '.'], params.build_path, redirect_output=params.redirect_output + ) + do_execute_sub_process( + ['cmake', '--install', '.', '--prefix', params.target_path], + params.build_path, redirect_output=params.redirect_output + ) def build_sdktool(qt_src_url, qt_build_base, sdktool_src_path, sdktool_build_path, sdktool_target_path, @@ -142,5 +145,6 @@ def build_sdktool(qt_src_url, qt_build_base, sdktool_src_path, sdktool_build_pat def zip_sdktool(sdktool_target_path, out_7zip, redirect_output=None): glob = "*.exe" if is_windows() else "*" - do_execute_sub_process(['7z', 'a', out_7zip, glob], - sdktool_target_path, redirect_output=redirect_output) + do_execute_sub_process( + ['7z', 'a', out_7zip, glob], sdktool_target_path, redirect_output=redirect_output + ) diff --git a/packaging-tools/bld_utils.py b/packaging-tools/bld_utils.py index 596bd6910..811aec7e7 100644 --- a/packaging-tools/bld_utils.py +++ b/packaging-tools/bld_utils.py @@ -256,8 +256,8 @@ def runCommand(command, currentWorkingDirectory, callerArguments=None, extra_env environment = getEnvironment(extra_environment, callerArguments) # if we can not find the command, just check the current working dir - if not os.path.lexists(commandAsList[0]) and currentWorkingDirectory and \ - os.path.isfile(os.path.abspath(os.path.join(currentWorkingDirectory, commandAsList[0]))): + if (not os.path.lexists(commandAsList[0]) and currentWorkingDirectory + and os.path.isfile(os.path.abspath(os.path.join(currentWorkingDirectory, commandAsList[0])))): commandAsList[0] = os.path.abspath(os.path.join(currentWorkingDirectory, commandAsList[0])) pathEnvironment = environment['PATH'] @@ -283,12 +283,16 @@ def runCommand(command, currentWorkingDirectory, callerArguments=None, extra_env lastStdOutLines = [] lastStdErrLines = [] if threading.currentThread().name == "MainThread" and not onlyErrorCaseOutput: - process = subprocess.Popen(commandAsList, shell=useShell, - cwd=currentWorkingDirectory, bufsize=-1, env=environment) + process = subprocess.Popen( + commandAsList, shell=useShell, + cwd=currentWorkingDirectory, bufsize=-1, env=environment + ) else: - process = subprocess.Popen(commandAsList, shell=useShell, + process = subprocess.Popen( + commandAsList, shell=useShell, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=currentWorkingDirectory, bufsize=-1, env=environment) + cwd=currentWorkingDirectory, bufsize=-1, env=environment + ) maxSavedLineNumbers = 1000 lastStdOutLines = collections.deque(maxlen=maxSavedLineNumbers) @@ -383,8 +387,10 @@ def runBuildCommand(arguments=None, currentWorkingDirectory=None, callerArgument @deep_copy_arguments def getReturnValue(command, currentWorkingDirectory=None, extra_environment=None, callerArguments=None): commandAsList = command[:].split(' ') - return subprocess.Popen(commandAsList, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - cwd=currentWorkingDirectory, env=getEnvironment(extra_environment, callerArguments)).communicate()[0].strip() + return subprocess.Popen( + commandAsList, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + cwd=currentWorkingDirectory, env=getEnvironment(extra_environment, callerArguments) + ).communicate()[0].strip() def gitSHA(path, callerArguments=None): diff --git a/packaging-tools/bldinstallercommon.py b/packaging-tools/bldinstallercommon.py index 7442d209e..d5f1fd67f 100644 --- a/packaging-tools/bldinstallercommon.py +++ b/packaging-tools/bldinstallercommon.py @@ -364,9 +364,11 @@ def requires_rpath(file_path): if is_linux(): if not os.access(file_path, os.X_OK): return False - return (re.search(r':*.R.*PATH=', - subprocess.Popen(['chrpath', '-l', file_path], - stdout=subprocess.PIPE).stdout.read().decode()) is not None) + return ( + re.search(r':*.R.*PATH=', subprocess.Popen( + ['chrpath', '-l', file_path], stdout=subprocess.PIPE + ).stdout.read().decode()) is not None + ) return False @@ -473,9 +475,12 @@ def handle_component_rpath(component_root_path, destination_lib_paths): rpaths.append(rp) # look for existing $ORIGIN path in the binary - origin_rpath = re.search(r'\$ORIGIN[^:\n]*', - subprocess.Popen(['chrpath', '-l', file_full_path], - stdout=subprocess.PIPE).stdout.read().decode()) + origin_rpath = re.search( + r'\$ORIGIN[^:\n]*', + subprocess.Popen( + ['chrpath', '-l', file_full_path], stdout=subprocess.PIPE + ).stdout.read().decode() + ) if origin_rpath and origin_rpath.group() not in rpaths: rpaths.append(origin_rpath.group()) @@ -711,8 +716,9 @@ def create_qt_download_task(module_urls, target_qt5_path, temp_path, caller_argu # add Qt modules for module_url in module_urls: if is_content_url_valid(module_url): - (download_task, extract_task) = create_download_and_extract_tasks(module_url, - target_qt5_path, temp_path, caller_arguments) + (download_task, extract_task) = create_download_and_extract_tasks( + module_url, target_qt5_path, temp_path, caller_arguments + ) download_work.addTaskObject(download_task) unzip_task.addFunction(extract_task.do) else: @@ -720,24 +726,28 @@ def create_qt_download_task(module_urls, target_qt5_path, temp_path, caller_argu # add icu, d3dcompiler, opengl32, openssl target_path = os.path.join(target_qt5_path, 'bin' if is_windows() else 'lib') if not is_macos() and hasattr(caller_arguments, 'icu7z') and caller_arguments.icu7z: - (download_task, extract_task) = create_download_and_extract_tasks(caller_arguments.icu7z, - target_path, temp_path, caller_arguments) + (download_task, extract_task) = create_download_and_extract_tasks( + caller_arguments.icu7z, target_path, temp_path, caller_arguments + ) download_work.addTaskObject(download_task) unzip_task.addFunction(extract_task.do) if is_windows(): if hasattr(caller_arguments, 'd3dcompiler7z') and caller_arguments.d3dcompiler7z: - (download_task, extract_task) = create_download_and_extract_tasks(caller_arguments.d3dcompiler7z, - target_path, temp_path, caller_arguments) + (download_task, extract_task) = create_download_and_extract_tasks( + caller_arguments.d3dcompiler7z, target_path, temp_path, caller_arguments + ) download_work.addTaskObject(download_task) unzip_task.addFunction(extract_task.do) if hasattr(caller_arguments, 'opengl32sw7z') and caller_arguments.opengl32sw7z: - (download_task, extract_task) = create_download_and_extract_tasks(caller_arguments.opengl32sw7z, - target_path, temp_path, caller_arguments) + (download_task, extract_task) = create_download_and_extract_tasks( + caller_arguments.opengl32sw7z, target_path, temp_path, caller_arguments + ) download_work.addTaskObject(download_task) unzip_task.addFunction(extract_task.do) if hasattr(caller_arguments, 'openssl7z') and caller_arguments.openssl7z: - (download_task, extract_task) = create_download_and_extract_tasks(caller_arguments.openssl7z, - target_path, temp_path, caller_arguments) + (download_task, extract_task) = create_download_and_extract_tasks( + caller_arguments.openssl7z, target_path, temp_path, caller_arguments + ) download_work.addTaskObject(download_task) unzip_task.addFunction(extract_task.do) qt_task.addFunction(download_work.run) diff --git a/packaging-tools/build_clang.py b/packaging-tools/build_clang.py index e4addec3e..77f7bd6e1 100644 --- a/packaging-tools/build_clang.py +++ b/packaging-tools/build_clang.py @@ -184,8 +184,8 @@ def mingw_training(base_path, qtcreator_path, environment, bitness): + training_qt_version() + '/' + training_qt_long_version() + '-final-released/Qt' + training_qt_long_version() msvc_year_ver = msvc_year_version() if bitness == 64: - qt_mingw_postfix = '-Windows-Windows_10-Mingw-Windows-Windows_10-X86_64.7z' - qt_postfix = '-Windows-Windows_10-' + msvc_year_ver + '-Windows-Windows_10-X86_64.7z' + qt_mingw_postfix = '-Windows-Windows_10-Mingw-Windows-Windows_10-X86_64.7z' + qt_postfix = '-Windows-Windows_10-' + msvc_year_ver + '-Windows-Windows_10-X86_64.7z' qt_module_urls = [qt_base_url + '/' + module + '/' + module + qt_postfix for module in qt_modules] qt_mingw_module_urls = [qt_base_url + '/' + module + '/' + module + qt_mingw_postfix for module in qt_modules] @@ -214,37 +214,37 @@ def mingw_training(base_path, qtcreator_path, environment, bitness): # WITH_TESTS is required for QtCreator to support running .batch files cmake_command = os.path.join(cmake_dir, 'cmake-' + cmake_version() + '-' + cmake_arch_suffix, 'bin', 'cmake') qtc_cmake = [cmake_command, - '-GNinja', - '-DCMAKE_BUILD_TYPE=Release', - '-DWITH_TESTS=ON', - '-DBUILD_WITH_PCH=OFF', - - '-DBUILD_QBS=OFF', - - '-DBUILD_PLUGINS_BY_DEFAULT=OFF', - '-DBUILD_EXECUTABLES_BY_DEFAULT=OFF', - - '-DBUILD_PLUGIN_CORE=ON', - '-DBUILD_PLUGIN_TEXTEDITOR=ON', - '-DBUILD_PLUGIN_PROJECTEXPLORER=ON', - '-DBUILD_PLUGIN_CPPEDITOR=ON', - '-DBUILD_PLUGIN_QMAKEPROJECTMANAGER=ON', - '-DBUILD_PLUGIN_CLANGCODEMODEL=ON', - '-DBUILD_PLUGIN_CLANGTOOLS=ON', - '-DBUILD_PLUGIN_DEBUGGER=ON', - '-DBUILD_PLUGIN_DESIGNER=ON', - '-DBUILD_PLUGIN_LANGUAGECLIENT=ON', - '-DBUILD_PLUGIN_QTSUPPORT=ON', - '-DBUILD_PLUGIN_RESOURCEEDITOR=ON', - - '-DBUILD_EXECUTABLE_QTCREATOR=ON', - '-DBUILD_EXECUTABLE_ECHO=ON', - '-DBUILD_EXECUTABLE_CLANGBACKEND=ON', - '-DBUILD_EXECUTABLE_QTCREATOR_PROCESSLAUNCHER=ON', - - '-DCMAKE_PREFIX_PATH=' + qt_mingw_dir + ';' + os.path.join(base_path, 'libclang'), - '-S' + qtcreator_path, - '-B' + creator_build_dir] + '-GNinja', + '-DCMAKE_BUILD_TYPE=Release', + '-DWITH_TESTS=ON', + '-DBUILD_WITH_PCH=OFF', + + '-DBUILD_QBS=OFF', + + '-DBUILD_PLUGINS_BY_DEFAULT=OFF', + '-DBUILD_EXECUTABLES_BY_DEFAULT=OFF', + + '-DBUILD_PLUGIN_CORE=ON', + '-DBUILD_PLUGIN_TEXTEDITOR=ON', + '-DBUILD_PLUGIN_PROJECTEXPLORER=ON', + '-DBUILD_PLUGIN_CPPEDITOR=ON', + '-DBUILD_PLUGIN_QMAKEPROJECTMANAGER=ON', + '-DBUILD_PLUGIN_CLANGCODEMODEL=ON', + '-DBUILD_PLUGIN_CLANGTOOLS=ON', + '-DBUILD_PLUGIN_DEBUGGER=ON', + '-DBUILD_PLUGIN_DESIGNER=ON', + '-DBUILD_PLUGIN_LANGUAGECLIENT=ON', + '-DBUILD_PLUGIN_QTSUPPORT=ON', + '-DBUILD_PLUGIN_RESOURCEEDITOR=ON', + + '-DBUILD_EXECUTABLE_QTCREATOR=ON', + '-DBUILD_EXECUTABLE_ECHO=ON', + '-DBUILD_EXECUTABLE_CLANGBACKEND=ON', + '-DBUILD_EXECUTABLE_QTCREATOR_PROCESSLAUNCHER=ON', + + '-DCMAKE_PREFIX_PATH=' + qt_mingw_dir + ';' + os.path.join(base_path, 'libclang'), + '-S' + qtcreator_path, + '-B' + creator_build_dir] bld_utils.runCommand(qtc_cmake, creator_build_dir, None, environment) bld_utils.runCommand([cmake_command, '--build', creator_build_dir], creator_build_dir, None, environment) @@ -257,8 +257,10 @@ def mingw_training(base_path, qtcreator_path, environment, bitness): # Train mingw libclang library with build QtCreator # First time open the project, then close it. This will generate initial settings and .user files. Second time do the actual training. for batchFile in ['qtc.openProject.batch', 'qtc.fileTextEditorCpp.batch']: - bld_utils.runCommand([os.path.join(training_dir, 'runBatchFiles.bat'), msvc_version(), 'x64' if bitness == 64 else 'x86', batchFile], - base_path, callerArguments=None, extra_environment=None, onlyErrorCaseOutput=False, expectedExitCodes=[0, 1]) + bld_utils.runCommand( + [os.path.join(training_dir, 'runBatchFiles.bat'), msvc_version(), 'x64' if bitness == 64 else 'x86', batchFile], + base_path, callerArguments=None, extra_environment=None, onlyErrorCaseOutput=False, expectedExitCodes=[0, 1] + ) def is_msvc_toolchain(toolchain): @@ -303,8 +305,9 @@ def profile_data_flags(toolchain, profile_data_path, first_run): return cmake_flags if is_mingw_toolchain(toolchain): linker_flags = '-static-libgcc -static-libstdc++ -static' - return ['-DCMAKE_SHARED_LINKER_FLAGS=' + linker_flags, - '-DCMAKE_EXE_LINKER_FLAGS=' + linker_flags, + return [ + '-DCMAKE_SHARED_LINKER_FLAGS=' + linker_flags, + '-DCMAKE_EXE_LINKER_FLAGS=' + linker_flags, ] return [] diff --git a/packaging-tools/build_clang_qdoc.py b/packaging-tools/build_clang_qdoc.py index dc0321f4b..3280dca20 100644 --- a/packaging-tools/build_clang_qdoc.py +++ b/packaging-tools/build_clang_qdoc.py @@ -144,8 +144,9 @@ def cmake_generator(toolchain): def static_flags(toolchain): if is_mingw_toolchain(toolchain): linker_flags = '-static-libgcc -static-libstdc++ -static' - return ['-DCMAKE_SHARED_LINKER_FLAGS=' + linker_flags, - '-DCMAKE_EXE_LINKER_FLAGS=' + linker_flags, + return [ + '-DCMAKE_SHARED_LINKER_FLAGS=' + linker_flags, + '-DCMAKE_EXE_LINKER_FLAGS=' + linker_flags, ] return [] diff --git a/packaging-tools/build_wrapper.py b/packaging-tools/build_wrapper.py index 70b07d010..da694a0c2 100644 --- a/packaging-tools/build_wrapper.py +++ b/packaging-tools/build_wrapper.py @@ -304,13 +304,12 @@ class BuildLog: def check_call_log(args, execution_path, extra_env=dict(os.environ), log_filepath=None, log_overwrite=False): if not log_filepath: - do_execute_sub_process(args, execution_path, - extra_env=extra_env) + do_execute_sub_process(args, execution_path, extra_env=extra_env) else: with BuildLog(log_filepath, log_overwrite) as f: - do_execute_sub_process(args, execution_path, - extra_env=extra_env, - redirect_output=f) + do_execute_sub_process( + args, execution_path, extra_env=extra_env, redirect_output=f + ) def create_qtcreator_source_package(source_path, plugin_name, version, edition, target_path, log_filepath): diff --git a/packaging-tools/dump_debug_infos.py b/packaging-tools/dump_debug_infos.py index 000f886d6..44d8929d2 100644 --- a/packaging-tools/dump_debug_infos.py +++ b/packaging-tools/dump_debug_infos.py @@ -106,27 +106,44 @@ def dump_syms(dump_syms_path, architectures, search_pathes, output_path, verbose ################################################################################ def _main(): - parser = argparse.ArgumentParser(description="fetch Breakpad symbols", - formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('--dump-syms-path', required=True, - help="path to dump_syms tool") - parser.add_argument('--sentry-cli-path', required=True, - help="path to sentry-cli tool") - parser.add_argument('--search-pathes', required=True, - help="directories where the binaries can be found, separated by comma") - parser.add_argument('--architectures', default="", - help="if there is more then one architecture for example on mac: \"x86_64,arm64\"") - parser.add_argument('--output-path', required=True, - help="directory where the extracted debug information will be stored") - parser.add_argument('--clean-output-path', action='store_true', - help="empty the output directory at the beginning") - parser.add_argument('-v', '--verbose', action='store_true', - help="Gives some output what the tool is actual doing.") + parser = argparse.ArgumentParser( + description="fetch Breakpad symbols", + formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument( + '--dump-syms-path', required=True, + help="path to dump_syms tool" + ) + parser.add_argument( + '--sentry-cli-path', required=True, + help="path to sentry-cli tool" + ) + parser.add_argument( + '--search-pathes', required=True, + help="directories where the binaries can be found, separated by comma" + ) + parser.add_argument( + '--architectures', default="", + help="if there is more then one architecture for example on mac: \"x86_64,arm64\"" + ) + parser.add_argument( + '--output-path', required=True, + help="directory where the extracted debug information will be stored" + ) + parser.add_argument( + '--clean-output-path', action='store_true', + help="empty the output directory at the beginning" + ) + parser.add_argument( + '-v', '--verbose', action='store_true', + help="Gives some output what the tool is actual doing." + ) args = parser.parse_args() # make sure we can call dump_syms dump_syms_output = subprocess.check_output( - [args.dump_syms_path, '--help']).decode('utf-8') + [args.dump_syms_path, '--help'] + ).decode('utf-8') if len(dump_syms_output) <= 0: raise Exception("dump_syms can not be called: \n{}".format(dump_syms_output)) @@ -160,7 +177,8 @@ def _main(): if args.verbose: print(source_bundle_command) testoutput = subprocess.check_output( - source_bundle_command, cwd=args.output_path).decode('utf-8') + source_bundle_command, cwd=args.output_path + ).decode('utf-8') print(testoutput) return diff --git a/packaging-tools/environmentfrombatchfile.py b/packaging-tools/environmentfrombatchfile.py index bbc92ec05..cd28ebdb3 100644 --- a/packaging-tools/environmentfrombatchfile.py +++ b/packaging-tools/environmentfrombatchfile.py @@ -59,8 +59,9 @@ class PackagingError(Exception): def sanity_check_env(env_cmd, env_dict): # throw error if 'Path' exists in environment dictionary if "Path" in env_dict: - raise PackagingError("Configuration error, possible duplicate path 'Path' found in {0}" - .format(env_cmd)) + raise PackagingError( + "Configuration error, possible duplicate path 'Path' found in {0}".format(env_cmd) + ) def get(env_cmd, initial=None, arguments=None): diff --git a/packaging-tools/libclang_training/libclangtimings2csv.py b/packaging-tools/libclang_training/libclangtimings2csv.py index c2655beff..595780650 100644 --- a/packaging-tools/libclang_training/libclangtimings2csv.py +++ b/packaging-tools/libclang_training/libclangtimings2csv.py @@ -41,11 +41,13 @@ import sys def constructRecordMatcher(): - regex = '( Parsing)' \ - + '|( Precompiling preamble)' \ - + '|( Reparsing)' \ - + '|( Cache global code completions)' \ - + '|( Code completion)' + regex = ( + '( Parsing)' + + '|( Precompiling preamble)' + + '|( Reparsing)' + + '|( Cache global code completions)' + + '|( Code completion)' + ) return re.compile(regex) @@ -59,9 +61,7 @@ def constructTimeNeededMatcher(): notRelevantParts = r'(\s*\d+\.\d+ \(\d+\.\d+\%\)){0,3}' wallClockTime = r'\s*(\d+\.\d+) \(\d+\.\d+\%\)' - regex = startIndicator \ - + notRelevantParts \ - + wallClockTime + regex = startIndicator + notRelevantParts + wallClockTime return re.compile(regex) diff --git a/packaging-tools/release_repo_updater.py b/packaging-tools/release_repo_updater.py index 68e4896de..6b81cec36 100755 --- a/packaging-tools/release_repo_updater.py +++ b/packaging-tools/release_repo_updater.py @@ -728,11 +728,11 @@ def load_export_summary_data(config_file: Path) -> Dict[str, str]: if __name__ == "__main__": args_from_file_parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - # Turn off help, so we print all options in response to -h - add_help=False - ) + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + # Turn off help, so we print all options in response to -h + add_help=False + ) args_from_file_parser.add_argument("--config", dest="config", type=str, default=os.getenv("RELEASE_DESCRIPTION_FILE"), help="Path to top level release config file", metavar="FILE") args, _ = args_from_file_parser.parse_known_args() diff --git a/packaging-tools/tests/test_release_repo_meta_update.py b/packaging-tools/tests/test_release_repo_meta_update.py index 6a3231250..ce8732fa5 100755 --- a/packaging-tools/tests/test_release_repo_meta_update.py +++ b/packaging-tools/tests/test_release_repo_meta_update.py @@ -35,10 +35,10 @@ import unittest import tempfile from ddt import ddt from typing import List -from release_repo_meta_update import IfwRepoUpdateError, scan_repositories, \ - swap_repositories, create_converted_repositories, \ - check_repos_which_can_be_updated, \ - convert_suffix, backup_suffix +from release_repo_meta_update import ( + IfwRepoUpdateError, scan_repositories, swap_repositories, create_converted_repositories, + check_repos_which_can_be_updated, convert_suffix, backup_suffix +) @ddt @@ -50,43 +50,43 @@ class TestReleaseRepoMetaUpdate(unittest.TestCase): @classmethod def setUpClass(self) -> None: self.paths = [ - "repo1/Updates.xml", - "repo2/Updates.xml", - "repo2" + convert_suffix + "/Updates.xml", - "repo2" + convert_suffix + "/123456_meta.7z", - "repo3/Updates.xml", - "repo3" + convert_suffix + "/Updates.xml", - "repo3" + convert_suffix + "/123456_meta.7z", - "repo4/Updates.xml", - "repo5/Updates.xml", - "repo5/123456_meta.7z", - "repo6/Updates.xml", - "repo7/Updates.xml", - "repo7/123456_meta.7z", - "repo7" + convert_suffix + "/Updates.xml", - "repo7" + convert_suffix + "/123456_meta.7z", - "repo8/Updates.xml", - "repo9/Updates.xml", - "repo9" + convert_suffix + "/Updates.xml", - "repo9" + convert_suffix + "/", # meta.7z missing - "repo10" + backup_suffix + "123456" - ] + "repo1/Updates.xml", + "repo2/Updates.xml", + "repo2" + convert_suffix + "/Updates.xml", + "repo2" + convert_suffix + "/123456_meta.7z", + "repo3/Updates.xml", + "repo3" + convert_suffix + "/Updates.xml", + "repo3" + convert_suffix + "/123456_meta.7z", + "repo4/Updates.xml", + "repo5/Updates.xml", + "repo5/123456_meta.7z", + "repo6/Updates.xml", + "repo7/Updates.xml", + "repo7/123456_meta.7z", + "repo7" + convert_suffix + "/Updates.xml", + "repo7" + convert_suffix + "/123456_meta.7z", + "repo8/Updates.xml", + "repo9/Updates.xml", + "repo9" + convert_suffix + "/Updates.xml", + "repo9" + convert_suffix + "/", # meta.7z missing + "repo10" + backup_suffix + "123456" + ] self.non_migrated_paths = [ - "repo1/Updates.xml", - "repo2/sub2/Updates.xml", - "repo3/sub3/subsub3/Updates.xml", - "repo4/Updates.xml", - "repo5/Updates.xml", - ] + "repo1/Updates.xml", + "repo2/sub2/Updates.xml", + "repo3/sub3/subsub3/Updates.xml", + "repo4/Updates.xml", + "repo5/Updates.xml", + ] self.mixed_migrated_paths = [ - "repo1/Updates.xml", - "repo1" + convert_suffix + "/Updates.xml", - "repo2/sub2/Updates.xml", - "repo3/sub3/subsub3/Updates.xml", - "repo3/sub3/subsub3" + convert_suffix + "/Updates.xml", - "repo4/Updates.xml", - "repo5/Updates.xml", - ] + "repo1/Updates.xml", + "repo1" + convert_suffix + "/Updates.xml", + "repo2/sub2/Updates.xml", + "repo3/sub3/subsub3/Updates.xml", + "repo3/sub3/subsub3" + convert_suffix + "/Updates.xml", + "repo4/Updates.xml", + "repo5/Updates.xml", + ] def _write_test_repo(self, tmp_base_dir: str, paths: List[str]) -> None: for path in paths: diff --git a/packaging-tools/tests/test_release_repo_updater.py b/packaging-tools/tests/test_release_repo_updater.py index 1f71681da..7d9290f74 100755 --- a/packaging-tools/tests/test_release_repo_updater.py +++ b/packaging-tools/tests/test_release_repo_updater.py @@ -224,38 +224,39 @@ class TestReleaseRepoUpdater(unittest.TestCase): ("Cannot find new components to update"), unpack=False ) async def test_check_valid_repogen_output(self, repogen_output) -> None: - # should not throw exception - check_repogen_output(repogen_output) + # should not throw exception + check_repogen_output(repogen_output) - @testhelpers.asyncio_test_parallel_data(([], ["repository"]), - (["linux,common"], ["repository,linux,common"]), - (["", "linux,common"], ["repository", "repository,linux,common"]) + @testhelpers.asyncio_test_parallel_data( + ([], ["repository"]), + (["linux,common"], ["repository,linux,common"]), + (["", "linux,common"], ["repository", "repository,linux,common"]) ) async def test_append_to_task_filters(self, task_filters: str, expected_result: bool) -> None: self.assertEqual(append_to_task_filters(task_filters, "repository"), expected_result) - @testhelpers.asyncio_test_parallel_data((["task.repository.linux.x64.feature1"], ["task,repository,linux,x64,feature1"]), - (["task.repository.linux.x64.feature1", "windows.x64,feature2"], - ["task,repository,linux,x64,feature1", "windows,x64,feature2"]), - (["offline,linux.x64,feature1"], ["offline,linux,x64,feature1"]), - (["linux"], ["linux"]), - ([""], [""]) + @testhelpers.asyncio_test_parallel_data( + (["task.repository.linux.x64.feature1"], ["task,repository,linux,x64,feature1"]), + (["task.repository.linux.x64.feature1", "windows.x64,feature2"], + ["task,repository,linux,x64,feature1", "windows,x64,feature2"]), + (["offline,linux.x64,feature1"], ["offline,linux,x64,feature1"]), + (["linux"], ["linux"]), + ([""], [""]) ) async def test_format_task_filters(self, task_filters: str, expected_result: bool) -> None: print("test") self.assertEqual(format_task_filters(task_filters), expected_result) - @testhelpers.asyncio_test_parallel_data(("qtsdkrepository/windows_x86/desktop/tools_maintenance/log-s3-2020-12-03" - "--10:18:11-xml.txt:fatal error: Could not connect to the endpoint URL: 'https" - "://qt-cdn.s3.eu-west-1.amazonaws.com/?list-type=2&prefix=qtsdkreposit" - "ory%2Fwindows_x86%2Fdesktop%2Ftools_maintenance%2F&encoding-type=url'" - "qtsdkrepository/", True), - ("qtsdkrepository/windows_x86/desktop/tools_maintenance/log-s3-2020-12-03" - "--10:18:11-xml.txt:fatal error: to the endpoint URL: 'https" - "://qt-cdn.s3.eu-west-1.amazonaws.com/?list-type=2&prefix=qtsdkreposit" - "ory%2Fwindows_x86%2Fdesktop%2Ftools_maintenance%2F&encoding-type=url'" - "qtsdkrepository/", False), - ("", False) + @testhelpers.asyncio_test_parallel_data( + ("qtsdkrepository/windows_x86/desktop/tools_maintenance/log-s3-2020-12-03--10:18:11-xml.t" + "xt:fatal error: Could not connect to the endpoint URL: 'https://qt-cdn.s3.eu-west-1.ama" + "zonaws.com/?list-type=2&prefix=qtsdkrepository%2Fwindows_x86%2Fdesktop%2Ftools_maintena" + "nce%2F&encoding-type=url'qtsdkrepository/", True), + ("qtsdkrepository/windows_x86/desktop/tools_maintenance/log-s3-2020-12-03--10:18:11-xml.t" + "xt:fatal error: to the endpoint URL: 'https://qt-cdn.s3.eu-west-1.amazonaws.com/?list-t" + "ype=2&prefix=qtsdkrepository%2Fwindows_x86%2Fdesktop%2Ftools_maintenance%2F&encoding-ty" + "pe=url'qtsdkrepository/", False), + ("", False) ) async def test_has_connection_error(self, output: str, expected_result: bool) -> None: self.assertEqual(expected_result, has_connection_error(output)) diff --git a/packaging-tools/tests/test_runCommand.py b/packaging-tools/tests/test_runCommand.py index 0340f8095..dd68bad28 100644 --- a/packaging-tools/tests/test_runCommand.py +++ b/packaging-tools/tests/test_runCommand.py @@ -51,16 +51,16 @@ def baseCommand(): def crash(): - '''\ - crash the Python interpreter... - ''' - i = ctypes.c_char(b'a') - j = ctypes.pointer(i) - c = 0 - while True: - j[c] = b'a' - c += 1 - j + '''\ + crash the Python interpreter... + ''' + i = ctypes.c_char(b'a') + j = ctypes.pointer(i) + c = 0 + while True: + j[c] = b'a' + c += 1 + j def printLines(count): @@ -89,15 +89,17 @@ class TestRunCommand(unittest.TestCase): def test_Crash_onlyErrorCaseOutput(self): with self.assertRaises(Exception) as contextManager: - useRunCommand("--printLines 10 --crash", os.getcwd(), - # callerArguments= - None, - # extra_environment= - None, - # onlyErrorCaseOutput= - True, - # expectedExitCodes= - [0]) + useRunCommand( + "--printLines 10 --crash", os.getcwd(), + # callerArguments= + None, + # extra_environment= + None, + # onlyErrorCaseOutput= + True, + # expectedExitCodes= + [0] + ) self.assertIsNotNone(contextManager) self.assertIsNotNone(contextManager.exception) expectedMessageStart = "Different exit code then expected" @@ -108,15 +110,19 @@ class TestRunCommand(unittest.TestCase): self.assertTrue(messageEnd.__contains__(expectedMessageEnd)) def test_differentExitCode_onlyErrorCaseOutput(self): - self.assertEqual(useRunCommand("--printLines 10 --exitCode 5", os.getcwd(), - # callerArguments= - None, - # extra_environment= - None, - # onlyErrorCaseOutput= - True, - # expectedExitCodes= - [0, 5]), 5) + self.assertEqual( + useRunCommand( + "--printLines 10 --exitCode 5", os.getcwd(), + # callerArguments= + None, + # extra_environment= + None, + # onlyErrorCaseOutput= + True, + # expectedExitCodes= + [0, 5] + ), 5 + ) def test_withThreadedWork(self): currentMethodName = sys._getframe().f_code.co_name -- cgit v1.2.3