From 01d17cf6af71dab74831825887667611fc90234a Mon Sep 17 00:00:00 2001 From: Patrik Teivonen Date: Mon, 16 Jan 2023 15:01:25 +0200 Subject: PL103: os.makedirs(path) should be replaced by path.mkdir(parents=True) Change-Id: Ie72f6555f99eeb8e499011d43d3035424ed9647f Reviewed-by: Iikka Eklund --- .pre-commit-config.yaml | 2 +- packaging-tools/bld_ifw_tools.py | 11 +++++------ packaging-tools/bld_lib.py | 6 +++--- packaging-tools/bld_python.py | 6 +++--- packaging-tools/bld_utils.py | 12 +++--------- packaging-tools/build_clang.py | 10 +++++----- packaging-tools/build_clang_qdoc.py | 2 +- packaging-tools/build_wrapper.py | 10 ++++------ packaging-tools/create_conan_executable.py | 2 +- packaging-tools/dump_debug_infos.py | 2 +- packaging-tools/installer_utils.py | 2 +- packaging-tools/libclang_training/run_batch_files.py | 2 +- packaging-tools/release_repo_meta_update.py | 2 +- packaging-tools/tests/test_content_cleaner.py | 3 ++- packaging-tools/tests/test_release_repo_meta_update.py | 5 +++-- packaging-tools/tests/test_release_repo_updater.py | 6 +++--- 16 files changed, 38 insertions(+), 45 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d0e65e66..c7a9666eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: # Disable E203 for compatibility with blackformatter, and W503 as it goes against PEP8 # Disable checks that are not relevant to this patch, they will be introduced later # - pathlib checks PLXXX - entry: pipenv run python3 -m flake8 --max-line-length=99 --ignore=E203,E501,W503,PL100,PL103,PL104,PL106,PL107,PL110,PL112,PL113,PL114,PL115,PL116,PL117,PL118,PL120,PL123 + entry: pipenv run python3 -m flake8 --max-line-length=99 --ignore=E203,E501,W503,PL100,PL104,PL106,PL107,PL110,PL112,PL113,PL114,PL115,PL116,PL117,PL118,PL120,PL123 language: system types: [python] fail_fast: true diff --git a/packaging-tools/bld_ifw_tools.py b/packaging-tools/bld_ifw_tools.py index dda9ae544..407760416 100755 --- a/packaging-tools/bld_ifw_tools.py +++ b/packaging-tools/bld_ifw_tools.py @@ -513,9 +513,9 @@ def create_installer_package(options: IfwOptions) -> None: log.info("Creating installer for Qt Installer Framework") # Temporary dir for creating installer containing the Qt Installer Framework itself package_dir = options.installer_framework_pkg_dir - os.makedirs(package_dir) + Path(package_dir).mkdir(parents=True, exist_ok=True) # Final directory for the installer containing the Qt Installer Framework itself - os.makedirs(options.installer_framework_target_dir) + Path(options.installer_framework_target_dir).mkdir(parents=True, exist_ok=True) target_dir = os.path.join(options.installer_framework_target_dir, 'QtInstallerFramework' + '-' + options.plat_suffix + '-' + options.architecture) with ch_dir(package_dir): shutil.copytree(os.path.join(options.installer_framework_build_dir, 'bin'), os.path.join(package_dir, 'bin'), ignore=shutil.ignore_patterns("*.exe.manifest", "*.exp", "*.lib")) @@ -533,11 +533,10 @@ def create_installer_package(options: IfwOptions) -> None: run_cmd(cmd=cmd_args, cwd=ROOT_DIR) shutil.move(os.path.join(ROOT_DIR, options.installer_framework_payload_arch), options.build_artifacts_dir) # create 7z - archive_file = os.path.join(options.installer_framework_source_dir, 'dist', 'packages', 'org.qtproject.ifw.binaries', 'data', 'data.7z') - if not os.path.exists(os.path.dirname(archive_file)): - os.makedirs(os.path.dirname(archive_file)) + archive_file = Path(options.installer_framework_source_dir, 'dist', 'packages', 'org.qtproject.ifw.binaries', 'data', 'data.7z') + archive_file.parent.mkdir(parents=True, exist_ok=True) archivegen = os.path.join(package_dir, 'bin', 'archivegen') - run_cmd(cmd=[archivegen, archive_file, '*'], cwd=package_dir) + run_cmd(cmd=[archivegen, str(archive_file), '*'], cwd=package_dir) # run installer binary_creator = os.path.join(options.installer_framework_build_dir, 'bin', 'binarycreator') config_file = os.path.join(options.installer_framework_source_dir, 'dist', 'config', 'config.xml') diff --git a/packaging-tools/bld_lib.py b/packaging-tools/bld_lib.py index e57b64a30..0913a0aff 100644 --- a/packaging-tools/bld_lib.py +++ b/packaging-tools/bld_lib.py @@ -96,7 +96,7 @@ def download_qt_pkg(args: argparse.Namespace, current_dir: str) -> Tuple[str, st def extract_archive(save_as: str, current_dir: str) -> str: qt_dest_dir = os.path.join(current_dir, "qt_pkg") if not os.path.exists(qt_dest_dir): - os.makedirs(qt_dest_dir) + Path(qt_dest_dir).mkdir(parents=True) log.info("Extracting to: %s", qt_dest_dir) if save_as.endswith("tar.gz"): with tarfile.open(save_as, "r:gz") as tar: @@ -133,11 +133,11 @@ def build(args: argparse.Namespace, qt_dest_dir: str, current_dir: str) -> str: install_root_dir = os.path.join(current_dir, "lib_install_root") shutil.rmtree(install_root_dir, ignore_errors=True) - os.makedirs(install_root_dir) + Path(install_root_dir).mkdir(parents=True) bld_dir = os.path.join(current_dir, "lib_bld") shutil.rmtree(bld_dir, ignore_errors=True) # ignore if path did not exist - os.makedirs(bld_dir) + Path(bld_dir).mkdir(parents=True) try: run_cmd(cmd=[qmake_tool, pro_file], cwd=bld_dir) diff --git a/packaging-tools/bld_python.py b/packaging-tools/bld_python.py index 26d479725..57f07dbb2 100644 --- a/packaging-tools/bld_python.py +++ b/packaging-tools/bld_python.py @@ -65,10 +65,10 @@ async def prepare_sources(src: str, tmp_base_dir: str) -> str: else: return src elif os.path.isfile(src): - os.makedirs(src_tmp_dir) + Path(src_tmp_dir).mkdir(parents=True) await extract_archive(src, src_tmp_dir) elif is_valid_url_path(src): - os.makedirs(src_tmp_dir) + Path(src_tmp_dir).mkdir(parents=True) dest_file = download_archive(src, tmp_base_dir) await extract_archive(dest_file, src_tmp_dir) else: @@ -150,7 +150,7 @@ async def _build_python(src_dir: str, bld_dir: str, prefix: str) -> str: make_install_cmd = ['make', 'install'] rmtree(bld_dir, ignore_errors=True) - os.makedirs(bld_dir) + Path(bld_dir).mkdir(parents=True) with ch_dir(bld_dir): await run_cmd_async(cmd=configure_cmd) diff --git a/packaging-tools/bld_utils.py b/packaging-tools/bld_utils.py index 9be6e8c7e..a71d8fb41 100644 --- a/packaging-tools/bld_utils.py +++ b/packaging-tools/bld_utils.py @@ -152,10 +152,7 @@ def download(url: str, target: str, read_block_size: int = 1048576) -> None: def local_download(local_file_path: str, target_file_path: str) -> None: if os.path.isfile(local_file_path): log.info("copying file from '%s' to '%s'", local_file_path, target_file_path) - try: - os.makedirs(os.path.dirname(target_file_path)) - except Exception: - pass + Path(target_file_path).parent.mkdir(parents=True, exist_ok=True) shutil.copy2(local_file_path, target) log.info("Done") @@ -173,10 +170,7 @@ def download(url: str, target: str, read_block_size: int = 1048576) -> None: return savefile_tmp = os.extsep.join((target, 'tmp')) - try: - os.makedirs(os.path.dirname(savefile_tmp)) - except Exception: - pass + Path(savefile_tmp).parent.mkdir(parents=True, exist_ok=True) try: # use urlopen which raise an error if that file is not existing @@ -268,7 +262,7 @@ def run_command(command: Union[List[str], str], cwd: str, extra_environment: Opt command_as_list[0] = found_executable if cwd and not os.path.lexists(cwd): - os.makedirs(cwd) + Path(cwd).mkdir(parents=True) log.info("========================== do ... ==========================") if cwd: diff --git a/packaging-tools/build_clang.py b/packaging-tools/build_clang.py index 4247b7b16..e0d3abd00 100644 --- a/packaging-tools/build_clang.py +++ b/packaging-tools/build_clang.py @@ -178,8 +178,8 @@ def mingw_training( qt_mingw_dir = os.path.join(base_path, 'qt_mingw') # Create some paths - os.makedirs(creator_settings_dir) - os.makedirs(creator_logs_dir) + Path(creator_settings_dir).mkdir(parents=True) + Path(creator_logs_dir).mkdir(parents=True) pkg_server = get_pkg_value("PACKAGE_STORAGE_SERVER") @@ -408,7 +408,7 @@ def build_clang( build_type: str = "Release", ) -> None: if build_path and not os.path.lexists(build_path): - os.makedirs(build_path) + Path(build_path).mkdir(parents=True) cmake_cmd = get_cmake_command( toolchain, src_path, install_path, profile_data_path, first_run, bitness, build_type @@ -438,7 +438,7 @@ def build_clazy( environment: Optional[Dict[str, str]] = None, ) -> None: if build_path and not os.path.lexists(build_path): - os.makedirs(build_path) + Path(build_path).mkdir(parents=True) cmake_cmd = ['cmake', '-G', cmake_generator(), @@ -559,7 +559,7 @@ def main() -> None: if os.path.exists(profile_data_path): rmtree(profile_data_path) - os.makedirs(profile_data_path) + Path(profile_data_path).mkdir(parents=True) # Update the regular build, so that we can see the differences result_file_path = os.path.join(base_path, 'libclang-' + branch + '-' + os.environ['CLANG_PLATFORM'] + '-regular.7z') diff --git a/packaging-tools/build_clang_qdoc.py b/packaging-tools/build_clang_qdoc.py index 97377032a..bb0993c21 100644 --- a/packaging-tools/build_clang_qdoc.py +++ b/packaging-tools/build_clang_qdoc.py @@ -229,7 +229,7 @@ def build_clang( build_type: str = "Release", ) -> None: if build_path and not os.path.lexists(build_path): - os.makedirs(build_path) + Path(build_path).mkdir(parents=True) cmake_cmd = cmake_command(toolchain, src_path, install_path, bitness, build_type) diff --git a/packaging-tools/build_wrapper.py b/packaging-tools/build_wrapper.py index a31f3138e..ca465fe76 100644 --- a/packaging-tools/build_wrapper.py +++ b/packaging-tools/build_wrapper.py @@ -371,8 +371,7 @@ def create_qtcreator_source_package(option_dict: Dict[str, str], source_path: st create_tar = is_linux() create_zip = is_windows() and '32' not in option_dict['TARGET_ENV'] if create_tar or create_zip: - if not os.path.exists(target_base): - os.makedirs(target_base) + Path(target_base).mkdir(parents=True, exist_ok=True) copy_tree(source_path, target_base) if create_tar: check_call_log(['tar', 'czf', file_base + '.tar.gz', '--exclude', '.git', file_base], @@ -596,8 +595,7 @@ def repackage_qtcreator( log_filepath: Optional[str] = None, ) -> None: extract_path = os.path.join(work_dir, 'temp_repackaged_qtc') - if not os.path.exists(extract_path): - os.makedirs(extract_path) + Path(extract_path).mkdir(parents=True, exist_ok=True) # extract Qt Creator check_call_log(['7z', 'x', '-y', os.path.join(work_dir, qtcreator_package), @@ -633,7 +631,7 @@ def handle_qt_creator_build(option_dict: Dict[str, str], qtcreator_plugins: List qtcreator_source_directory = os.path.join(work_dir, 'qt-creator') if os.path.exists(qtcreator_source_directory): shutil.rmtree(qtcreator_source_directory) - os.makedirs(qtcreator_source_directory) + Path(qtcreator_source_directory).mkdir(parents=True) clone_repository(option_dict['QT_CREATOR_GIT_URL'], option_dict['QT_CREATOR_GIT_BRANCH'], qtcreator_source_directory, full_clone=True, init_subrepos=True) # Get Qt Creator plugin sources if not present yet @@ -642,7 +640,7 @@ def handle_qt_creator_build(option_dict: Dict[str, str], qtcreator_plugins: List if plugin_conf.git_url: if os.path.exists(checkout_dir): shutil.rmtree(checkout_dir) - os.makedirs(checkout_dir) + Path(checkout_dir).mkdir(parents=True) clone_repository(plugin_conf.git_url, plugin_conf.branch_or_tag, checkout_dir, full_clone=True) # Build time variables diff --git a/packaging-tools/create_conan_executable.py b/packaging-tools/create_conan_executable.py index ff8da5794..fcf01e974 100644 --- a/packaging-tools/create_conan_executable.py +++ b/packaging-tools/create_conan_executable.py @@ -59,7 +59,7 @@ def locate_file_from_venv(venv_folder: str, file_name: str) -> str: async def clone_repo(url: str, destination_dir: str, env: Dict[str, str]) -> None: assert not os.path.isdir(destination_dir), f"Destination dir already exists: {destination_dir}" - os.makedirs(os.path.dirname(destination_dir), exist_ok=True) + Path(destination_dir).parent.mkdir(parents=True, exist_ok=True) log.info("Cloning repo: %s -> %s", url, destination_dir) cmd = ["git", "clone", url, destination_dir] await run_cmd_async(cmd=cmd, env=env, timeout=60 * 15) # give it 15 mins diff --git a/packaging-tools/dump_debug_infos.py b/packaging-tools/dump_debug_infos.py index 391b9aab5..b1759583d 100644 --- a/packaging-tools/dump_debug_infos.py +++ b/packaging-tools/dump_debug_infos.py @@ -162,7 +162,7 @@ def _main() -> None: if os.path.exists(args.output_path): if args.clean_output_path: rmtree(args.output_path, ignore_errors=True) - os.makedirs(args.output_path) + Path(args.output_path).mkdir(parents=True) for search_path in args.search_pathes.split(","): if not os.path.isdir(search_path): diff --git a/packaging-tools/installer_utils.py b/packaging-tools/installer_utils.py index cb6a7b549..91c4ec48a 100644 --- a/packaging-tools/installer_utils.py +++ b/packaging-tools/installer_utils.py @@ -89,7 +89,7 @@ async def extract_archive(artifact: str, destination_dir: str) -> None: log.info("Extracting file: %s into: %s", artifact, destination_dir) extract_cmd = get_extract_cmd(artifact) try: - os.makedirs(destination_dir, exist_ok=True) + Path(destination_dir).mkdir(parents=True, exist_ok=True) with ch_dir(destination_dir): run_cmd(cmd=extract_cmd) except Exception: diff --git a/packaging-tools/libclang_training/run_batch_files.py b/packaging-tools/libclang_training/run_batch_files.py index d48bb1d39..244661d41 100644 --- a/packaging-tools/libclang_training/run_batch_files.py +++ b/packaging-tools/libclang_training/run_batch_files.py @@ -231,7 +231,7 @@ def create_dir(dir_path: str) -> None: if not os.path.exists(dir_path): if Config.Verbose: print(f"info: creating not existent {dir_path}") - os.makedirs(dir_path) + Path(dir_path).mkdir(parents=True) def create_backup_file(file_path: str) -> None: diff --git a/packaging-tools/release_repo_meta_update.py b/packaging-tools/release_repo_meta_update.py index 9b546a2c5..ca646e74b 100755 --- a/packaging-tools/release_repo_meta_update.py +++ b/packaging-tools/release_repo_meta_update.py @@ -65,7 +65,7 @@ async def fetch_repogen(ifw_tools_url: str) -> str: current_dir = Path.cwd() ifw_tools_dir = os.path.join(current_dir, "ifw_tools") if not os.path.isdir(ifw_tools_dir): - os.makedirs(ifw_tools_dir) + Path(ifw_tools_dir).mkdir(parents=True) dest_file = download_archive(ifw_tools_url, ifw_tools_dir) await extract_archive(dest_file, ifw_tools_dir) tool_name = "repogen" diff --git a/packaging-tools/tests/test_content_cleaner.py b/packaging-tools/tests/test_content_cleaner.py index 377252894..3de0d45a4 100644 --- a/packaging-tools/tests/test_content_cleaner.py +++ b/packaging-tools/tests/test_content_cleaner.py @@ -32,6 +32,7 @@ import os import unittest +from pathlib import Path from typing import List from ddt import data, ddt, unpack # type: ignore @@ -45,7 +46,7 @@ class TestContentCleaner(unittest.TestCase): def generate_test_content(self, test_base_dir: str, test_content_paths: List[str]) -> None: for test_content_path in test_content_paths: test_path = os.path.join(test_base_dir, test_content_path) - os.makedirs(os.path.dirname(test_path), exist_ok=True) + Path(test_path).parent.mkdir(parents=True, exist_ok=True) if not test_path.endswith("/"): with open(test_path, "w+", encoding="utf-8") as handle: handle.write("") diff --git a/packaging-tools/tests/test_release_repo_meta_update.py b/packaging-tools/tests/test_release_repo_meta_update.py index 643a7ea70..42eec247b 100755 --- a/packaging-tools/tests/test_release_repo_meta_update.py +++ b/packaging-tools/tests/test_release_repo_meta_update.py @@ -31,6 +31,7 @@ import os import unittest +from pathlib import Path from typing import List from ddt import ddt # type: ignore @@ -98,7 +99,7 @@ class TestReleaseRepoMetaUpdate(unittest.TestCase): def _write_test_repo(self, tmp_base_dir: str, paths: List[str]) -> None: for path in paths: tmp = os.path.join(tmp_base_dir, path) - os.makedirs(os.path.dirname(tmp), exist_ok=True) + Path(tmp).parent.mkdir(parents=True, exist_ok=True) if tmp.endswith((".xml", ".7z")): with open(tmp, 'w+', encoding="utf-8") as handle: handle.write("\n") @@ -155,7 +156,7 @@ class TestReleaseRepoMetaUpdate(unittest.TestCase): self.assertTrue(not failed_conversions) # as it was dry-run we need to create the dummy migrated repo directories here for _, migrated_repo in successful_conversions.items(): - os.makedirs(migrated_repo) + Path(migrated_repo).mkdir(parents=True) operations_ok, operations_nok = swap_repositories(successful_conversions) self.assertTrue(not operations_nok) self.assertListEqual(sorted(successful_conversions.keys()), sorted(operations_ok.keys())) diff --git a/packaging-tools/tests/test_release_repo_updater.py b/packaging-tools/tests/test_release_repo_updater.py index f45507aa8..8052d4b39 100755 --- a/packaging-tools/tests/test_release_repo_updater.py +++ b/packaging-tools/tests/test_release_repo_updater.py @@ -65,13 +65,13 @@ from tests.testhelpers import ( def _write_dummy_file(path: str) -> None: - os.makedirs(os.path.dirname(path), exist_ok=True) + Path(path).parent.mkdir(parents=True, exist_ok=True) with open(path, 'w+', encoding="utf-8") as handle: handle.write("\n") def _write_package_xml(path: str, version: str, release_date: str) -> None: - os.makedirs(os.path.dirname(path), exist_ok=True) + Path(path).parent.mkdir(parents=True, exist_ok=True) with open(path, 'w+', encoding="utf-8") as handle: handle.write("\n") handle.write("\n") @@ -84,7 +84,7 @@ def _write_package_xml(path: str, version: str, release_date: str) -> None: def _write_updates_xml(path: str, version: str, release_date: str) -> None: - os.makedirs(os.path.dirname(path), exist_ok=True) + Path(path).parent.mkdir(parents=True, exist_ok=True) with open(path, 'w+', encoding="utf-8") as handle: handle.write("\n") handle.write(" {AnyApplication}\n") -- cgit v1.2.3