From d139575a56526190d9ab5dd4cb0be171a197675d Mon Sep 17 00:00:00 2001 From: Patrik Teivonen Date: Mon, 5 Sep 2022 14:54:30 +0300 Subject: Remove win32api dependency, adapt create_installer.py -When path limit disabled in Windows VMs, short pathnames no longer needed -Changes in create_installer.py, release_repo_updater.py: -Log warning in InstallerTask when running with path limit enabled -Force long path support when calling via the CLI interface -Add CLI option --disable-path-limit-check to bypass the check Change-Id: Ifed8f491ca4cdcb5b166365faccff8782059cecd Reviewed-by: Iikka Eklund --- Pipfile | 1 - packaging-tools/bldinstallercommon.py | 37 +++++++++++++++++++++------------ packaging-tools/create_installer.py | 23 +++++++++++++------- packaging-tools/release_repo_updater.py | 13 +++++++++++- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/Pipfile b/Pipfile index bc0cecb70..5112f4ed9 100644 --- a/Pipfile +++ b/Pipfile @@ -13,7 +13,6 @@ pexpect = "==4.8.0" wget = "==3.2" beautifulsoup4 = "==4.11.1" pysftp = "==0.2.9" -pywin32 = {version = "==303", os_name = "== 'nt'"} configparser = "==5.2.0" dataclasses = {version = "==0.8", markers="(python_version >= '3.6' and python_version < '3.7')"} asyncio-backport = {version = "==0.1.1", markers="(python_version >= '3.6' and python_version < '3.7')"} diff --git a/packaging-tools/bldinstallercommon.py b/packaging-tools/bldinstallercommon.py index 05a8bee75..f039d484a 100644 --- a/packaging-tools/bldinstallercommon.py +++ b/packaging-tools/bldinstallercommon.py @@ -29,6 +29,7 @@ # ############################################################################# +import ctypes import errno import os import re @@ -56,15 +57,33 @@ from logging_util import init_logger from runner import run_cmd from threadedwork import Task, ThreadedWork -# need to include this for win platforms as long path names cause problems -if is_windows(): - import win32api # type: ignore # pylint: disable=E0401 - log = init_logger(__name__, debug_mode=False) MAX_DEBUG_PRINT_LENGTH = 10000 +def is_long_path_supported() -> bool: + """ + Check whether long paths (~260+) are supported by the current environment + + On Windows, the limitations can be removed via Group Policy or Registry Key + For more information on Windows API maximum path length limitations, see: + https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation + + Returns: + bool: whether the current process environment has path limitation disabled (Windows) + True: non-Windows platforms + """ + if not is_windows(): + return True + ntdll = ctypes.WinDLL('ntdll') # type: ignore # false positive mypy + if not hasattr(ntdll, 'RtlAreLongPathsEnabled'): + return False + ntdll.RtlAreLongPathsEnabled.restype = ctypes.c_ubyte + ntdll.RtlAreLongPathsEnabled.argtypes = () + return bool(ntdll.RtlAreLongPathsEnabled()) + + def file_uri_to_path(uri: str) -> Path: """ Convert file:// uris and string paths to pathlib path @@ -173,10 +192,6 @@ def search_for_files( # function ############################### def move_tree(srcdir: str, dstdir: str, pattern: Optional[str] = None) -> None: - # windows has length limit for path names so try to truncate them as much as possible - if is_windows(): - srcdir = win32api.GetShortPathName(srcdir) - dstdir = win32api.GetShortPathName(dstdir) # dstdir must exist first srcnames = os.listdir(srcdir) for name in srcnames: @@ -202,10 +217,6 @@ def move_tree(srcdir: str, dstdir: str, pattern: Optional[str] = None) -> None: # function ############################### def copy_tree(source_dir: str, dest_dir: str) -> None: - # windows has length limit for path names so try to truncate them as much as possible - if is_windows(): - source_dir = win32api.GetShortPathName(source_dir) - dest_dir = win32api.GetShortPathName(dest_dir) src_files = os.listdir(source_dir) for file_name in src_files: full_file_name = os.path.join(source_dir, file_name) @@ -253,7 +264,7 @@ def handle_remove_readonly( def remove_tree(path: str) -> bool: if os.path.isdir(path) and os.path.exists(path): if is_windows(): - path = win32api.GetShortPathName(path.replace('/', '\\')) + path = path.replace('/', '\\') # a funny thing is that rmdir does not set an exitcode it is just using the last set one try: cmd = ['rmdir', path, '/S', '/Q'] diff --git a/packaging-tools/create_installer.py b/packaging-tools/create_installer.py index 49c817bac..3359e736e 100644 --- a/packaging-tools/create_installer.py +++ b/packaging-tools/create_installer.py @@ -51,6 +51,7 @@ from bldinstallercommon import ( copy_tree, extract_file, handle_component_rpath, + is_long_path_supported, locate_executable, locate_path, locate_paths, @@ -69,9 +70,6 @@ from runner import run_cmd from sdkcomponent import IfwPayloadItem, IfwSdkComponent, parse_ifw_sdk_comp from threadedwork import ThreadedWork -if is_windows(): - import win32api # type: ignore # pylint: disable=E0401 - log = init_logger(__name__, debug_mode=False) QtInstallerTaskType = TypeVar("QtInstallerTaskType", bound="QtInstallerTask[Any]") @@ -842,9 +840,6 @@ def create_target_components(task: QtInstallerTaskType) -> None: # Create needed data dirs before the threads start to work install_dir.mkdir(parents=True, exist_ok=True) data_dir_dest.mkdir(parents=True, exist_ok=True) - if is_windows(): - install_dir = Path(win32api.GetShortPathName(str(install_dir))) - data_dir_dest = Path(win32api.GetShortPathName(str(data_dir_dest))) get_component_data_work.add_task( f"adding {archive.archive_name} to {sdk_comp.ifw_sdk_comp_name}", get_component_data, @@ -1187,6 +1182,8 @@ class QtInstallerTask(Generic[QtInstallerTaskType]): self.config.get("PackageTemplates", "template_dirs"), self.configurations_dir ) self._parse_substitutions() + if not is_long_path_supported(): + log.warning("Path names longer than 260 are not supported by the current environment") def __str__(self) -> str: return f"""Installer task: @@ -1212,7 +1209,8 @@ class QtInstallerTask(Generic[QtInstallerTaskType]): Build timestamp: {self.build_timestamp} Force version number increase: {self.force_version_number_increase} Version number auto increase value: {self.version_number_auto_increase_value} - Mac cpu count: {self.max_cpu_count}""" + Mac cpu count: {self.max_cpu_count} + Long paths supported: {is_long_path_supported()}""" def _parse_substitutions(self) -> None: for item in self.substitution_list: # pylint: disable=not-an-iterable @@ -1354,9 +1352,20 @@ def main() -> None: parser.add_argument("--max-cpu-count", dest="max_cpu_count", type=int, default=8, help="Set maximum number of CPU's used on packaging") + parser.add_argument( + "--disable-path-limit-check", + dest="require_long_path_support", + action="store_false", + ) args = parser.parse_args(sys.argv[1:]) + if args.require_long_path_support is True and is_long_path_supported() is False: + log.error("Path names longer than 260 are not supported by the current environment") + log.error("To continue, the maximum path limitation must be disabled in Windows registry") + log.error("Set --disable-path-limit-check to bypass this check") + raise SystemExit("Long path support is required to build the installer/repository") + task: QtInstallerTask[Any] = QtInstallerTask( configurations_dir=args.configurations_dir, configuration_file=args.configuration_file, diff --git a/packaging-tools/release_repo_updater.py b/packaging-tools/release_repo_updater.py index 8abe796fd..6ad82b057 100755 --- a/packaging-tools/release_repo_updater.py +++ b/packaging-tools/release_repo_updater.py @@ -52,7 +52,7 @@ from urllib.request import urlopen, urlretrieve from temppathlib import TemporaryDirectory from bld_utils import is_linux -from bldinstallercommon import locate_path +from bldinstallercommon import is_long_path_supported, locate_path from create_installer import DryRunMode, QtInstallerTask, create_installer from installer_utils import PackagingError, download_archive, extract_archive, is_valid_url_path from logging_util import init_logger @@ -963,8 +963,19 @@ def main() -> None: parser.add_argument("--event-injector", dest="event_injector", type=str, default=os.getenv('PKG_EVENT_INJECTOR'), help="Register events to monitoring system with the given injector. " "The --config file must point to export summary file.") + parser.add_argument( + "--disable-path-limit-check", + dest="require_long_path_support", + action="store_false", + default=True + ) parser.set_defaults(**defaults) # these are from provided --config file args = parser.parse_args(sys.argv[1:]) + if args.require_long_path_support is True and is_long_path_supported() is False: + log.error("Path names longer than 260 are not supported by the current environment") + log.error("To continue, the maximum path limitation must be disabled in Windows registry") + log.error("Set --disable-path-limit-check to bypass this check") + raise SystemExit("Long path support is required to build the installer/repository") if args.dry_run: assert not args.sync_s3, "'--sync-s3' specified with '--dry-run'" assert not args.sync_ext, "'--sync-ext' specified with '--dry-run'" -- cgit v1.2.3