diff options
26 files changed, 284 insertions, 265 deletions
@@ -7,7 +7,6 @@ name = "pypi" pip = {version = ">=18.0"} sh = {version = "==1.14.2", os_name = "!= 'nt'"} typed-ast = "==1.5.3" -rainbow_logging_handler = "==2.2.2" gitpython = "==3.1.18" colorlog = "==6.6.0" pexpect = "==4.8.0" diff --git a/packaging-tools/archiveresolver.py b/packaging-tools/archiveresolver.py index 051a54257..e7a47f8c1 100644 --- a/packaging-tools/archiveresolver.py +++ b/packaging-tools/archiveresolver.py @@ -35,11 +35,13 @@ from typing import List from urllib.parse import urlparse from bldinstallercommon import config_section_map, is_content_url_valid, safe_config_key_fetch +from logging_util import init_logger from pkg_constants import PKG_TEMPLATE_BASE_DIR_NAME SERVER_NAMESPACE = 'ArchiveRemoteLocation' PACKAGE_REMOTE_LOCATION_RELEASE = 'release' PACKAGE_ARCHIVE_TAG = 'ARCHIVE_TAG' +log = init_logger(__name__, debug_mode=False) ############################### @@ -163,9 +165,9 @@ class ArchiveLocationResolver: # Print out server list ############################### def print_server_list(self) -> None: - print('--------------------------------------------------') - print(' Server list:') + log.info("--------------------------------------------------") + log.info(" Server list:") for server in self.server_list: - print(' ---------------------------------------------') - print(f' Server name: {server.server_name}') - print(f' Server url: {server.server_url}') + log.info(" ---------------------------------------------") + log.info(" Server name: %s", server.server_name) + log.info(" Server url: %s", server.server_url) diff --git a/packaging-tools/bld_ifw_tools.py b/packaging-tools/bld_ifw_tools.py index 456a67545..389caf698 100755 --- a/packaging-tools/bld_ifw_tools.py +++ b/packaging-tools/bld_ifw_tools.py @@ -54,10 +54,12 @@ from bldinstallercommon import ( retrieve_url, ) from installer_utils import PackagingError, ch_dir +from logging_util import init_logger from pkg_constants import IFW_BUILD_ARTIFACTS_DIR from read_remote_config import get_pkg_value from runner import do_execute_sub_process +log = init_logger(__name__, debug_mode=False) ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) ARCH_EXT = '.zip' if is_windows() else '.tar.xz' QT_VERSION = '5.12' @@ -67,11 +69,11 @@ QT_VERSION_MINOR = '5.12.7' ARCHIVE_PROGRAM = '7z' if is_macos(): try: - print("Trying to use 7z from") - subprocess.call(['where', '7z']) + log.info("Trying to use 7z from") + subprocess.call(["where", "7z"]) except OSError: - print("7z not found from path. Try to use 7z from /usr/local/bin") - ARCHIVE_PROGRAM = '/usr/local/bin/7z' + log.warning("7z not found from path. Try to use 7z from /usr/local/bin") + ARCHIVE_PROGRAM = "/usr/local/bin/7z" ################################################################## @@ -257,38 +259,38 @@ class IfwOptions: raise SystemExit(f"Qt src package uri is invalid: {self.qt_source_package_uri}") if self.product_key_checker_pri: if os.path.isfile(self.product_key_checker_pri): - print(f'Using product key checker: {self.product_key_checker_pri}') + log.info("Using product key checker: %s", self.product_key_checker_pri) else: raise SystemExit(f"Invalid product key checker: {self.product_key_checker_pri}") def print_data(self) -> None: - print("-----------------------------------------") - print(f"make cmd: {self.make_cmd}") - print(f"make doc_cmd: {self.make_doc_cmd}") - print(f"qt_src_pkg_uri: {self.qt_source_package_uri}") - print(f"qt_src_pkg_uri_saveas: {self.qt_source_package_uri_saveas}") - print(f"qt_src_dir: {self.qt_source_dir}") - print(f"qt_bld_dir: {self.qt_build_dir}") - print(f"qt_bld_dir_dynamic: {self.qt_build_dir_dynamic}") - print(f"qt_configure_options: {self.qt_configure_options}") - print(f"qt_qmake_bin: {self.qt_qmake_bin}") - print(f"qt_configure_bin: {self.qt_configure_bin}") - print(f"qt_ifw_uri: {self.qt_installer_framework_uri}") - print(f"qt_ifw_branch: {self.qt_installer_framework_branch}") - print(f"qt_ifw_qmake_args: {self.qt_installer_framework_qmake_args}") - print(f"openssl_dir: {self.openssl_dir}") - print(f"ifw_src_dir: {self.installer_framework_source_dir}") - print(f"ifw_bld_dir: {self.installer_framework_build_dir}") - print(f"ifw_archive_name: {self.installer_framework_archive_name}") - print(f"installer_base_archive_name: {self.installer_base_archive_name}") - print(f"binarycreator_archive_name: {self.binarycreator_archive_name}") - print(f"ifw_pkg_dir: {self.installer_framework_pkg_dir}") - print(f"ifw_tgt_dir: {self.installer_framework_target_dir}") - print(f"ifw_payload_arch: {self.installer_framework_payload_arch}") - print(f"product_key_checker: {self.product_key_checker_pri}") - print(f"qt_binaries_static: {self.qt_binaries_static}") - print(f"qt_binaries_dynamic: {self.qt_binaries_dynamic}") - print("-----------------------------------------") + log.info("-----------------------------------------") + log.info("make cmd: %s", self.make_cmd) + log.info("make doc_cmd: %s", self.make_doc_cmd) + log.info("qt_source_package_uri: %s", self.qt_source_package_uri) + log.info("qt_source_package_uri_saveas: %s", self.qt_source_package_uri_saveas) + log.info("qt_source_dir: %s", self.qt_source_dir) + log.info("qt_build_dir: %s", self.qt_build_dir) + log.info("qt_build_dir_dynamic: %s", self.qt_build_dir_dynamic) + log.info("qt_configure_options: %s", self.qt_configure_options) + log.info("qt_qmake_bin: %s", self.qt_qmake_bin) + log.info("qt_configure_bin: %s", self.qt_configure_bin) + log.info("qt_installer_framework_uri: %s", self.qt_installer_framework_uri) + log.info("qt_installer_framework_branch: %s", self.qt_installer_framework_branch) + log.info("qt_installer_framework_qmake_args: %s", self.qt_installer_framework_qmake_args) + log.info("openssl_dir: %s", self.openssl_dir) + log.info("installer_framework_source_dir: %s", self.installer_framework_source_dir) + log.info("installer_framework_build_dir: %s", self.installer_framework_build_dir) + log.info("installer_framework_archive_name: %s", self.installer_framework_archive_name) + log.info("installer_base_archive_name: %s", self.installer_base_archive_name) + log.info("binarycreator_archive_name: %s", self.binarycreator_archive_name) + log.info("installer_framework_pkg_dir: %s", self.installer_framework_pkg_dir) + log.info("installer_framework_target_dir: %s", self.installer_framework_target_dir) + log.info("installer_framework_payload_arch: %s", self.installer_framework_payload_arch) + log.info("product_key_checker_pri: %s", self.product_key_checker_pri) + log.info("qt_binaries_static: %s", self.qt_binaries_static) + log.info("qt_binaries_dynamic: %s", self.qt_binaries_dynamic) + log.info("-----------------------------------------") ############################### @@ -339,8 +341,8 @@ def build_ifw(options: IfwOptions, create_installer: bool = False, build_ifw_exa def prepare_qt_sources(options: IfwOptions) -> None: if options.incremental_mode and os.path.exists(options.qt_source_dir): return - print('--------------------------------------------------------------------') - print(f"Prepare Qt src package: {options.qt_source_package_uri}") + log.info("--------------------------------------------------------------------") + log.info("Prepare Qt src package: %s", options.qt_source_package_uri) prepare_compressed_package(options.qt_source_package_uri, options.qt_source_package_uri_saveas, options.qt_source_dir) if is_windows(): @@ -351,15 +353,15 @@ def prepare_qt_sources(options: IfwOptions) -> None: # function ############################### def prepare_compressed_package(src_pkg_uri: str, src_pkg_saveas: str, destination_dir: str) -> None: - print(f"Fetching package from: {src_pkg_uri}") + log.info("Fetching package from: %s", src_pkg_uri) if not os.path.isfile(src_pkg_saveas): if not is_content_url_valid(src_pkg_uri): raise SystemExit("Src package uri is invalid! Abort!") retrieve_url(src_pkg_uri, src_pkg_saveas) else: - print(f"Found old local package, using that: {src_pkg_saveas}") - print('Done') - print('--------------------------------------------------------------------') + log.info("Found old local package, using that: %s", src_pkg_saveas) + log.info("Done") + log.info("--------------------------------------------------------------------") Path(destination_dir).mkdir(parents=True, exist_ok=True) extract_file(src_pkg_saveas, destination_dir) dir_contents = os.listdir(destination_dir) @@ -387,21 +389,21 @@ def build_qt(options: IfwOptions, qt_build_dir: str, qt_configure_options: str, pass Path(qt_build_dir).mkdir(parents=True, exist_ok=True) # configure first - print('--------------------------------------------------------------------') - print('Configuring Qt') + log.info("--------------------------------------------------------------------") + log.info("Configuring Qt") configure_options = re.sub(' +', ' ', qt_configure_options) cmd_args = options.qt_configure_bin + ' ' + configure_options # shlex does not like backslashes cmd_args = cmd_args.replace('\\', '/') do_execute_sub_process(shlex.split(cmd_args), options.qt_source_dir, True, False, get_build_env(options.openssl_dir)) - print('--------------------------------------------------------------------') - print('Building Qt') + log.info("--------------------------------------------------------------------") + log.info("Building Qt") cmd_args = options.make_cmd for module in qt_modules: cmd_args += " module-" + module do_execute_sub_process(cmd_args.split(' '), options.qt_source_dir, True, False, get_build_env(options.openssl_dir)) - print('--------------------------------------------------------------------') - print('Installing Qt') + log.info("--------------------------------------------------------------------") + log.info("Installing Qt") cmd_args = options.make_install_cmd for module in qt_modules: module_dir = os.path.join(options.qt_source_dir, module) @@ -414,8 +416,8 @@ def build_qt(options: IfwOptions, qt_build_dir: str, qt_configure_options: str, def prepare_installer_framework(options: IfwOptions) -> None: if options.incremental_mode and os.path.exists(options.installer_framework_source_dir): return - print('--------------------------------------------------------------------') - print('Prepare Installer Framework source') + log.info("--------------------------------------------------------------------") + log.info("Prepare Installer Framework source") # create dirs Path(options.installer_framework_build_dir).mkdir(parents=True, exist_ok=True) if options.qt_installer_framework_uri.endswith('.git'): @@ -428,10 +430,10 @@ def prepare_installer_framework(options: IfwOptions) -> None: def start_ifw_build(options: IfwOptions, cmd_args: List[str], installer_framework_build_dir: str) -> None: - print(f"cmd_args: {list_as_string(cmd_args)}") + log.info("cmd_args: %s", list_as_string(cmd_args)) do_execute_sub_process(cmd_args, installer_framework_build_dir) cmd_args = options.make_cmd.split(' ') - print(f"cmd_args: {list_as_string(cmd_args)}") + log.info("cmd_args: %s", list_as_string(cmd_args)) do_execute_sub_process(cmd_args, installer_framework_build_dir) @@ -446,8 +448,8 @@ def build_installer_framework(options: IfwOptions) -> None: if os.path.exists(file_to_check): return - print('--------------------------------------------------------------------') - print('Building Installer Framework') + log.info("--------------------------------------------------------------------") + log.info("Building Installer Framework") qmake_bin = os.path.join(options.qt_build_dir, 'qtbase', 'bin', options.qt_qmake_bin) if not os.path.isfile(qmake_bin): raise SystemExit(f"Unable to find qmake from {qmake_bin}, aborting!") @@ -459,8 +461,8 @@ def build_installer_framework(options: IfwOptions) -> None: def build_installer_framework_examples(options: IfwOptions) -> None: - print('--------------------------------------------------------------------') - print('Building Installer Framework Examples') + log.info("--------------------------------------------------------------------") + log.info("Building Installer Framework Examples") file_binarycreator = os.path.join(options.installer_framework_build_dir, 'bin', 'binarycreator') if is_windows(): file_binarycreator += '.exe' @@ -476,7 +478,7 @@ def build_installer_framework_examples(options: IfwOptions) -> None: if 'translations' in dirs: dirs.remove('translations') # for now don't visit translation example as qm files needs to be generated first for directory in dirs: - print(f"********** building example {directory}") + log.info("Building example %s", directory) config_file = os.path.join(root, directory, 'config', 'config.xml') package_dir = os.path.join(root, directory, 'packages') target_filename = os.path.join(root, directory, 'installer') @@ -492,8 +494,8 @@ def build_installer_framework_examples(options: IfwOptions) -> None: # function ############################### def build_ifw_docs(options: IfwOptions) -> None: - print('--------------------------------------------------------------------') - print('Building Qt Installer Framework Documentation') + log.info("--------------------------------------------------------------------") + log.info("Building Qt Installer Framework Documentation") qmake_bin = os.path.join(options.qt_build_dir_dynamic, 'qtbase', 'bin', options.qt_qmake_bin) if not os.path.isfile(qmake_bin): raise SystemExit(f"Aborting doc build, qmake not found in: {options.qt_build_dir_dynamic}") @@ -512,8 +514,8 @@ def build_ifw_docs(options: IfwOptions) -> None: # Create installer containing the Qt Installer Framework itself ################################################################ def create_installer_package(options: IfwOptions) -> None: - print('--------------------------------------------------------------------') - print('Creating installer for Qt Installer Framework') + log.info("--------------------------------------------------------------------") + log.info("Creating installer for Qt Installer Framework") # Temporary dir for creating installer containing the Qt Installer Framework itself package_dir = options.installer_framework_pkg_dir os.makedirs(package_dir) @@ -545,7 +547,7 @@ def create_installer_package(options: IfwOptions) -> None: config_file = os.path.join(options.installer_framework_source_dir, 'dist', 'config', 'config.xml') package_dir = os.path.join(options.installer_framework_source_dir, 'dist', 'packages') do_execute_sub_process(args=[binary_creator, '--offline-only', '-c', config_file, '-p', package_dir, target_dir], execution_path=package_dir) - print(f"Installer package is at: {target_dir}") + log.info("Installer package is at: %s", target_dir) artifacts = os.listdir(options.installer_framework_target_dir) for artifact in artifacts: dest_file_name = os.path.join(options.build_artifacts_dir, artifact) @@ -558,18 +560,18 @@ def create_installer_package(options: IfwOptions) -> None: # Build and archive Qt for IFW builds ################################################################ def build_and_archive_qt(options: IfwOptions) -> None: - print('--------------------------------------------------------------------') - print('Build static Qt') + log.info("--------------------------------------------------------------------") + log.info("Build static Qt") prepare_qt_sources(options) build_qt(options, options.qt_build_dir, options.qt_configure_options, options.qt_build_modules) - print('--------------------------------------------------------------------') - print('Archive static Qt binaries') + log.info("--------------------------------------------------------------------") + log.info("Archive static Qt binaries") cmd_args_archive = [ARCHIVE_PROGRAM, 'a', options.qt_static_binary_name, options.qt_build_dir] do_execute_sub_process(cmd_args_archive, ROOT_DIR) - print('--------------------------------------------------------------------') - print('Build shared Qt') + log.info("--------------------------------------------------------------------") + log.info("Build shared Qt") configure_options = get_dynamic_qt_configure_options() + '-prefix ' + options.qt_build_dir_dynamic + os.sep + 'qtbase' # Although we have a shadow build qt sources are still contaminated. Unpack sources again. if os.path.exists(options.qt_source_dir): @@ -577,8 +579,8 @@ def build_and_archive_qt(options: IfwOptions) -> None: prepare_qt_sources(options) build_qt(options, options.qt_build_dir_dynamic, configure_options, options.qt_build_modules_docs) - print('--------------------------------------------------------------------') - print('Archive shared Qt binaries') + log.info("--------------------------------------------------------------------") + log.info("Archive shared Qt binaries") cmd_args_archive = [ARCHIVE_PROGRAM, 'a', options.qt_shared_binary_name, options.qt_build_dir_dynamic] do_execute_sub_process(cmd_args_archive, ROOT_DIR) @@ -621,8 +623,8 @@ def clean_build_environment(options: IfwOptions) -> None: # function ############################### def archive_installer_framework(installer_framework_build_dir: str, installer_framework_archive_name: str, options: IfwOptions, create_tagged_package: bool) -> None: - print('--------------------------------------------------------------------') - print('Archive Installer Framework') + log.info("--------------------------------------------------------------------") + log.info("Archive Installer Framework") # first strip out all unnecessary files for root, dummy, files in os.walk(installer_framework_build_dir): for filename in files: @@ -637,9 +639,9 @@ def archive_installer_framework(installer_framework_build_dir: str, installer_fr if options.qt_installer_framework_uri.endswith('.git') and create_tagged_package: tag = get_tag_from_branch(options.installer_framework_source_dir) if tag: - print(f"Create archive from tag {tag}") + log.info("Create archive from tag %s", tag) installer_framework_tagged_archive = 'installer-framework-build-' + tag + "-" + options.plat_suffix + '-' + options.architecture + '.7z' - print(f"Create archive {installer_framework_tagged_archive}") + log.info("Create archive %s", installer_framework_tagged_archive) shutil.copy(os.path.join(options.build_artifacts_dir, installer_framework_archive_name), os.path.join(options.build_artifacts_dir, installer_framework_tagged_archive)) @@ -647,8 +649,8 @@ def archive_installer_framework(installer_framework_build_dir: str, installer_fr # function ############################### def archive_installerbase(options: IfwOptions) -> None: - print('--------------------------------------------------------------------') - print('Archive Installerbase') + log.info("--------------------------------------------------------------------") + log.info("Archive Installerbase") cmd_args_archive = [] cmd_args_clean = [] bin_temp = '' @@ -677,8 +679,8 @@ def archive_installerbase(options: IfwOptions) -> None: # function ############################### def archive_binarycreator(options: IfwOptions) -> None: - print('--------------------------------------------------------------------') - print('Archive Installerbase and Binarycreator') + log.info("--------------------------------------------------------------------") + log.info("Archive Installerbase and Binarycreator") cmd_args_archive = [] if is_linux() or is_macos(): bin_path = locate_executable(options.installer_framework_build_dir, ['installerbase']) @@ -699,8 +701,8 @@ def archive_binarycreator(options: IfwOptions) -> None: # sign windows installerbase ############################### def sign_windows_installerbase(file_name: str) -> None: - print('--------------------------------------------------------------------') - print('Sign Windows Installerbase') + log.info("--------------------------------------------------------------------") + log.info("Sign Windows Installerbase") sign_tools_temp_dir = r'C:\Utils\sign_tools_temp' for item in ["signtool32.exe", "keys.pfx", "capicom.dll"]: dst = os.path.join(sign_tools_temp_dir, item) @@ -717,10 +719,10 @@ def sign_windows_installerbase(file_name: str) -> None: log_entry = cmd_args[:] log_entry[4] = "****" log_entry[6] = "****" - print("Calling: %s", " ".join(log_entry)) + log.info("Calling: %s", " ".join(log_entry)) subprocess.check_call(cmd_args, stderr=subprocess.STDOUT) # check_call() will consume output shutil.rmtree(sign_tools_temp_dir) - print(f"Successfully signed: {file_name}") + log.info("Successfully signed: %s", file_name) ############################### @@ -728,7 +730,7 @@ def sign_windows_installerbase(file_name: str) -> None: ############################### def patch(file: str, replacements: Dict[str, str]) -> None: filedata = None - print(f"Patching {file} ...") + log.info("Patching %s ...", file) with open(file, 'r', encoding="utf-8") as handle: filedata = handle.read() @@ -743,8 +745,8 @@ def patch(file: str, replacements: Dict[str, str]) -> None: # Patch win32 mkspecs ############################### def patch_win32_mkspecs(mkspecsdir: str) -> None: - print('--------------------------------------------------------------------') - print(f'Patching win32 mkspecs in {mkspecsdir} ...') + log.info("--------------------------------------------------------------------") + log.info("Patching win32 mkspecs in %s ...", mkspecsdir) for root, dummy, files in os.walk(mkspecsdir): for file in files: if "win32" in root and file == "qmake.conf": diff --git a/packaging-tools/bld_lib.py b/packaging-tools/bld_lib.py index e5e1bc711..cd019ac42 100644 --- a/packaging-tools/bld_lib.py +++ b/packaging-tools/bld_lib.py @@ -30,7 +30,6 @@ ############################################################################# import argparse -import logging import os import platform as plat # import as plat to not shadow the "import platform" import re @@ -46,22 +45,13 @@ from typing import List, Tuple from urllib.parse import urlparse from urllib.request import urlretrieve -from rainbow_logging_handler import RainbowLoggingHandler # type: ignore - from bld_utils import is_windows from installer_utils import ch_dir +from logging_util import init_logger from read_remote_config import get_pkg_value from remote_uploader import RemoteUploader -LOG_FMT_CI = "%(asctime)s %(levelname)s:%(filename)s:%(lineno)d(%(process)d): %(message)s" -log = logging.getLogger("Bld") -log.setLevel(logging.INFO) -# Unify format of all messages -handler = RainbowLoggingHandler(sys.stderr, color_asctime=(None, None, False)) - -formatter = logging.Formatter(LOG_FMT_CI) -handler.setFormatter(formatter) -log.addHandler(handler) +log = init_logger(__name__, debug_mode=False) def find_file(search_path: str, file_name: str) -> str: @@ -184,7 +174,7 @@ def archive(args: argparse.Namespace, install_root_dir: str, current_dir: str) - with ch_dir(archive_path): check_call(['7z', 'a', '-m0=lzma2', '-mmt=16', artifacts_file_path, '*']) except Exception as error: - print(str(error)) + log.error(str(error)) raise log.info("Created artifact: %s", artifacts_file_path) diff --git a/packaging-tools/bld_module.py b/packaging-tools/bld_module.py index 0d06d23ca..bda7a1180 100755 --- a/packaging-tools/bld_module.py +++ b/packaging-tools/bld_module.py @@ -59,9 +59,12 @@ from bldinstallercommon import ( search_for_files, ) from installer_utils import PackagingError +from logging_util import init_logger from runner import do_execute_sub_process from threadedwork import ThreadedWork +log = init_logger(__name__, debug_mode=False) + SCRIPT_ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) MODULE_SRC_DIR_NAME = 'module_src' MODULE_SRC_DIR = os.path.join(SCRIPT_ROOT_DIR, MODULE_SRC_DIR_NAME) @@ -88,7 +91,7 @@ def get_qt_install_prefix(qt_path: str) -> str: # function ############################### def erase_qmake_prl_build_dir(search_path: str) -> None: - print('--- Fix .prl files ---') + log.info("--- Fix .prl files ---") # fetch all .prl files file_list = locate_paths(search_path, ['*.prl'], filters=[os.path.isfile]) # erase lines starting with 'QMAKE_PRL_BUILD_DIR' from .prl files @@ -97,11 +100,11 @@ def erase_qmake_prl_build_dir(search_path: str) -> None: for line in FileInput(item, inplace=True): if line.startswith('QMAKE_PRL_BUILD_DIR'): found = True - print(''.rstrip('\n')) + log.info("".rstrip("\n")) else: - print(line.rstrip('\n')) + log.info(line.rstrip("\n")) if found: - print(f"Erased 'QMAKE_PRL_BUILD_DIR' from: {item}") + log.info("Erased 'QMAKE_PRL_BUILD_DIR' from: %s", item) ############################### @@ -113,12 +116,12 @@ def patch_build_time_paths(search_path: str, search_strings: List[str], qt_insta file_list = search_for_files(search_path, extension_list, search_regexp) for item in file_list: - print(f"Replacing {search_strings} paths from file: {item}") + log.info("Replacing %s paths from file: %s", search_strings, item) for line in FileInput(item, inplace=True): patched_line = reduce(lambda accum, value: accum.replace(value, qt_install_prefix), search_strings, line) - print(patched_line.rstrip('\n')) + log.info(patched_line.rstrip("\n")) def main() -> None: @@ -185,7 +188,10 @@ def main() -> None: # cleanup some values inside the caller_arguments object strip_vars(caller_arguments, "\"") if caller_arguments.qt5path != os.path.abspath(caller_arguments.qt5path): - print(f"changing the value of --qt5path from {caller_arguments.qt5path} to {os.path.abspath(caller_arguments.qt5path)}") + log.info( + "Changing the value of --qt5path: %s -> %s", + caller_arguments.qt5path, os.path.abspath(caller_arguments.qt5path) + ) caller_arguments.qt5path = os.path.abspath(caller_arguments.qt5path) if not caller_arguments.module_name: @@ -205,7 +211,7 @@ def main() -> None: my_get_qt_module.run() qt_module_source_directory = MODULE_SRC_DIR else: - print(f"Using local copy of {caller_arguments.module_name}") + log.info("Using local copy of %s", caller_arguments.module_name) qt_module_source_directory = caller_arguments.module_dir # install directory @@ -220,14 +226,17 @@ def main() -> None: # clean step if caller_arguments.clean: - print("##### clean old builds #####") + log.info("##### clean old builds #####") remove_tree(caller_arguments.qt5path) remove_tree(qt_module_install_directory) remove_tree(temp_path) if not os.path.lexists(caller_arguments.qt5path) and not caller_arguments.qt5_module_urls: parser.print_help() - print(f"error: Please add the missing qt5_module_url arguments if the {caller_arguments.qt5path} does not exist {os.linesep}{os.linesep}") + log.error( + "The --qt5path %s does not exist, please add the missing --qt5_module_url arguments", + caller_arguments.qt5path + ) raise RuntimeError() qmake_binary = os.path.abspath(os.path.join(caller_arguments.qt5path, 'bin', 'qmake')) diff --git a/packaging-tools/bld_openssl.py b/packaging-tools/bld_openssl.py index b7feb2a23..cce797c02 100644 --- a/packaging-tools/bld_openssl.py +++ b/packaging-tools/bld_openssl.py @@ -39,8 +39,11 @@ import subprocess import sys from typing import List +from logging_util import init_logger from runner import do_execute_sub_process +log = init_logger(__name__, debug_mode=False) + ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/packaging-tools/bld_utils.py b/packaging-tools/bld_utils.py index 3b677de85..8224f6c7d 100644 --- a/packaging-tools/bld_utils.py +++ b/packaging-tools/bld_utils.py @@ -48,6 +48,9 @@ from urllib.request import pathname2url, urlopen # 3rd party module to read process output in a convenient way from asynchronousfilereader import AsynchronousFileReader +from logging_util import init_logger + +log = init_logger(__name__, debug_mode=False) # make a timeout for download jobs setdefaulttimeout(30) @@ -82,8 +85,8 @@ class DirRenamer(): def __init__(self, path: str, new_name: str) -> None: self.old_name = path self.new_name = os.path.join(os.path.split(path)[0], new_name) - print(f"self.old_name: {self.old_name}") - print(f"self.new_name: {self.new_name}") + log.info("self.old_name: %s", self.old_name) + log.info("self.new_name: %s", self.new_name) def __enter__(self) -> None: if self.old_name != self.new_name: @@ -146,13 +149,13 @@ def download(url: str, target: str, read_block_size: int = 1048576) -> None: def local_download(local_file_path: str, target_file_path: str) -> None: if os.path.isfile(local_file_path): - print(f"copying file from '{local_file_path}' to '{target_file_path}'") + log.info("copying file from '%s' to '%s'", local_file_path, target_file_path) try: os.makedirs(os.path.dirname(target_file_path)) except Exception: pass shutil.copy2(local_file_path, target) - print("Done" + os.linesep) + log.info("Done") if os.path.lexists(url[len("file:///"):]): # because scheme of a absolute windows path is the drive letter in python 2, @@ -177,7 +180,7 @@ def download(url: str, target: str, read_block_size: int = 1048576) -> None: # use urlopen which raise an error if that file is not existing with urlopen(url) as response: total_size = response.info().get('Content-Length').strip() - print(f"Downloading file from '{url}' with size {total_size} bytes to {target}") + log.info("Download file from '%s' sized %s bytes to %s", url, total_size, target) # run the download received_size = urllib2_response_read(response, savefile_tmp, read_block_size, total_size) if received_size != int(total_size): @@ -265,12 +268,12 @@ def run_command(command: Union[List[str], str], cwd: str, extra_environment: Opt if cwd and not os.path.lexists(cwd): os.makedirs(cwd) - print('\n========================== do ... ==========================') + log.info("========================== do ... ==========================") if cwd: - print(f"Working Directory: {cwd}") + log.info("Working Directory: %s", cwd) else: - print("No cwd set!") - print("Last command: ", ' '.join(command_as_list)) + log.warning("No cwd set!") + log.info("Last command: %s", " ".join(command_as_list)) sys.stdout.flush() if cwd and not os.path.lexists(cwd): diff --git a/packaging-tools/bldinstallercommon.py b/packaging-tools/bldinstallercommon.py index cf41c24f3..28eb16953 100644 --- a/packaging-tools/bldinstallercommon.py +++ b/packaging-tools/bldinstallercommon.py @@ -49,6 +49,7 @@ from urllib.request import urlcleanup, urlopen, urlretrieve from bld_utils import download, is_linux, is_macos, is_windows, run_command from installer_utils import PackagingError +from logging_util import init_logger from runner import do_execute_sub_process from threadedwork import Task, ThreadedWork @@ -56,7 +57,8 @@ from threadedwork import Task, ThreadedWork if is_windows(): import win32api # type: ignore # pylint: disable=E0401 -DEBUG_RPATH = False +log = init_logger(__name__, debug_mode=False) + MAX_DEBUG_PRINT_LENGTH = 10000 @@ -90,7 +92,7 @@ def dl_progress(count: int, block_size: int, total_size: int) -> None: sys.stdout.flush() if count * block_size >= total_size: current_download_percent = 0 - print('\n') + sys.stdout.write("\n") ############################### @@ -105,7 +107,7 @@ def retrieve_url(url: str, savefile: str) -> None: shutil.move(savefile_tmp, savefile) except Exception as err: exc = sys.exc_info()[0] - print(exc) + log.error(exc) try: os.remove(savefile_tmp) except Exception: # swallow, do not shadow actual error @@ -237,7 +239,7 @@ def replace_in_files(filelist: List[str], regexp: str, replacement_string: str) old_contents = handle.read() new_contents = re.sub(regexp_obj, replacement_string, old_contents) if old_contents != new_contents: - print(f"Replacement '{replacement_string}' applied into: {xfile}") + log.info("Replacement '%s' applied into: %s", replacement_string, xfile) handle.seek(0) handle.write(new_contents) handle.truncate() @@ -264,9 +266,9 @@ def config_section_map(conf: ConfigParser, section: str) -> Dict[str, Any]: try: dict1[option] = conf.get(section, option) if dict1[option] == -1: - print(f"skip: {option}") - except Exception: - print(f"exception on {option}!") + log.info("skip: %s", option) + except Exception as error: + log.exception("exception on %s!", option, exc_info=error) dict1[option] = '' return dict1 @@ -326,16 +328,16 @@ def sanity_check_rpath_max_length(file_path: str, new_rpath: str) -> bool: if proc.stdout is not None: result = re.search(r":*.R.*PATH=.*", proc.stdout.read().decode()) if result is None: - print(f'*** No RPath found from given file: {file_path}') + log.info("No RPath found from given file: %s", file_path) else: rpath = result.group() index = rpath.index('=') rpath = rpath[index + 1:] space_for_new_rpath = len(rpath) if len(new_rpath) > space_for_new_rpath: - print(f'*** Warning - Not able to process RPath for file: {file_path}') - print(f'*** Required length for new RPath [{new_rpath}] is: {str(len(new_rpath))}') - print(f'*** Space available for new RPath inside the binary is: {str(space_for_new_rpath)}') + log.warning("Warning - Not able to process RPath for file: %s", file_path) + log.warning("New RPath [%s] length: %s", new_rpath, str(len(new_rpath))) + log.warning("Space available inside the binary: %s", str(space_for_new_rpath)) raise IOError() return True @@ -390,11 +392,10 @@ def calculate_rpath(file_full_path: str, destination_lib_path: str) -> str: rpath = calculate_relpath(bin_path, path_to_lib) full_rpath = '$ORIGIN' + os.sep + rpath - if DEBUG_RPATH: - print(' ----------------------------------------') - print(f' RPath target folder: {path_to_lib}') - print(f' Bin file: {file_full_path}') - print(f' Calculated RPath: {full_rpath}') + log.debug("----------------------------------------") + log.debug(" RPath target folder: %s", path_to_lib) + log.debug(" Bin file: %s", file_full_path) + log.debug(" Calculated RPath: %s", full_rpath) return full_rpath @@ -403,11 +404,11 @@ def calculate_rpath(file_full_path: str, destination_lib_path: str) -> str: # Handle the RPath in the given component files ############################################################## def handle_component_rpath(component_root_path: str, destination_lib_paths: str) -> None: - print(' @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@') - print(' Handle RPath') - print('') - print(f' Component root path: {component_root_path}') - print(f' Destination lib path: {destination_lib_paths}') + log.info("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@") + log.info("Handle RPath") + log.info("") + log.info("Component root path: %s", component_root_path) + log.info("Destination lib path: %s", destination_lib_paths) # loop on all files for root, _, files in os.walk(component_root_path): @@ -434,7 +435,7 @@ def handle_component_rpath(component_root_path: str, destination_lib_paths: str) rpath = ':'.join(rpaths) if sanity_check_rpath_max_length(file_full_path, rpath): - # print ' RPath value: [' + rpath + '] for file: [' + file_full_path + ']' + log.debug("RPath value: [%s] for file: [%s]", rpath, file_full_path) cmd_args = ['chrpath', '-r', rpath, file_full_path] # force silent operation work_dir = os.path.dirname(os.path.realpath(__file__)) @@ -451,11 +452,11 @@ def clone_repository( full_clone: bool = False, init_subrepos: bool = False, ) -> None: - print('--------------------------------------------------------------------') - print(f'Cloning repository: {repo_url}') - print(f' branch/tag: {repo_branch_or_tag}') - print(f'Dest: {destination_folder}') - print('--------------------------------------------------------------------') + log.info("--------------------------------------------------------------------") + log.info("Cloning repository: %s", repo_url) + log.info("Repo branch/tag: %s", repo_branch_or_tag) + log.info("Destination: %s", destination_folder) + log.info("--------------------------------------------------------------------") work_dir = os.path.dirname(os.path.realpath(__file__)) if full_clone: @@ -486,7 +487,7 @@ def get_tag_from_branch(directory: str) -> str: cmd_args = ['git', 'symbolic-ref', 'HEAD'] return_code, tag = do_execute_sub_process(cmd_args, directory, False, True) if return_code != -1: - print("Already checked out a tag. THIS IS TOTALLY OK, PLEASE IGNORE THE ABOVE ERROR.") + log.info("Already checked out a tag. THIS IS OKAY, PLEASE IGNORE THE ABOVE ERROR.") tag = "" else: # Check what sha1 we have checked out @@ -499,7 +500,7 @@ def get_tag_from_branch(directory: str) -> str: return_code, tag = do_execute_sub_process(cmd_args, directory, False, True) tag = tag.strip('\n') if return_code != -1: - print('No tag found for branch. THIS IS TOTALLY OK, PLEASE IGNORE THE ABOVE ERROR.') + log.info("No tag found for branch. THIS IS OKAY, PLEASE IGNORE THE ABOVE ERROR.") tag = "" return tag @@ -525,7 +526,7 @@ def git_archive_repo(repo_and_ref: str) -> str: # git archive repo with given name with open(archive_name, "w", encoding="utf-8") as archive_file: check_call(f"git --no-pager archive {ref}", stdout=archive_file, stderr=STDOUT, shell=True, cwd=checkout_dir) - print(f"Created archive: {archive_name}") + log.info("Created archive: %s", archive_name) shutil.rmtree(checkout_dir, ignore_errors=True) return archive_name @@ -546,7 +547,7 @@ def extract_file(path: str, to_directory: str = ".") -> bool: elif path.endswith('.7z') or path.endswith('.zip'): cmd_args = ['7z', 'x', path] else: - print(f'Did not extract the file! Not archived or no appropriate extractor was found: {path}') + log.warning("Extract fail: %s. Not an archive or appropriate extractor was not found", path) return False ret = run_command(cmd_args, cwd=to_directory, only_error_case_output=True) @@ -640,7 +641,7 @@ def create_qt_download_task(module_urls: List[str], target_qt5_path: str, temp_p download_work.add_task_object(download_task) unzip_task.add_function(extract_task.do_task) else: - print(f"warning: could not find '{module_url}' for download") + log.warning("could not find '%s' for download", module_url) # add icu, d3dcompiler, opengl32, openssl target_path = os.path.join(target_qt5_path, 'bin' if is_windows() else 'lib') if caller_arguments: @@ -675,7 +676,7 @@ def create_qt_download_task(module_urls: List[str], target_qt5_path: str, temp_p def patch_qt(qt5_path: str) -> None: - print("##### patch Qt #####") + log.info("##### patch Qt #####") qmake_binary = os.path.join(qt5_path, 'bin', 'qmake') # write qt.conf with open(os.path.join(qt5_path, 'bin', 'qt.conf'), "w", encoding="utf-8") as qt_conf_file: @@ -684,5 +685,5 @@ def patch_qt(qt5_path: str) -> None: # fix rpaths if is_linux(): handle_component_rpath(qt5_path, 'lib') - print("##### patch Qt ##### ... done") + log.info("##### patch Qt ##### ... done") run_command(qmake_binary + " -query", qt5_path) diff --git a/packaging-tools/build_wrapper.py b/packaging-tools/build_wrapper.py index f1a914a98..3c7e45e30 100644 --- a/packaging-tools/build_wrapper.py +++ b/packaging-tools/build_wrapper.py @@ -67,12 +67,14 @@ from bldinstallercommon import ( git_archive_repo, safe_config_key_fetch, ) +from logging_util import init_logger from optionparser import get_pkg_options from read_remote_config import get_pkg_value from runner import do_execute_sub_process from threadedwork import Task, ThreadedWork -# ---------------------------------------------------------------------- +log = init_logger(__name__, debug_mode=False) + SCRIPT_ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) WORK_DIR = os.getenv('PKG_NODE_ROOT', os.path.abspath(os.path.join(__file__, '../../../'))) LOCAL_MODE = os.getenv('LOCAL_MODE') # if set, installers will be copied to a local directory @@ -80,7 +82,7 @@ LOCAL_INSTALLER_DIR = os.getenv('LOCAL_INSTALLER_DIR', os.path.join(WORK_DIR, 'i if LOCAL_MODE: assert os.path.exists(LOCAL_INSTALLER_DIR), f"Local installer dest dir does not exist: {LOCAL_INSTALLER_DIR}" - print(f"Installer files will be copied to local directory: {LOCAL_INSTALLER_DIR}") + log.info("Installer files will be copied to local directory: %s", LOCAL_INSTALLER_DIR) ############################### @@ -158,7 +160,7 @@ def handle_qt_licheck_build(option_dict: Dict[str, str]) -> None: do_execute_sub_process(cmd_args, exe_dir, True) else: # opensource, do nothing - print('*** opensource build, nothing to build ...') + log.info("Opensource build, nothing to build...") ############################### @@ -166,7 +168,7 @@ def handle_qt_licheck_build(option_dict: Dict[str, str]) -> None: ############################### def move_files_to_parent_dir(source: str) -> None: destination = os.path.abspath(os.path.join(source, os.pardir)) - print(f"Moving files from: [{source}] to: [{destination}]") + log.info("Moving files from: [%s] to: [%s]", source, destination) assert os.path.isdir(source), f"The given source is not a directory: {source}" assert os.path.isdir(destination), f"The destination is not a directory: {destination}" files_list = os.listdir(source) @@ -181,7 +183,7 @@ def create_download_documentation_task( doc_base_url = base_url + "/doc" use_local = urlparse(doc_base_url).scheme == "file" - print(f"doc_base_url: {doc_base_url} use_local: {use_local}") + log.info("doc_base_url: %s use_local: %s", doc_base_url, use_local) if use_local: file_list = os.listdir(doc_base_url[len("file:///"):]) else: @@ -325,7 +327,7 @@ class BuildLog: try: self.file = open(self.log_filepath, 'w' if self.log_overwrite else 'a', encoding="utf-8") except Exception: - print(f"Failed to write log file '{self.log_filepath}'") + log.error("Failed to write log file '%s'", self.log_filepath) sys.stdout.flush() raise return self.file @@ -334,8 +336,8 @@ class BuildLog: if self.file is not None: self.file.close() if exc_type: # exception raised -> print the log and re-raise - with open(self.log_filepath, 'r', encoding="utf-8") as handle: - print(handle.read()) + with open(self.log_filepath, "r", encoding="utf-8") as handle: + log.info(handle.read()) return True # re-raise return None @@ -1258,7 +1260,7 @@ def main() -> None: elif args.command == archive_repository: do_git_archive_repo(option_dict, args.archive_repo) else: - print('Unsupported command') + log.info("Unsupported command: %s", args.command) if __name__ == '__main__': diff --git a/packaging-tools/create_installer.py b/packaging-tools/create_installer.py index 8e01c3d34..ca1b070dc 100644 --- a/packaging-tools/create_installer.py +++ b/packaging-tools/create_installer.py @@ -36,7 +36,6 @@ import shutil import sys from argparse import ArgumentParser, ArgumentTypeError, Namespace from configparser import ConfigParser, ExtendedInterpolation -from logging import getLogger from multiprocessing import cpu_count from pathlib import Path from subprocess import check_call @@ -61,6 +60,7 @@ from bldinstallercommon import ( safe_config_key_fetch, ) from installer_utils import PackagingError +from logging_util import init_logger from patch_qt import patch_files, patch_qt_edition from pkg_constants import INSTALLER_OUTPUT_DIR_NAME from runner import do_execute_sub_process @@ -70,8 +70,7 @@ from threadedwork import ThreadedWork if is_windows(): import win32api # type: ignore # pylint: disable=E0401 -log = getLogger("create_installer") -log.setLevel("INFO") +log = init_logger(__name__, debug_mode=False) # ---------------------------------------------------------------------- TARGET_INSTALL_DIR_NAME_TAG = '%TARGET_INSTALL_DIR%' @@ -835,9 +834,9 @@ def create_maintenance_tool_resource_file(task: Any) -> None: def inject_update_rcc_to_archive(archive_file_path: str, file_to_be_injected: str) -> None: log.info("Injecting file [%s] into [%s]", file_to_be_injected, archive_file_path) if not os.path.isfile(file_to_be_injected): - log.error("*** Unable to locate file: %s", file_to_be_injected) + log.error("Unable to locate file: %s", file_to_be_injected) if not os.path.isfile(archive_file_path): - log.error("*** Unable to locate file: %s", archive_file_path) + log.error("Unable to locate file: %s", archive_file_path) archive_file_name = os.path.basename(archive_file_path) # copy to tmp location tmp_dir = os.path.join(os.path.dirname(archive_file_path), '_tmp') diff --git a/packaging-tools/dump_debug_infos.py b/packaging-tools/dump_debug_infos.py index d1df1b799..4d2d0381a 100644 --- a/packaging-tools/dump_debug_infos.py +++ b/packaging-tools/dump_debug_infos.py @@ -36,6 +36,10 @@ import sys from shutil import rmtree from typing import List +from logging_util import init_logger + +log = init_logger(__name__, debug_mode=False) + def is_file_with_debug_information_windows(path: str) -> bool: if not path.endswith('.pdb'): @@ -70,7 +74,7 @@ def read_output(args: List[str]) -> bytes: def dump_sym(dump_syms_path: str, architecture: str, absolute_path: str, sym_path: str, verbose: bool) -> bool: dump_syms_command = f'{dump_syms_path} {architecture} "{absolute_path}" > "{sym_path}"' if verbose: - print(f"call: {dump_syms_command}") + log.info("call: %s", dump_syms_command) dump_syms_return = subprocess.call(dump_syms_command, shell=True) if os.path.exists(sym_path) and os.stat(sym_path).st_size > 0 and dump_syms_return == 0: return True @@ -181,11 +185,11 @@ def _main() -> None: source_bundle_command = [args.sentry_cli_path, "difutil", "bundle-sources"] source_bundle_command.extend(sym_filenames) if args.verbose: - print(source_bundle_command) + log.info(source_bundle_command) testoutput = subprocess.check_output( source_bundle_command, cwd=args.output_path ).decode('utf-8') - print(testoutput) + log.info(testoutput) if __name__ == '__main__': diff --git a/packaging-tools/install_qt.py b/packaging-tools/install_qt.py index 64fce45de..0c1d6f0ab 100644 --- a/packaging-tools/install_qt.py +++ b/packaging-tools/install_qt.py @@ -33,8 +33,11 @@ import argparse import os from bldinstallercommon import create_qt_download_task, patch_qt +from logging_util import init_logger from threadedwork import ThreadedWork +log = init_logger(__name__, debug_mode=False) + def get_arguments() -> argparse.Namespace: parser = argparse.ArgumentParser(description='Build Qt Creator') diff --git a/packaging-tools/logging_util.py b/packaging-tools/logging_util.py index b999fa1df..e0c711169 100755 --- a/packaging-tools/logging_util.py +++ b/packaging-tools/logging_util.py @@ -30,26 +30,20 @@ ############################################################################# import logging +import os import colorlog def init_logger(dunder_name: str, debug_mode: bool) -> logging.Logger: - log_format = ( - '%(asctime)s ' - '%(levelname)s: ' - '%(message)s' - ) - bold_seq = '\033[1m' - colorlog_format = ( - f'{bold_seq} ' - '%(log_color)s ' - f'{log_format}' - ) + log_format = "%(asctime)s %(levelname)s: %(message)s" + log_format_file = "%(asctime)s %(levelname)s:%(filename)s:%(lineno)d(%(process)d): %(message)s" + bold_seq = "\033[1m" + colorlog_format = f"{bold_seq} " "%(log_color)s " f"{log_format}" colorlog.basicConfig(format=colorlog_format) logger = logging.getLogger(dunder_name) - if debug_mode: + if debug_mode or os.environ.get("QT_PACKAGING_DEBUG"): logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) @@ -57,21 +51,21 @@ def init_logger(dunder_name: str, debug_mode: bool) -> logging.Logger: # Output full log file_handler = logging.FileHandler('packaging.log') file_handler.setLevel(logging.DEBUG) - formatter = logging.Formatter(log_format) + formatter = logging.Formatter(log_format_file) file_handler.setFormatter(formatter) logger.addHandler(file_handler) # Output warning log file_handler = logging.FileHandler('packaging.warning.log') file_handler.setLevel(logging.WARNING) - formatter = logging.Formatter(log_format) + formatter = logging.Formatter(log_format_file) file_handler.setFormatter(formatter) logger.addHandler(file_handler) # Output error log file_handler = logging.FileHandler('packaging.error.log') file_handler.setLevel(logging.ERROR) - formatter = logging.Formatter(log_format) + formatter = logging.Formatter(log_format_file) file_handler.setFormatter(formatter) logger.addHandler(file_handler) diff --git a/packaging-tools/notarize.py b/packaging-tools/notarize.py index b09710393..8612bfe01 100755 --- a/packaging-tools/notarize.py +++ b/packaging-tools/notarize.py @@ -31,28 +31,16 @@ import argparse import asyncio -import logging import sys from shutil import which from subprocess import STDOUT, CalledProcessError, TimeoutExpired from time import gmtime, sleep, strftime from typing import List +from logging_util import init_logger from read_remote_config import get_pkg_value -LOG_FMT_CI = "%(asctime)s %(levelname)s:%(filename)s:%(lineno)d(%(process)d): %(message)s" -log = logging.getLogger("Notarizer") -log.setLevel(logging.INFO) -# Unify format of all messages -try: - from rainbow_logging_handler import RainbowLoggingHandler # type: ignore - handler = RainbowLoggingHandler(sys.stderr, color_asctime=(None, None, False)) -except ImportError: - handler = logging.StreamHandler() - -formatter = logging.Formatter(LOG_FMT_CI) -handler.setFormatter(formatter) -log.addHandler(handler) +log = init_logger(__name__, debug_mode=False) class NotarizationError(Exception): diff --git a/packaging-tools/optionparser.py b/packaging-tools/optionparser.py index 936113369..0ba90a2d1 100644 --- a/packaging-tools/optionparser.py +++ b/packaging-tools/optionparser.py @@ -34,6 +34,10 @@ import sys from configparser import ConfigParser from typing import Dict +from logging_util import init_logger + +log = init_logger(__name__, debug_mode=False) + class PackagingOptions: """Utility class to read options from configuration file that follows .ini file format.""" @@ -51,9 +55,9 @@ class PackagingOptions: try: dict1[option] = self.config.get(section, option) if dict1[option] == "-1": - print(f"skip: {option}") + log.info("skip: %s", option) except Exception: - print(f"exception on {option}!") + log.warning("exception on %s!", option) dict1[option] = "" return dict1 @@ -71,11 +75,10 @@ class PackagingOptions: def verbose(self) -> None: for section in self.config.sections(): - print(f"[{section}]") + log.info("[%s]", section) options = self.config.options(section) for option in options: - print(f"{option} = {self.config.get(section, option)}") - print() + log.info("%s = %s", option, self.config.get(section, option)) def get_pkg_options(conf_file_path: str) -> PackagingOptions: @@ -91,7 +94,7 @@ def main() -> None: # Print out all options options = get_pkg_options(args.conf_file) config_map = options.config_map() - print(config_map) + log.info(config_map) if __name__ == '__main__': diff --git a/packaging-tools/patch_qt.py b/packaging-tools/patch_qt.py index 798e92c64..864c1d441 100755 --- a/packaging-tools/patch_qt.py +++ b/packaging-tools/patch_qt.py @@ -34,9 +34,13 @@ import re from fileinput import FileInput from typing import Any, Callable, Generator, List, Match +from logging_util import init_logger + +log = init_logger(__name__, debug_mode=False) + def _file_iterator(artifacts_dir: str) -> Generator[Any, Any, Any]: - print(f"Patching build time paths from: {artifacts_dir}") + log.info("Patching build time paths from: %s", artifacts_dir) for root, _, files in os.walk(artifacts_dir): for file_name in files: yield os.path.join(os.path.join(root, file_name)) @@ -50,7 +54,7 @@ def _get_patchers(product: str) -> List[Callable[[Any], Any]]: def patch_files(artifacts_dir: str, product: str) -> None: - print(f"Patching files from: {artifacts_dir}") + log.info("Patching files from: %s", artifacts_dir) patchers = _get_patchers(product) for file_path in _file_iterator(artifacts_dir): for patcher in patchers: @@ -71,17 +75,17 @@ def _patch_qt_edition(file_path: str, licheck_file_name: str, release_date: str) edition_line = 'QT_EDITION = Enterprise' licheck_line = 'QT_LICHECK = ' + licheck_file_name release_line = 'QT_RELEASE_DATE = ' + release_date - print(edition_line.rstrip('\n')) - print(licheck_line.rstrip('\n')) - print(release_line.rstrip('\n')) + log.info(edition_line.rstrip("\n")) + log.info(licheck_line.rstrip("\n")) + log.info(release_line.rstrip("\n")) else: - print(line.rstrip('\n')) + log.info(line.rstrip("\n")) def patch_qconfig_pri(file_path: str) -> None: for line in FileInput(file_path, inplace=True): patched_line = patch_qconfig_pri_from_line(line) - print(patched_line.rstrip('\n')) + log.info(patched_line.rstrip("\n")) def patch_qconfig_pri_from_line(line: str) -> str: @@ -96,7 +100,7 @@ def erase_qmake_prl_build_dir(file_path: str) -> None: # Erase lines starting with 'QMAKE_PRL_BUILD_DIR' from .prl files for line in FileInput(file_path, inplace=True): patched_line = patch_qmake_prl_build_dir_from_line(line) - print(patched_line.rstrip('\n')) + log.info(patched_line.rstrip("\n")) def patch_qmake_prl_build_dir_from_line(line: str) -> str: @@ -106,7 +110,7 @@ def patch_qmake_prl_build_dir_from_line(line: str) -> str: def patch_absolute_lib_paths_from_file(file_path: str) -> None: for line in FileInput(file_path, inplace=True): patched_line = patch_absolute_lib_paths_from_line(line, file_path.split(".")[-1]) - print(patched_line.rstrip('\n')) + log.info(patched_line.rstrip("\n")) def patch_absolute_lib_paths_from_line(line: str, file_extension: str) -> str: diff --git a/packaging-tools/python_env.py b/packaging-tools/python_env.py index fca3ee055..ca2bc61bc 100644 --- a/packaging-tools/python_env.py +++ b/packaging-tools/python_env.py @@ -115,7 +115,7 @@ async def create_venv(python_src: str, get_pip_file: str) -> Tuple[str, str, Dic log.info("Installing pipenv requirements into: %s", prefix) await async_exec_cmd(cmd=cmd, timeout=60 * 30, env=env) # give it 30 mins venv_folder = locate_venv(pipenv, env) - print(f"The pipenv virtualenv is created located in: {venv_folder}") + log.info("The pipenv virtualenv is created located in: %s", venv_folder) return (venv_folder, pipenv, env) diff --git a/packaging-tools/read_remote_config.py b/packaging-tools/read_remote_config.py index 565281e39..d4c854280 100755 --- a/packaging-tools/read_remote_config.py +++ b/packaging-tools/read_remote_config.py @@ -37,6 +37,10 @@ from io import StringIO from typing import Any, Optional from urllib.request import urlopen +from logging_util import init_logger + +log = init_logger(__name__, debug_mode=False) + class RemotePkgConfigError(Exception): pass @@ -79,7 +83,7 @@ def main() -> None: parser.print_help(sys.stderr) raise SystemExit("--url or --section missing") - print(f"{args.key}: '{get_pkg_value(args.key, args.section, args.url)}'") + log.info("%s: '%s'", args.key, get_pkg_value(args.key, args.section, args.url)) if __name__ == "__main__": diff --git a/packaging-tools/remote_uploader.py b/packaging-tools/remote_uploader.py index 0126559e0..ff013b1d6 100755 --- a/packaging-tools/remote_uploader.py +++ b/packaging-tools/remote_uploader.py @@ -36,6 +36,10 @@ import sys from shutil import which from subprocess import CalledProcessError, check_call +from logging_util import init_logger + +log = init_logger(__name__, debug_mode=False) + class RemoteUploaderError(Exception): pass @@ -80,9 +84,9 @@ class RemoteUploader: def ensure_remote_dir(self, remote_dir: str) -> None: assert self.init_finished, "RemoteUploader not initialized!" - print(f"Creating remote directory: {remote_dir}") + log.info("Creating remote directory: %s", remote_dir) cmd = self.ssh_cmd + ['mkdir', '-p', remote_dir] - print("Executing: ", ' '.join(cmd)) + log.info("Executing: %s", " ".join(cmd)) if not self.dry_run: check_call(cmd, timeout=60) # give it 60s @@ -94,9 +98,9 @@ class RemoteUploader: remote_destination = remote_destination + '/' + dest_dir_name + '/' if "windows" in platform.system().lower(): self.ensure_remote_dir(self.remote_target_dir + '/' + dest_dir_name + '/') - print(f"Copying [{file_name}] to [{remote_destination}]") + log.info("Copying [%s] to [%s]", file_name, remote_destination) cmd = self.copy_cmd + [file_name, remote_destination] - print("Executing: ", ' '.join(cmd)) + log.info("Executing: %s", " ".join(cmd)) if not self.dry_run: check_call(cmd, timeout=60 * 10) # give it 10 mins @@ -107,15 +111,15 @@ class RemoteUploader: def update_latest_symlink(self, force_update: bool = True) -> None: assert self.init_finished, "RemoteUploader not initialized!" - print(f"Creating remote symlink: [{self.remote_latest_link}] -> [{self.remote_target_dir}]") + log.info("Create remote symlink: %s -> %s", self.remote_latest_link, self.remote_target_dir) options = ["-sfn"] if force_update else ["-sn"] try: cmd = self.ssh_cmd + ['ln'] + options + [self.remote_target_dir, self.remote_latest_link] - print("Executing: ", ' '.join(cmd)) + log.info("Executing: %s", " ".join(cmd)) if not self.dry_run: check_call(cmd, timeout=60) # give it 60s - except CalledProcessError: - print("Failed to execute: ", ' '.join(cmd)) + except CalledProcessError as error: + log.exception("Failed to execute: %s", " ".join(cmd), exc_info=error) raise diff --git a/packaging-tools/runner.py b/packaging-tools/runner.py index eda52715e..490538412 100755 --- a/packaging-tools/runner.py +++ b/packaging-tools/runner.py @@ -53,9 +53,10 @@ def exec_cmd( cmd: List[str], timeout: float = 60.0, env: Optional[Dict[str, str]] = None ) -> str: env = env if env else os.environ.copy() - log.info("Calling: %s", ' '.join(cmd)) - output = check_output(' '.join(cmd), shell=True, env=env, timeout=timeout).decode("utf-8").strip() - print(output) + args = " ".join(cmd) + log.info("Calling: %s", args) + output = check_output(args, shell=True, env=env, timeout=timeout).decode("utf-8").strip() + log.info(output) return output @@ -88,11 +89,11 @@ def do_execute_sub_process( args_log: Optional[str] = None, ) -> Tuple[int, str]: extra_env = extra_env or os.environ.copy() - _args_log = args_log or ' '.join([str(i) for i in args]) - print(' --------------------------------------------------------------------') - print(f' Executing: [{_args_log}]') - print(f' Execution path: [{execution_path}]') - print(f' Abort on fail: [{str(abort_on_fail)}]') + _args_log = args_log or " ".join([str(i) for i in args]) + log.info("--------------------------------------------------------------------") + log.info("Executing: [%s]", _args_log) + log.info("Execution path: [%s]", execution_path) + log.info("Abort on fail: [%s]", str(abort_on_fail)) sys.stdout.flush() theproc: Any return_code = -1 @@ -127,12 +128,12 @@ def do_execute_sub_process( return_code = theproc.returncode if output: output = output[len(output) - MAX_DEBUG_PRINT_LENGTH:] if len(output) > MAX_DEBUG_PRINT_LENGTH else output - print(output) + log.info(output) else: - print('Note, no output from the sub process!') + log.info("Note, no output from the sub process!") sys.stdout.flush() raise Exception(f"*** Execution failed with code: {theproc.returncode}") - print(' --------------------------------------------------------------------') + log.info("--------------------------------------------------------------------") sys.stdout.flush() except Exception: sys.stderr.write(' ERROR - ERROR - ERROR - ERROR - ERROR - ERROR !!!' + os.linesep) diff --git a/packaging-tools/sdkcomponent.py b/packaging-tools/sdkcomponent.py index 938817b28..231921331 100644 --- a/packaging-tools/sdkcomponent.py +++ b/packaging-tools/sdkcomponent.py @@ -36,6 +36,9 @@ from typing import Any, List from archiveresolver import ArchiveLocationResolver from bldinstallercommon import config_section_map, is_content_url_valid, safe_config_key_fetch +from logging_util import init_logger + +log = init_logger(__name__, debug_mode=False) ONLINE_ARCHIVE_LIST_TAG = '<!--ONLINE_ARCHIVE_LIST-->' @@ -173,9 +176,9 @@ class SdkComponent: else: # sanity check, duplicate template should not exist to avoid # problems! - print(f'*** Found duplicate template for: {self.package_name}') - print(f'*** Ignoring: {template_full_path}') - print(f'*** Using: {self.pkg_template_dir}') + log.warning("Found duplicate template for: %s", self.package_name) + log.warning("Ignoring: %s", template_full_path) + log.warning("Using: %s", self.pkg_template_dir) self.parse_archives(self.target_config, self.archive_location_resolver) self.check_component_data() @@ -228,7 +231,7 @@ class SdkComponent: archives_list = self.archives.split(',') for archive in archives_list: if not archive: - print(f"Warning: There appears to be ',' issues in the config file archive list for component: {self.package_name}") + log.warning("[%s]: Archive list in config has ',' issues", self.package_name) continue # check that archive template exists if not target_config.has_section(archive): @@ -252,24 +255,24 @@ class SdkComponent: return temp_list def print_component_data(self) -> None: - print('=============================================================') - print(f' [{self.package_name}]') + log.info("=============================================================") + log.info("[%s]", self.package_name) if self.static_component: - print(f' Static component: {self.static_component}') + log.info("Static component: %s", self.static_component) return if self.root_component: - print(f' Root component: {self.root_component}') - print(f' Include filter: {self.include_filter}') - print(f' Target install base: {self.target_install_base}') - print(f' Version: {self.version}') - print(f' Version tag: {self.version_tag}') - print(f' Package default: {self.package_default}') + log.info("Root component: %s", self.root_component) + log.info("Include filter: %s", self.include_filter) + log.info("Target install base: %s", self.target_install_base) + log.info("Version: %s", self.version) + log.info("Version tag: %s", self.version_tag) + log.info("Package default: %s", self.package_default) if self.downloadable_archive_list: - print(' Archives:') + log.info(" Archives:") for archive in self.downloadable_archive_list: - print(' ---------------------------------------------------------------') - print(f' Downloadable archive name: {archive.archive_name}') - print(f' Archive strip dirs: {archive.package_strip_dirs}') - print(f' Archive target install dir: {archive.get_archive_installation_directory()}') - print(f' Archive RPath target: {archive.rpath_target}') - print(f' Archive URI: {archive.archive_uri}') + log.info("---------------------------------------------------------------") + log.info("Downloadable archive name: %s", archive.archive_name) + log.info("Strip dirs: %s", archive.package_strip_dirs) + log.info("Target install dir: %s", archive.get_archive_installation_directory()) + log.info("RPath target: %s", archive.rpath_target) + log.info("URI: %s", archive.archive_uri) diff --git a/packaging-tools/send_header_diff_kmail.py b/packaging-tools/send_header_diff_kmail.py index f31b55b30..bd4d2d8cb 100755 --- a/packaging-tools/send_header_diff_kmail.py +++ b/packaging-tools/send_header_diff_kmail.py @@ -56,21 +56,25 @@ import os import sys from subprocess import check_call +from logging_util import init_logger + +log = init_logger(__name__, debug_mode=False) + def send_headers(version: str, message_id: str, simulate: bool) -> None: receiver = 'development@qt-project.org' subject = 'Qt ' + version + ' header diff: ' - print(f"Header diff: {subject}") + log.info("Header diff: %s", subject) for diff in os.listdir('.'): if diff.endswith('.diff'): - print(f'Sending: {diff}') + log.info("Sending: %s", diff) subj = subject + diff head1 = 'In-Reply-To:<' + message_id + '>' head2 = 'References:<' + message_id + '>' args = ['kmail', '--subject', subj, '--attach', diff, '--header', head1, '--header', head2, receiver] if simulate: - print("Simulate:", " ".join(args)) + log.info("Simulate: %s", " ".join(args)) else: check_call(args) diff --git a/packaging-tools/sign_installer.py b/packaging-tools/sign_installer.py index eac26a198..138104b99 100755 --- a/packaging-tools/sign_installer.py +++ b/packaging-tools/sign_installer.py @@ -30,16 +30,15 @@ ############################################################################# import argparse -import logging import os import sys from shutil import rmtree from subprocess import DEVNULL, check_call +from logging_util import init_logger from read_remote_config import get_pkg_value -log = logging.getLogger("Sign-utility") -logging.basicConfig(level=logging.INFO, format='%(message)s') +log = init_logger(__name__, debug_mode=False) def sign_mac_app(app_path: str, signing_identity: str) -> None: diff --git a/packaging-tools/tests/test_build_wrapper.py b/packaging-tools/tests/test_build_wrapper.py index 036bd1d0e..1834f9868 100644 --- a/packaging-tools/tests/test_build_wrapper.py +++ b/packaging-tools/tests/test_build_wrapper.py @@ -68,7 +68,6 @@ class TestBuildWrapper(unittest.TestCase): remote_path_base = os.path.join(temp_dir, project_name, version_branch) remote_path_snapshot_dir = os.path.join(remote_path_base, build_number) remote_path_latest_link = os.path.join(remote_path_base, 'latest') - print(remote_path_latest_link) self.assertTrue(os.path.isdir(remote_path_base)) self.assertTrue(os.path.isdir(remote_path_snapshot_dir)) self.assertTrue(os.path.islink(remote_path_latest_link)) diff --git a/packaging-tools/tests/test_packaging.py b/packaging-tools/tests/test_packaging.py index c31f33a59..b466d9f8e 100755 --- a/packaging-tools/tests/test_packaging.py +++ b/packaging-tools/tests/test_packaging.py @@ -141,8 +141,8 @@ class TestPackaging(unittest.TestCase): idx = 0 for line in FileInput(temp_file, inplace=False): - print(f"Received data: [{line.strip()}] expected data: [{expected_data[idx]}]") - self.assertEqual(line.strip(), expected_data[idx], f"Received data: [{line}] differs from expected data: [{expected_data[idx]}]") + msg = f"Received data: [{line}] differs from expected data: [{expected_data[idx]}]" + self.assertEqual(line.strip(), expected_data[idx], msg) idx += 1 finally: rmtree(temp_dir) diff --git a/packaging-tools/tests/test_release_repo_updater.py b/packaging-tools/tests/test_release_repo_updater.py index bd2708dc5..9c5b2cbcf 100755 --- a/packaging-tools/tests/test_release_repo_updater.py +++ b/packaging-tools/tests/test_release_repo_updater.py @@ -272,7 +272,6 @@ class TestReleaseRepoUpdater(unittest.TestCase): ([""], [""]) ) async def test_format_task_filters(self, task_filters: List[str], exp_result: bool) -> None: - print("test") self.assertEqual(format_task_filters(task_filters), exp_result) @asyncio_test_parallel_data( # type: ignore |