diff options
author | Cristian Adam <cristian.adam@qt.io> | 2023-02-09 15:33:30 +0100 |
---|---|---|
committer | Cristian Adam <cristian.adam@qt.io> | 2023-02-22 11:41:21 +0000 |
commit | 74bb7823c7a5c7bb636d4419df4bd52eb79f2924 (patch) | |
tree | 006761c45b56b224afc710a5bdfa65754444d077 | |
parent | de23418413dd565d16e03942514ef590a191eb89 (diff) |
Qtcreator_libclang: Support for LLVM 16.0.0-rc2
Set HAVE_CLANG_REPL_SUPPORT to OFF, which fixes a linking error
with Visual C++ due to the high number of symbols (>65k)
Qt 6.4 has a different naming scheme.
Use ClangCodeModel plugin tests for the PGO training, which use
clangd.
Change-Id: I9c823d8bcdeaa8d7ce638624f6de285fc3fbf9c5
Reviewed-by: Eike Ziller <eike.ziller@qt.io>
-rw-r--r-- | packaging-tools/build_clang.py | 32 | ||||
-rw-r--r-- | packaging-tools/libclang_training/libclangtimings2csv.py | 144 | ||||
-rw-r--r-- | packaging-tools/libclang_training/merge_csv_files.py | 152 | ||||
-rw-r--r-- | packaging-tools/libclang_training/qtc.fileTextEditorCpp.batch | 39 | ||||
-rw-r--r-- | packaging-tools/libclang_training/qtc.openProject.batch | 4 | ||||
-rw-r--r-- | packaging-tools/libclang_training/runBatchFiles.bat | 9 | ||||
-rw-r--r-- | packaging-tools/libclang_training/run_batch_files.py | 218 |
7 files changed, 39 insertions, 559 deletions
diff --git a/packaging-tools/build_clang.py b/packaging-tools/build_clang.py index e0d3abd00..8c30e048c 100644 --- a/packaging-tools/build_clang.py +++ b/packaging-tools/build_clang.py @@ -187,11 +187,11 @@ def mingw_training( qt_modules = ['qtbase', 'qtdeclarative', 'qtimageformats', 'qt5compat', 'qtshadertools', 'qtsvg', 'qttools'] qt_base_url = 'http://' + pkg_server + '/packages/jenkins/archive/qt/' \ - + training_qt_version() + '/' + training_qt_long_version() + '-final-released/Qt' + training_qt_long_version() + + training_qt_version() + '/' + training_qt_long_version() + '-released/Qt' msvc_year_ver = msvc_year_version() if bitness == 64: - qt_mingw_postfix = '-Windows-Windows_10-Mingw-Windows-Windows_10-X86_64.7z' - qt_postfix = '-Windows-Windows_10-' + msvc_year_ver + '-Windows-Windows_10-X86_64.7z' + qt_mingw_postfix = '-Windows-Windows_10_21H2-Mingw-Windows-Windows_10_21H2-X86_64.7z' + qt_postfix = '-Windows-Windows_10_21H2-' + msvc_year_ver + '-Windows-Windows_10_21H2-X86_64.7z' qt_module_urls = [qt_base_url + '/' + module + '/' + module + qt_postfix for module in qt_modules] qt_mingw_module_urls = [qt_base_url + '/' + module + '/' + module + qt_mingw_postfix for module in qt_modules] @@ -243,8 +243,7 @@ def mingw_training( '-DBUILD_PLUGIN_RESOURCEEDITOR=ON', '-DBUILD_EXECUTABLE_QTCREATOR=ON', - '-DBUILD_EXECUTABLE_ECHO=ON', - '-DBUILD_EXECUTABLE_CLANGBACKEND=ON', + '-DBUILD_EXECUTABLE_PROCESSTESTAPP=ON', '-DBUILD_EXECUTABLE_QTCREATOR_PROCESSLAUNCHER=ON', '-DCMAKE_PREFIX_PATH=' + qt_mingw_dir + ';' + os.path.join(base_path, 'libclang'), @@ -256,16 +255,10 @@ def mingw_training( run_command([cmake_command, '--install', creator_build_dir, '--prefix', creator_install_dir], creator_build_dir, environment) run_command([cmake_command, '--install', creator_build_dir, '--prefix', creator_install_dir, '--component', 'Dependencies'], creator_build_dir, environment) - # Remove the regular libclang.dll which got deployed via 'Dependencies' qtcreator install target - os.remove(os.path.join(creator_install_dir, 'bin', 'libclang.dll')) - - # Train mingw libclang library with build QtCreator - # First time open the project, then close it. This will generate initial settings and .user files. Second time do the actual training. - for batch_file in ['qtc.openProject.batch', 'qtc.fileTextEditorCpp.batch']: - run_command( - [os.path.join(training_dir, 'runBatchFiles.bat'), msvc_version(), 'x64' if bitness == 64 else 'x86', batch_file], - base_path, extra_environment=None, only_error_case_output=False, expected_exit_codes=[0, 1] - ) + # Train mingw clangd executable with build QtCreator + run_command( + [os.path.join(training_dir, 'runBatchFiles.bat'), msvc_version(), 'x64' if bitness == 64 else 'x86'], + base_path, extra_environment=None, only_error_case_output=False, expected_exit_codes=[0, 1]) def is_msvc_toolchain(toolchain: str) -> bool: @@ -367,7 +360,7 @@ def get_cmake_command( ) -> List[str]: enabled_projects = 'clang;clang-tools-extra;openmp' if profile_data_path and first_run: - enabled_projects = 'clang' + enabled_projects = 'clang;clang-tools-extra' command = ['cmake', '-DCMAKE_INSTALL_PREFIX=' + install_path, @@ -378,6 +371,7 @@ def get_cmake_command( '-DLLVM_ENABLE_LIBXML2=OFF', '-DLLVM_ENABLE_ZLIB=OFF', '-DLLVM_ENABLE_TERMINFO=OFF', + '-DHAVE_CLANG_REPL_SUPPORT=OFF', '-DLLVM_TARGETS_TO_BUILD=X86;AArch64', '-DLLVM_ENABLE_PROJECTS=' + enabled_projects, "-DLLVM_LIT_ARGS='-v'"] @@ -416,15 +410,15 @@ def build_clang( run_cmd(cmd=cmake_cmd, cwd=build_path, env=environment) - build_targets = ['libclang', 'clang', 'llvm-config'] + build_targets = ['clang', 'clangd', 'llvm-config'] install_targets = ['install/strip'] if is_msvc_toolchain(toolchain): install_targets = ['install'] # There is no 'install/strip' for nmake. if profile_data_path and first_run: - build_targets = ['libclang'] - install_targets = ['tools/clang/tools/libclang/install/strip'] # we only want to build / install libclang + build_targets = ['clangd'] + install_targets = ['tools/clang/tools/extra/clangd/install/strip'] # we only want to build / install clangd build_and_install(build_path, environment, build_targets, install_targets) diff --git a/packaging-tools/libclang_training/libclangtimings2csv.py b/packaging-tools/libclang_training/libclangtimings2csv.py deleted file mode 100644 index ee51d8b5d..000000000 --- a/packaging-tools/libclang_training/libclangtimings2csv.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -############################################################################ -# -# Copyright (C) 2023 The Qt Company Ltd. -# Contact: https://www.qt.io/licensing/ -# -# This file is part of Qt Creator. -# -# Commercial License Usage -# Licensees holding valid commercial Qt licenses may use this file in -# accordance with the commercial license agreement provided with the -# Software or, alternatively, in accordance with the terms contained in -# a written agreement between you and The Qt Company. For licensing terms -# and conditions see https://www.qt.io/terms-conditions. For further -# information use the contact form at https://www.qt.io/contact-us. -# -# GNU General Public License Usage -# Alternatively, this file may be used under the terms of the GNU -# General Public License version 3 as published by the Free Software -# Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -# included in the packaging of this file. Please review the following -# information to ensure the GNU General Public License requirements will -# be met: https://www.gnu.org/licenses/gpl-3.0.html. -# -############################################################################ - -"""Convert the output of libclang when running with LIBCLANG_TIMING=1 to csv data. - -Only the wall clock time value is extracted. - -Use this converter together with /usr/bin/paste to create whole csv tables. - -Usage: ./$0 <log-file of libclang output>""" - -import getopt -import re -import sys -from pathlib import Path -from typing import List, Pattern - - -def construct_record_matcher() -> Pattern[str]: - regex = ( - '( Parsing)' - + '|( Precompiling preamble)' - + '|( Reparsing)' - + '|( Cache global code completions)' - + '|( Code completion)' - ) - - return re.compile(regex) - - -def construct_time_needed_matcher() -> Pattern[str]: - # An output line looks like: - # : 2.5625 (100.0%) 0.1563 (100.0%) 2.7188 (100.0%) 2.7813 (100.0%) - # Note: There is always at least the wall clock time at the utmost right, - # the others in front (up to 3) are optional. - start_indicator = r'\s*:' - not_relevant_parts = r'(\s*\d+\.\d+ \(\d+\.\d+\%\)){0,3}' - wall_clock_time = r'\s*(\d+\.\d+) \(\d+\.\d+\%\)' - - regex = start_indicator + not_relevant_parts + wall_clock_time - - return re.compile(regex) - - -def csv_line(values: List[str]) -> str: - return ",".join(values) + "\n" - - -def extract_records(file_content: str) -> List[List[str]]: - record_matcher = construct_record_matcher() - time_needed_matcher = construct_time_needed_matcher() - - records: List[List[str]] = [] - previous_time_match_end = -1 - - for record_start_match in record_matcher.finditer(file_content): - time_needed_in_ms = "" - if previous_time_match_end >= record_start_match.start(): - # Ops, we've detected a missing time record. - previous_record = records[-1] - records[-1] = [previous_record[0], '-1'] - time_needed_in_ms = previous_record[1] - - if not time_needed_in_ms: - time_match = next(time_needed_matcher.finditer(file_content, record_start_match.end())) - previous_time_match_end = time_match.end() - time_needed_in_ms = time_match.group(2) - - record_id = record_start_match.group().strip() - record = [record_id, time_needed_in_ms] - records.append(record) - - # for record in records: print record - return records - - -def records_to_string(records: List[List[str]]) -> str: - string = "" - for record in records: - string += csv_line(record) - - return string - - -def convert(input_file: str, column_label: str = "") -> str: - if not column_label: - column_label = Path(input_file).name - with open(input_file, 'r', encoding="utf-8") as file_content: - records = [[column_label, column_label]] + extract_records(file_content.read()) - - return records_to_string(records) - - -def print_usage_and_exit() -> None: - print(__doc__) - raise SystemExit(0) - - -def main() -> None: - # parse command line options - try: - opts, args = getopt.getopt(sys.argv[1:], "h", ["help"]) - except getopt.error as msg: - raise SystemExit("for help use --help") from msg - - # process options - for opt, _ in opts: - if opt in ("-h", "--help"): - print_usage_and_exit() - - # process arguments - if not args: - print_usage_and_exit() - for arg in args: - print(convert(arg)) - - -if __name__ == "__main__": - main() diff --git a/packaging-tools/libclang_training/merge_csv_files.py b/packaging-tools/libclang_training/merge_csv_files.py deleted file mode 100644 index df486e51a..000000000 --- a/packaging-tools/libclang_training/merge_csv_files.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -############################################################################ -# -# Copyright (C) 2022 The Qt Company Ltd. -# Contact: https://www.qt.io/licensing/ -# -# This file is part of Qt Creator. -# -# Commercial License Usage -# Licensees holding valid commercial Qt licenses may use this file in -# accordance with the commercial license agreement provided with the -# Software or, alternatively, in accordance with the terms contained in -# a written agreement between you and The Qt Company. For licensing terms -# and conditions see https://www.qt.io/terms-conditions. For further -# information use the contact form at https://www.qt.io/contact-us. -# -# GNU General Public License Usage -# Alternatively, this file may be used under the terms of the GNU -# General Public License version 3 as published by the Free Software -# Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -# included in the packaging of this file. Please review the following -# information to ensure the GNU General Public License requirements will -# be met: https://www.gnu.org/licenses/gpl-3.0.html. -# -############################################################################ - - -"""Merge given *.csv files - -Usage: ./$0 <mergedFileName> <csvFile1> <csvFile2> ...""" - -import csv -import getopt -import sys -from typing import List, Tuple - - -class Global: - Delimiter = ',' - - -class FileWithValues: - def __init__(self, file_path: str, tag: str, values: List[List[str]]) -> None: - self.file_path = file_path - self.tag = tag - self.values = values - - -def read_csv(file_path: str, delimiter: str) -> List[Tuple[str, str]]: - lines = [] - with open(file_path, 'rt', encoding="utf-8") as handle: - lines = handle.readlines() - - records = [] - for line in lines: - identifier, value = line.split(delimiter) - identifier = identifier.strip() - value = value.strip() - records.append((identifier, value)) - - return records - - -def read_csv_files(file_paths: List[str]) -> List[FileWithValues]: - files = [] - - for file_path in file_paths: - with open(file_path, 'rt', encoding="utf-8") as handle: - reader = csv.reader(handle, delimiter=Global.Delimiter, quoting=csv.QUOTE_NONE) - - values = [] - for row in reader: - values.append(row) - - tag = values[0][0] # remember column label - values = values[1:] # skip header - - my_file = FileWithValues(file_path, tag, values) - files.append(my_file) - - return files - - -def check_consistency(files: List[FileWithValues]) -> List[str]: - reference_entry = files[0] - reference_entry_size = len(reference_entry.values) - reference_entry_identifiers = [v[0] for v in reference_entry.values] - - # Ensure same size of records - for file in files: - if not len(file.values) == reference_entry_size: - raise SystemExit(f"Entry count mismatch: {reference_entry.file_path}, {file.file_path}") - - # Ensure same identifier on the left - for file in files: - identifiers = [v[0] for v in file.values] - if not identifiers == reference_entry_identifiers: - raise SystemExit(f"Column 1 id mismatch: {reference_entry.file_path}, {file.file_path}") - - return reference_entry_identifiers - - -def merge_files_helper( - output_file_path: str, reference_identifiers: List[str], files: List[FileWithValues] -) -> None: - with open(output_file_path, "wt", encoding="utf-8") as csvfile: - delimiter = Global.Delimiter - writer = csv.writer(csvfile, delimiter=delimiter, quotechar='"', quoting=csv.QUOTE_MINIMAL) - - # Write header row - headers = ['action'] + [f.tag for f in files] - writer.writerow(headers) - - # Write values - columns = [[v[1] for v in f.values] for f in files] - rows = list(zip(reference_identifiers, *columns)) - for row in rows: - writer.writerow(row) - - -def merge_files(output_file_path: str, files_to_merge: List[str]) -> None: - files = read_csv_files(files_to_merge) - reference_identifiers = check_consistency(files) - merge_files_helper(output_file_path, reference_identifiers, files) - - -def print_help_and_exit() -> None: - print(__doc__) - raise SystemExit(0) - - -def main() -> None: - try: - opts, args = getopt.getopt(sys.argv[1:], "h", ["help"]) - except getopt.error as msg: - raise SystemExit("for help use --help") from msg - - for opt, _ in opts: - if opt in ("-h", "--help"): - print_help_and_exit() - if len(args) <= 2: - print_help_and_exit() - - output_file = args[0] - files_to_merge = args[1:] - merge_files(output_file, files_to_merge) - - -if __name__ == "__main__": - main() diff --git a/packaging-tools/libclang_training/qtc.fileTextEditorCpp.batch b/packaging-tools/libclang_training/qtc.fileTextEditorCpp.batch deleted file mode 100644 index dced9520e..000000000 --- a/packaging-tools/libclang_training/qtc.fileTextEditorCpp.batch +++ /dev/null @@ -1,39 +0,0 @@ -openProject "qt-creator/qtcreator.pro" - -# Train initial parsing -openDocument "qt-creator/src/plugins/texteditor/texteditor.cpp" -closeAllDocuments -openDocument "qt-creator/src/plugins/texteditor/texteditor.cpp" -closeAllDocuments -openDocument "qt-creator/src/plugins/texteditor/texteditor.cpp" -closeAllDocuments - -openDocument "qt-creator/src/plugins/texteditor/texteditor.cpp" - -setCursor 8342 1 - -# Train reparse -insertText " " -insertText " " -insertText " " -insertText " " -insertText " " -insertText " " -insertText " " -insertText " " -insertText " " -insertText " " - -# Train complete -complete -complete -complete - -# Train member-complete -insertText "d->" -complete -complete -complete - -# Wait in order to inspect the result -processEvents 3000 diff --git a/packaging-tools/libclang_training/qtc.openProject.batch b/packaging-tools/libclang_training/qtc.openProject.batch deleted file mode 100644 index 7b77f0c34..000000000 --- a/packaging-tools/libclang_training/qtc.openProject.batch +++ /dev/null @@ -1,4 +0,0 @@ -openProject "qt-creator/qtcreator.pro" - -# Wait in order to inspect the result -processEvents 3000 diff --git a/packaging-tools/libclang_training/runBatchFiles.bat b/packaging-tools/libclang_training/runBatchFiles.bat index ce36b3536..9ef21941b 100644 --- a/packaging-tools/libclang_training/runBatchFiles.bat +++ b/packaging-tools/libclang_training/runBatchFiles.bat @@ -10,11 +10,10 @@ set PATH=%BasePath%\qt\lib;%BasePath%\qt\bin;%BasePath%\qtcreator_install\bin;%B set PATH=%PATH%;%BasePath%\logs\ rem set up run_batch_files.py -set QTC_CLANG_BATCH_CONFIG_LOG_DIR=%BasePath%\logs -set QTC_CLANG_BATCH_CONFIG_SETTINGS_DIR=%BasePath%\qtc-settings -set QTC_CLANG_BATCH_CONFIG_TARGET_LIBCLANG=%BasePath%\logs\libclang.dll -set QTC_CLANG_BATCH_CONFIG_LIBCLANGS=%BasePath%\libclang-training\bin\libclang.dll -set QTC_CLANG_BATCH_CONFIG_FILES=%ScriptPath%\%3 +set QTC_CLANGD_CONFIG_LOG_DIR=%BasePath%\logs +set QTC_CLANGD_CONFIG_SETTINGS_DIR=%BasePath%\qtc-settings +set QTC_CLANGD_COMPLETION_RESULTS=0 +set QTC_CLANGD=%BasePath%\libclang-training\bin\clangd.exe rem run run_batch_files.py set diff --git a/packaging-tools/libclang_training/run_batch_files.py b/packaging-tools/libclang_training/run_batch_files.py index 244661d41..1ea06e020 100644 --- a/packaging-tools/libclang_training/run_batch_files.py +++ b/packaging-tools/libclang_training/run_batch_files.py @@ -27,32 +27,13 @@ ############################################################################ """ -Run Qt Creator in 'clang batch file' mode and produce csv data of resulting -libclang operations (e.g. parse, reparse, completion, preamble generation). - -Multiple batch files and libclang binaries can be specified for the runs, thus -producing data for an overview of libclang's performance in certain use cases -over multiple libclang binaries/versions. - -The process environment configures this script. The relevant variables and their -meaning are shown in this pseudo code of the main algorithm: - - for libclang in QTC_CLANG_BATCH_CONFIG_LIBCLANGS - copy libclang to QTC_CLANG_BATCH_CONFIG_TARGET_LIBCLANG - for batchfile in QTC_CLANG_BATCH_CONFIG_FILES - run qtcreator with batchfile - write log/csv data files into QTC_CLANG_BATCH_CONFIG_LOG_DIR - merge csv data files per batch file. +Run Qt Creator with the the ClangCodeModel tests The PATH must contain: * qtcreator executable and everything needed to run it. * qmake is in PATH for proper set up of an automatically created kit. * dbgview.exe on Windows since this is used to capture the output. -Notes: - * For convenience, create a *.sh/*.bat file setting up and running this script. - * Ensure that the specified libclang binaries expect the same intrinsics - (<libclang install dir>/lib/clang/x.y.z/include) since these are not copied! """ import os @@ -63,10 +44,6 @@ from subprocess import STDOUT, Popen from time import sleep, time from typing import Dict, List, Optional -import libclangtimings2csv -import merge_csv_files - - def verbose_start(args: List[str]) -> None: if Config.Verbose: print(f"info: starting {args}") @@ -77,9 +54,9 @@ def check_existence_or_die(file_path: str) -> None: raise SystemExit(f"file path does not exist: {file_path}") -def check_exit_code_or_die(exit_code: int, args: List[str]) -> None: +def check_exit_code(exit_code: int, args: List[str]) -> None: if exit_code != 0: - raise SystemExit(f"exit code {exit_code} for {' '.join(args)}") + print(f"Exit code {exit_code} for {' '.join(args)}") class Config: @@ -88,57 +65,21 @@ class Config: LogDir: str = "" QtCreatorSettingsDir: str = "" - TargetLibClangDll: str = "" - - LibClangDlls: List[str] = [] - BatchFiles: List[str] = [] @staticmethod def initialize_from_environment() -> None: - Config.LogDir = os.environ['QTC_CLANG_BATCH_CONFIG_LOG_DIR'] + Config.LogDir = os.environ['QTC_CLANGD_CONFIG_LOG_DIR'] check_existence_or_die(Config.LogDir) - Config.QtCreatorSettingsDir = os.environ['QTC_CLANG_BATCH_CONFIG_SETTINGS_DIR'] + Config.QtCreatorSettingsDir = os.environ['QTC_CLANGD_CONFIG_SETTINGS_DIR'] check_existence_or_die(Config.QtCreatorSettingsDir) - Config.TargetLibClangDll = os.environ['QTC_CLANG_BATCH_CONFIG_TARGET_LIBCLANG'] - - libclang_dlls = os.environ['QTC_CLANG_BATCH_CONFIG_LIBCLANGS'] - Config.LibClangDlls = libclang_dlls.split(os.pathsep) - assert len(Config.LibClangDlls) >= 1 - for dll in Config.LibClangDlls: - check_existence_or_die(dll) - - batch_files = os.environ['QTC_CLANG_BATCH_CONFIG_FILES'] - Config.BatchFiles = batch_files.split(os.pathsep) - assert len(Config.BatchFiles) >= 1 - for batch_file in Config.BatchFiles: - check_existence_or_die(batch_file) - # TODO: Check for format - @staticmethod def dump() -> None: print("log dir:") print(f" {Config.LogDir}") print("qt creator settings dir:") print(f" {Config.QtCreatorSettingsDir}") - print("target libclang:") - print(f" {Config.TargetLibClangDll}") - print("libclangs:") - for dll in Config.LibClangDlls: - print(f" {dll}") - print("batch files:") - for batch_file in Config.BatchFiles: - print(f" {batch_file}") - - -class RunRecord: - def __init__(self, libclang_id: str, batch_file_path: str): - self.libclang_id = libclang_id - parts = os.path.basename(batch_file_path).split('.') - self.batch_file_id = '.'.join(parts[0:-1]) # Remove suffix - self.log_file_path = self.batch_file_id + '___' + libclang_id + '.log' - self.csv_file_path = "" class DebugView: @@ -161,70 +102,45 @@ class DebugView: self.proc.wait() -def create_environment(batch_file_path: str) -> Dict[str, str]: +def create_environment() -> Dict[str, str]: env = os.environ.copy() - env['LIBCLANG_TIMING'] = '1' - env['QT_LOGGING_RULES'] = 'qtc.clangcodemodel.batch=true' - env['QTC_NO_CODE_INDEXER'] = '1' - env['QTC_CLANG_NO_ALIVE_TIMER'] = '1' - env['QTC_CLANG_NO_SUPPORTIVE_TRANSLATIONUNIT'] = '1' - env['QTC_CLANG_BATCH_TIMEOUT'] = '3000000' - env['QTC_CLANG_BATCH'] = batch_file_path + env['QT_LOGGING_RULES'] = 'qtc.clangcodemodel.clangd=true;qtc.clangcodemodel.clangd.timing=true' return env - -def run_sync_and_log_output_windows(args: List[str], batch_file_path: str, log_file_path: str) -> None: +def run_sync_and_log_output_windows(args: List[str], log_file_path: str) -> None: debug_view = DebugView(log_file_path) debug_view.start_async() verbose_start(args) - with Popen(args, env=create_environment(batch_file_path)) as proc: + with Popen(args, env=create_environment()) as proc: proc.communicate() debug_view.stop() - check_exit_code_or_die(proc.returncode, args) + check_exit_code(proc.returncode, args) -def run_sync_and_log_output_unix(args: List[str], batch_file_path: str, log_file_path: str) -> None: +def run_sync_and_log_output_unix(args: List[str], log_file_path: str) -> None: with open(log_file_path, 'w', encoding="utf-8") as log_file: verbose_start(args) - with Popen(args, stdout=log_file, stderr=STDOUT, env=create_environment(batch_file_path)) as proc: + with Popen(args, stdout=log_file, stderr=STDOUT, env=create_environment()) as proc: proc.communicate() - check_exit_code_or_die(proc.returncode, args) + check_exit_code(proc.returncode, args) -def run_qtcreator_with_batch_file(batch_file_path: str, log_file_path: str) -> None: +def run_qtcreator_with_log_file(log_file_path: str) -> None: args = [ 'qtcreator', - '-noload', 'all', - '-load', 'CppEditor', - '-load', 'QmakeProjectManager', - '-load', 'ClangCodeModel', - '-load', 'Designer', '-settingspath', Config.QtCreatorSettingsDir, + '-test', 'ClangCodeModel' ] if sys.platform == "win32": - run_sync_and_log_output_windows(args, batch_file_path, log_file_path) + run_sync_and_log_output_windows(args, log_file_path) else: - run_sync_and_log_output_unix(args, batch_file_path, log_file_path) - - -def convert_log_file_to_csv_file(log_file_path: str, column_label: str) -> str: - output = libclangtimings2csv.convert(log_file_path, column_label) - - csv_file_path = log_file_path + '.csv' - with open(csv_file_path, 'w', encoding="utf-8") as handle: - handle.write(output) - - return csv_file_path - - -def log_file_from_id(log_file_id: str) -> str: - return log_file_id + ".log" + run_sync_and_log_output_unix(args, log_file_path) def create_dir(dir_path: str) -> None: @@ -233,107 +149,17 @@ def create_dir(dir_path: str) -> None: print(f"info: creating not existent {dir_path}") Path(dir_path).mkdir(parents=True) - -def create_backup_file(file_path: str) -> None: - if os.path.exists(file_path): - backup_path = file_path[:-4] + ".backup_" + str(time()) + ".log" - if Config.Verbose: - print(f"info: creating backup of already existing '{file_path}'") - copyfile(file_path, backup_path) - - -def print_duration(seconds: float) -> None: - hours, remainder = divmod(seconds, 3600) - minutes, seconds = divmod(remainder, 60) - print(f"...needed {hours}:{minutes}:{seconds}") - - -def process_batch_file_timed(libclang_id: str, batch_file_path: str) -> RunRecord: - time_started = time() - print(f"processing {batch_file_path}", end=' ') - - run_record = process_batch_file(libclang_id, batch_file_path) - - print_duration(time() - time_started) - - return run_record - - -def process_batch_file(libclang_id: str, batch_file_path: str) -> RunRecord: - run_record = RunRecord(libclang_id, batch_file_path) - log_file_path = os.path.join(Config.LogDir, run_record.log_file_path) - - create_dir(Config.LogDir) - create_backup_file(log_file_path) - - run_qtcreator_with_batch_file(batch_file_path, log_file_path) - - csv_file_path = convert_log_file_to_csv_file(log_file_path, run_record.libclang_id) - run_record.csv_file_path = csv_file_path - - return run_record - - -def get_libclang_id(libclang_dll: str) -> str: - file_name = Path(libclang_dll).name - parts = file_name.split('.') - identifier = '.'.join(parts[0:-1]) - return identifier - - -def switch_libclang(libclang_dll: str) -> None: - print(f"copying '{libclang_dll}' -> '{Config.TargetLibClangDll}'") - copyfile(libclang_dll, Config.TargetLibClangDll) - - -def run_qtcreator_with_libclang(libclang_dll: str) -> List[RunRecord]: - print("") - switch_libclang(libclang_dll) - - run_records = [] - libclang_id = get_libclang_id(libclang_dll) - for batch_file in Config.BatchFiles: - run_record = process_batch_file_timed(libclang_id, batch_file) - run_records.append(run_record) - - return run_records - - -def log_id_part_from_libclang_dll(libclang_dll: str) -> str: - file_name = Path(libclang_dll).name - parts = file_name.split('.') - file_name = '.'.join(parts[1:-1]) - return file_name - - -def merge_generated_csv_files(run_records: List[RunRecord]) -> None: - batch_file_id_2_run_record: Dict[str, List[RunRecord]] = {} - for run_record in run_records: - new_value = [run_record] - if run_record.batch_file_id in batch_file_id_2_run_record: - new_value = batch_file_id_2_run_record[run_record.batch_file_id] - new_value.append(run_record) - batch_file_id_2_run_record[run_record.batch_file_id] = new_value - - for batch_file_id, runrecord_list in batch_file_id_2_run_record.items(): - csv_file_paths = [run_record.csv_file_path for run_record in runrecord_list] - merge_file_path = os.path.join(Config.LogDir, batch_file_id + ".csv") - - merge_csv_files.merge_files(merge_file_path, csv_file_paths) - print(f"generated: {merge_file_path}") - - def main() -> None: Config.initialize_from_environment() Config.dump() - run_records = [] - for libclang_dll in Config.LibClangDlls: - run_records += run_qtcreator_with_libclang(libclang_dll) + log_file_path = os.path.join(Config.LogDir, "qtcreator.log") + create_dir(Config.LogDir) - print() - merge_generated_csv_files(run_records) + run_qtcreator_with_log_file(log_file_path) + with open(log_file_path) as log: + print(log.read()) if __name__ == "__main__": main() |