#!/usr/bin/env python3 # -*- coding: utf-8 -*- ############################################################################# # # Copyright (C) 2023 The Qt Company Ltd. # Contact: https://www.qt.io/licensing/ # # This file is part of the release tools of the Qt Toolkit. # # $QT_BEGIN_LICENSE:GPL-EXCEPT$ # Commercial License Usage # Licensees holding valid commercial Qt licenses may use this file in # accordance with the commercial license agreement provided with the # Software or, alternatively, in accordance with the terms contained in # a written agreement between you and The Qt Company. For licensing terms # and conditions see https://www.qt.io/terms-conditions. For further # information use the contact form at https://www.qt.io/contact-us. # # GNU General Public License Usage # Alternatively, this file may be used under the terms of the GNU # General Public License version 3 as published by the Free Software # Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT # included in the packaging of this file. Please review the following # information to ensure the GNU General Public License requirements will # be met: https://www.gnu.org/licenses/gpl-3.0.html. # # $QT_END_LICENSE$ # ############################################################################# import os from contextlib import suppress from pathlib import Path from subprocess import CalledProcessError from typing import Dict, List, Optional import environmentfrombatchfile from bld_utils import is_linux, is_windows, run_command from read_remote_config import get_pkg_value from runner import run_cmd def git_clone_and_checkout(base_path: str, remote_repository_url: str, directory: str, revision: str) -> None: run_command(['git', 'clone', '--config', 'core.eol=lf', '--config', 'core.autocrlf=input', '--branch', revision, '--recursive', remote_repository_url, directory], base_path) def get_clang(base_path: str, llvm_revision: str) -> None: git_clone_and_checkout(base_path, 'git://code.qt.io/clang/llvm-project.git', 'llvm', llvm_revision) def msvc_version() -> str: msvc_ver = os.environ.get('MSVC_VERSION') if not msvc_ver: msvc_ver = '14.2' return msvc_ver def msvc_year_version() -> str: return { '12.0': 'MSVC2013', '14.0': 'MSVC2015', '14.1': 'MSVC2017', '14.2': 'MSVC2019' }.get(os.environ.get('MSVC_VERSION', ''), 'MSVC2019') def msvc_year() -> str: return { '12.0': '2013', '14.0': '2015', '14.1': '2017', '14.2': '2019' }.get(os.environ.get('MSVC_VERSION', ''), 'MSVC2019') def msvc_year_version_libclang() -> str: return { '12.0': 'vs2013', '14.0': 'vs2015', '14.1': 'vs2017', '14.2': 'vs2019' }.get(os.environ.get('MSVC_VERSION', ''), 'vs2019') def msvc_environment(bitness: int) -> Dict[str, str]: program_files = os.path.join('C:', '/Program Files (x86)') if not os.path.exists(program_files): program_files = os.path.join('C:', '/Program Files') msvc_ver = msvc_version() if msvc_ver in ('14.1', '14.2'): vcvarsall = os.path.join(program_files, 'Microsoft Visual Studio', msvc_year(), 'Professional', 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat') arg = 'x64' if bitness == 64 else 'x86' else: vcvarsall = os.path.join(program_files, 'Microsoft Visual Studio ' + msvc_ver, 'VC', 'vcvarsall.bat') arg = 'amd64' if bitness == 64 else 'x86' return environmentfrombatchfile.get(vcvarsall, arguments=arg) def paths_with_sh_exe_removed(path_value: str) -> str: items = path_value.split(os.pathsep) items = [i for i in items if not os.path.exists(os.path.join(i, 'sh.exe'))] return os.pathsep.join(items) def build_environment(toolchain: str, bitness: int) -> Optional[Dict[str, str]]: if is_windows(): if is_mingw_toolchain(toolchain): environment = dict(os.environ) # cmake says "For MinGW make to work correctly sh.exe must NOT be in your path." environment['PATH'] = paths_with_sh_exe_removed(environment['PATH']) environment['CC'] = 'gcc' environment['CXX'] = 'g++' return environment environment = msvc_environment(bitness) environment['CC'] = 'cl' environment['CXX'] = 'cl' return environment return None # == process environment def is_msvc_toolchain(toolchain: str) -> bool: return 'msvc' in toolchain def is_mingw_toolchain(toolchain: str) -> bool: return 'mingw' in toolchain def is_gcc_toolchain(toolchain: str) -> bool: return 'g++' in toolchain def cmake_generator() -> str: if is_windows(): return 'Ninja' return 'Unix Makefiles' def static_flags(toolchain: str) -> List[str]: if is_mingw_toolchain(toolchain): linker_flags = '-static-libgcc -static-libstdc++ -static' return [ '-DCMAKE_SHARED_LINKER_FLAGS=' + linker_flags, '-DCMAKE_EXE_LINKER_FLAGS=' + linker_flags, ] return [] def bitness_flags(bitness: int) -> List[str]: if bitness == 32 and is_linux(): return ['-DLLVM_BUILD_32_BITS=ON'] return [] def rtti_flags(toolchain: str) -> List[str]: if is_mingw_toolchain(toolchain): return ['-DLLVM_ENABLE_RTTI:BOOL=OFF'] return ['-DLLVM_ENABLE_RTTI:BOOL=ON'] def build_command() -> List[str]: if is_windows(): command = ['ninja'] else: command = ['make'] return command def install_command() -> List[str]: if is_windows(): command = ['ninja'] else: command = ['make', '-j1'] return command # For instrumented build we now use the same targets because clazy # requires the llvm installation to properly build def build_and_install( build_path: str, environment: Optional[Dict[str, str]], build_targets: List[str], install_targets: List[str], ) -> None: run_cmd(cmd=build_command() + build_targets, cwd=build_path, env=environment) run_cmd(cmd=install_command() + install_targets, cwd=build_path, env=environment) def cmake_command( toolchain: str, src_path: str, install_path: str, bitness: int, build_type: str ) -> List[str]: command = ['cmake', '-DCMAKE_INSTALL_PREFIX=' + install_path, '-G', cmake_generator(), '-DCMAKE_BUILD_TYPE=' + build_type, '-DCMAKE_POSITION_INDEPENDENT_CODE=ON', '-DLLVM_ENABLE_PROJECTS=clang', '-DLLVM_TARGETS_TO_BUILD=X86', '-DLLVM_ENABLE_LIBXML2=OFF', '-DLLVM_ENABLE_ZLIB=OFF', '-DLLVM_ENABLE_TERMINFO=OFF', '-DLIBCLANG_BUILD_STATIC=ON', '-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON', '-DLLVM_ENABLE_PIC=OFF', "-DLLVM_LIT_ARGS='-v'"] if is_msvc_toolchain(toolchain): command.append('-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=1') command.extend(bitness_flags(bitness)) command.extend(rtti_flags(toolchain)) command.extend(static_flags(toolchain)) command.append(src_path) return command def build_clang( toolchain: str, src_path: str, build_path: str, install_path: str, bitness: int = 64, environment: Optional[Dict[str, str]] = None, build_type: str = "Release", ) -> None: if build_path and not os.path.lexists(build_path): Path(build_path).mkdir(parents=True) cmake_cmd = cmake_command(toolchain, src_path, install_path, bitness, build_type) run_cmd(cmd=cmake_cmd, cwd=build_path, env=environment) build_targets = ["install/strip"] if is_msvc_toolchain(toolchain): build_targets = ['install'] # There is no 'install/strip' for nmake. build_and_install(build_path, environment, ['libclang', 'clang', 'llvm-config'], build_targets) def check_clang(toolchain: str, build_path: str, environment: Optional[Dict[str, str]]) -> None: environment = environment if environment else os.environ.copy() if is_msvc_toolchain(toolchain) or is_mingw_toolchain(toolchain): tools_path = os.environ.get('WINDOWS_UNIX_TOOLS_PATH') if tools_path: path_key = 'Path' if 'Path' in environment else 'PATH' environment[path_key] += ';' + tools_path with suppress(CalledProcessError): run_cmd(cmd=build_command() + ["check-clang"], cwd=build_path, env=environment) def package_clang(install_path: str, result_file_path: str) -> None: install_dir = Path(install_path) zip_cmd = ['cmake', '-E', 'tar', 'cvf', result_file_path, '--format=7zip', install_dir.name] run_command(zip_cmd, cwd=str(install_dir.parent)) def upload_clang(file_path: str, remote_path: str) -> None: local_path = Path(file_path) scp_bin = '%SCP%' if is_windows() else 'scp' scp_command = [scp_bin, local_path.name, remote_path] run_command(scp_command, cwd=str(local_path.parent)) def main() -> None: # Used Environment variables: # # PKG_NODE_ROOT # Absolute path of a working directory for this script. # It checks out LLVM and Clang in "$PKG_NODE_ROOT/llvm", # builds it in "$PKG_NODE_ROOT/build", and installs it to # "$PKG_NODE_ROOT/libclang" # # CLANG_BRANCH # "Branch" identifier for the resulting package name # # cfg # Configuration containing of platform and bitness information # like "linux-g++-Rhel7.2-x64", "mac-clang-10.11-x64", # "win-MinGW5.3.0-Windows10-x64", "win-MinGW5.3.0-Windows10-x86", # "win-msvc2015-Windows10-x64", "win-msvc2015-Windows10-x86" # # GENERATE_INSTRUMENTED_BINARIES # Set this to 1 if you want to build MinGW libraries with information # suitable for creating profile optimized builds # # PACKAGE_STORAGE_SERVER_USER # PACKAGE_STORAGE_SERVER # PACKAGE_STORAGE_SERVER_BASE_DIR # CLANG_UPLOAD_SERVER_PATH # Define a remote path where to upload the resulting package # "PACKAGE_STORAGE_SERVER_USER@PACKAGE_STORAGE_SERVER:PACKAGE_STORAGE_SERVER_BASE_DIR/CLANG_UPLOAD_SERVER_PATH" # # LLVM_REVISION # Git revision, branch or tag for LLVM/Clang check out base_path = os.path.join(os.environ['PKG_NODE_ROOT']) branch = os.environ['CLANG_BRANCH'] src_path = os.path.join(base_path, 'llvm/llvm') build_path = os.path.join(base_path, 'build') install_path = os.path.join(base_path, 'libclang') bitness = 64 if '64' in os.environ['cfg'] else 32 toolchain = os.environ['cfg'].split('-')[1].lower() environment = build_environment(toolchain, bitness) result_file_path = os.path.join(base_path, 'libclang-' + branch + '-' + os.environ['CLANG_PLATFORM'] + '.7z') remote_path = (get_pkg_value("PACKAGE_STORAGE_SERVER_USER") + '@' + get_pkg_value("PACKAGE_STORAGE_SERVER") + ':' + os.environ['PACKAGE_STORAGE_SERVER_BASE_DIR'] + '/' + os.environ['CLANG_UPLOAD_SERVER_PATH']) get_clang(base_path, os.environ['LLVM_REVISION']) # TODO: put args in some struct to improve readability, add error checks build_clang(toolchain, src_path, build_path, install_path, bitness, environment, build_type='Release') check_clang(toolchain, build_path, environment) package_clang(install_path, result_file_path) upload_clang(result_file_path, remote_path) if __name__ == "__main__": main()