diff options
Diffstat (limited to 'util/cmake')
46 files changed, 5567 insertions, 0 deletions
diff --git a/util/cmake/Pipfile b/util/cmake/Pipfile new file mode 100644 index 0000000000..7fbf716eb8 --- /dev/null +++ b/util/cmake/Pipfile @@ -0,0 +1,15 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +pytest = "*" +mypy = "*" +pyparsing = "*" +sympy = "*" + +[dev-packages] + +[requires] +python_version = "3.7" diff --git a/util/cmake/cmakeconversionrate.py b/util/cmake/cmakeconversionrate.py new file mode 100755 index 0000000000..3496ed1b91 --- /dev/null +++ b/util/cmake/cmakeconversionrate.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from argparse import ArgumentParser + +import os +import re +import subprocess +import sys +import typing + + +def _parse_commandline(): + parser = ArgumentParser(description='Calculate the conversion rate to cmake.') + parser.add_argument('--debug', dest='debug', action='store_true', + help='Turn on debug output') + parser.add_argument('source_directory', metavar='<Source Directory>', type=str, + help='The Qt module source directory') + parser.add_argument('binary_directory', metavar='<CMake build direcotry>', type=str, + help='The CMake build directory (might be empty)') + + return parser.parse_args() + + +def calculate_baseline(source_directory: str, *, debug: bool=False) -> int: + if debug: + print('Scanning "{}" for qmake-based tests.'.format(source_directory)) + result = subprocess.run('/usr/bin/git grep -E "^\\s*CONFIG\\s*\\+?=.*\\btestcase\\b" | sort -u | wc -l', + shell=True, capture_output=True, cwd=source_directory) + return int(result.stdout) + + +def build(source_directory: str, binary_directory: str, *, debug=False) -> None: + abs_source = os.path.abspath(source_directory) + if not os.path.isdir(binary_directory): + os.makedirs(binary_directory) + if not os.path.exists(os.path.join(binary_directory, 'CMakeCache.txt')): + + if debug: + print('Running cmake in "{}".'.format(binary_directory)) + result = subprocess.run(['/usr/bin/cmake', '-GNinja', abs_source], cwd=binary_directory) + if debug: + print('CMake return code: {}.'.format(result.returncode)) + + assert result.returncode == 0 + + if debug: + print('Running ninja in "{}".'.format(binary_directory)) + result = subprocess.run('/usr/bin/ninja', cwd=binary_directory) + if debug: + print('Ninja return code: {}.'.format(result.returncode)) + + assert result.returncode == 0 + + +def test(binary_directory: str, *, debug=False) -> typing.Tuple[int, int]: + if debug: + print('Running ctest in "{}".'.format(binary_directory)) + result = subprocess.run('/usr/bin/ctest -j 250 | grep "tests passed, "', + shell=True, capture_output=True, cwd=binary_directory) + summary = result.stdout.decode('utf-8').replace('\n', '') + if debug: + print('Test summary: {} ({}).'.format(summary, result.returncode)) + + matches = re.fullmatch(r'\d+% tests passed, (\d+) tests failed out of (\d+)', summary) + if matches: + if debug: + print('Matches: failed {}, total {}.'.format(matches.group(1), matches.group(2))) + return (int(matches.group(2)), int(matches.group(2)) - int(matches.group(1)), ) + + return (0, 0,) + + +def main() -> int: + args = _parse_commandline() + + base_line = calculate_baseline(args.source_directory, debug=args.debug) + if base_line <= 0: + print('Could not find the qmake baseline in {}.'.format(args.source_directory)) + return 1 + + if args.debug: + print('qmake baseline: {} test binaries.'.format(base_line)) + + cmake_total = 0 + cmake_success = 0 + try: + build(args.source_directory, args.binary_directory, debug=args.debug) + (cmake_total, cmake_success, ) = test(args.binary_directory, debug=args.debug) + finally: + if cmake_total == 0: + print('\n\n\nCould not calculate the cmake state.') + return 2 + else: + print('\n\n\nCMake test conversion rate: {:.2%}.'.format(cmake_total / base_line)) + print('CMake test success rate : {:.2%}.'.format(cmake_success / base_line)) + return 0 + + +if __name__ == '__main__': + main() diff --git a/util/cmake/configurejson2cmake.py b/util/cmake/configurejson2cmake.py new file mode 100755 index 0000000000..f929ac142d --- /dev/null +++ b/util/cmake/configurejson2cmake.py @@ -0,0 +1,976 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +import json_parser +import os.path +import re +import sys +from typing import Set, Union, List, Dict + +from helper import map_qt_library, featureName, map_platform, \ + find_3rd_party_library_mapping, generate_find_package_info + +knownTests = set() # type: Set[str] + + +class LibraryMapping: + def __init__(self, package: str, resultVariable: str, appendFoundSuffix: bool = True) -> None: + self.package = package + self.resultVariable = resultVariable + self.appendFoundSuffix = appendFoundSuffix + +def map_tests(test: str) -> str: + testmap = { + 'c++11': '$<COMPILE_FEATURES:cxx_std_11>', + 'c++14': '$<COMPILE_FEATURES:cxx_std_14>', + 'c++1z': '$<COMPILE_FEATURES:cxx_std_17>', + 'c99': '$<COMPILE_FEATURES:c_std_99>', + 'c11': '$<COMPILE_FEATURES:c_std_11>', + + 'x86SimdAlways': 'ON', # FIXME: Make this actually do a compile test. + + 'aesni': 'TEST_subarch_aes', + 'avx': 'TEST_subarch_avx', + 'avx2': 'TEST_subarch_avx2', + 'avx512f': 'TEST_subarch_avx512f', + 'avx512cd': 'TEST_subarch_avx512cd', + 'avx512dq': 'TEST_subarch_avx512dq', + 'avx512bw': 'TEST_subarch_avx512bw', + 'avx512er': 'TEST_subarch_avx512er', + 'avx512pf': 'TEST_subarch_avx512pf', + 'avx512vl': 'TEST_subarch_avx512vl', + 'avx512ifma': 'TEST_subarch_avx512ifma', + 'avx512vbmi': 'TEST_subarch_avx512vbmi', + 'avx512vbmi2': 'TEST_subarch_avx512vbmi2', + 'avx512vpopcntdq': 'TEST_subarch_avx512vpopcntdq', + 'avx5124fmaps': 'TEST_subarch_avx5124fmaps', + 'avx5124vnniw': 'TEST_subarch_avx5124vnniw', + 'bmi': 'TEST_subarch_bmi', + 'bmi2': 'TEST_subarch_bmi2', + 'cx16': 'TEST_subarch_cx16', + 'f16c': 'TEST_subarch_f16c', + 'fma': 'TEST_subarch_fma', + 'fma4': 'TEST_subarch_fma4', + 'fsgsbase': 'TEST_subarch_fsgsbase', + 'gfni': 'TEST_subarch_gfni', + 'ibt': 'TEST_subarch_ibt', + 'lwp': 'TEST_subarch_lwp', + 'lzcnt': 'TEST_subarch_lzcnt', + 'mmx': 'TEST_subarch_mmx', + 'movbe': 'TEST_subarch_movbe', + 'mpx': 'TEST_subarch_mpx', + 'no-sahf': 'TEST_subarch_no_shaf', + 'pclmul': 'TEST_subarch_pclmul', + 'popcnt': 'TEST_subarch_popcnt', + 'prefetchwt1': 'TEST_subarch_prefetchwt1', + 'prfchw': 'TEST_subarch_prfchw', + 'pdpid': 'TEST_subarch_rdpid', + 'rdpid': 'TEST_subarch_rdpid', + 'rdseed': 'TEST_subarch_rdseed', + 'rdrnd': 'TEST_subarch_rdseed', # FIXME: Is this the right thing? + 'rtm': 'TEST_subarch_rtm', + 'shani': 'TEST_subarch_sha', + 'shstk': 'TEST_subarch_shstk', + 'sse2': 'TEST_subarch_sse2', + 'sse3': 'TEST_subarch_sse3', + 'ssse3': 'TEST_subarch_ssse3', + 'sse4a': 'TEST_subarch_sse4a', + 'sse4_1': 'TEST_subarch_sse4_1', + 'sse4_2': 'TEST_subarch_sse4_2', + 'tbm': 'TEST_subarch_tbm', + 'xop': 'TEST_subarch_xop', + + 'neon': 'TEST_subarch_neon', + 'iwmmxt': 'TEST_subarch_iwmmxt', + 'crc32': 'TEST_subarch_crc32', + + 'vis': 'TEST_subarch_vis', + 'vis2': 'TEST_subarch_vis2', + 'vis3': 'TEST_subarch_vis3', + + 'dsp': 'TEST_subarch_dsp', + 'dspr2': 'TEST_subarch_dspr2', + + 'altivec': 'TEST_subarch_altivec', + 'spe': 'TEST_subarch_spe', + 'vsx': 'TEST_subarch_vsx', + + 'posix-iconv': 'TEST_posix_iconv', + 'sun-iconv': 'TEST_sun_iconv', + + 'openssl11': '(OPENSSL_VERSION VERSION_GREATER_EQUAL "1.1.0")', + + 'reduce_exports': 'CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY', + + 'libinput_axis_api': 'ON', + "xlib": "X11_FOUND", + } + if test in testmap: + return testmap.get(test, None) + if test in knownTests: + return 'TEST_{}'.format(featureName(test)) + return None + + +def cm(ctx, *output): + txt = ctx['output'] + if txt != '' and not txt.endswith('\n'): + txt += '\n' + txt += '\n'.join(output) + + ctx['output'] = txt + return ctx + + +def readJsonFromDir(dir): + path = os.path.join(dir, 'configure.json') + + print('Reading {}...'.format(path)) + assert os.path.exists(path) + + parser = json_parser.QMakeSpecificJSONParser() + return parser.parse(path) + + +def processFiles(ctx, data): + print(' files:') + if 'files' in data: + for (k, v) in data['files'].items(): + ctx[k] = v + return ctx + +def parseLib(ctx, lib, data, cm_fh, cmake_find_packages_set): + newlib = find_3rd_party_library_mapping(lib) + if not newlib: + print(' XXXX Unknown library "{}".'.format(lib)) + return + + if newlib.packageName is None: + print(' **** Skipping library "{}" -- was masked.'.format(lib)) + return + + print(' mapped library {} to {}.'.format(lib, newlib.targetName)) + + # Avoid duplicate find_package calls. + if newlib.targetName in cmake_find_packages_set: + return + + # If certain libraries are used within a feature, but the feature + # is only emitted conditionally with a simple condition (like + # 'on Windows' or 'on Linux'), we should enclose the find_package + # call for the library into the same condition. + emit_if = newlib.emit_if + + # Only look through features if a custom emit_if wasn't provided. + if not emit_if: + for feature in data['features']: + feature_data = data['features'][feature] + if 'condition' in feature_data and \ + 'libs.{}'.format(lib) in feature_data['condition'] and \ + 'emitIf' in feature_data and \ + 'config.' in feature_data['emitIf']: + emit_if = feature_data['emitIf'] + break + + if emit_if: + emit_if = map_condition(emit_if) + + cmake_find_packages_set.add(newlib.targetName) + + cm_fh.write(generate_find_package_info(newlib, emit_if=emit_if)) + + +def lineify(label, value, quote=True): + if value: + if quote: + return ' {} "{}"\n'.format(label, value.replace('"', '\\"')) + return ' {} {}\n'.format(label, value) + return '' + +def map_condition(condition): + # Handle NOT: + if isinstance(condition, list): + condition = '(' + ') AND ('.join(condition) + ')' + if isinstance(condition, bool): + if condition: + return 'ON' + else: + return 'OFF' + assert isinstance(condition, str) + + mapped_features = { + 'gbm': 'gbm_FOUND', + "system-xcb": "ON", + "system-freetype": "ON", + 'system-pcre2': 'ON', + } + + # Turn foo != "bar" into (NOT foo STREQUAL 'bar') + condition = re.sub(r"(.+)\s*!=\s*('.+')", '(! \\1 == \\2)', condition) + + condition = condition.replace('!', 'NOT ') + condition = condition.replace('&&', ' AND ') + condition = condition.replace('||', ' OR ') + condition = condition.replace('==', ' STREQUAL ') + + # explicitly handle input.sdk == '': + condition = re.sub(r"input\.sdk\s*==\s*''", 'NOT INPUT_SDK', condition) + + last_pos = 0 + mapped_condition = '' + has_failed = False + for match in re.finditer(r'([a-zA-Z0-9_]+)\.([a-zA-Z0-9_+-]+)', condition): + substitution = None + appendFoundSuffix = True + if match.group(1) == 'libs': + libmapping = find_3rd_party_library_mapping(match.group(2)) + + if libmapping and libmapping.packageName: + substitution = libmapping.packageName + if libmapping.resultVariable: + substitution = libmapping.resultVariable + if libmapping.appendFoundSuffix: + substitution += '_FOUND' + + elif match.group(1) == 'features': + feature = match.group(2) + if feature in mapped_features: + substitution = mapped_features.get(feature) + else: + substitution = 'QT_FEATURE_{}'.format(featureName(match.group(2))) + + elif match.group(1) == 'subarch': + substitution = 'TEST_arch_{}_subarch_{}'.format("${TEST_architecture_arch}", + match.group(2)) + + elif match.group(1) == 'call': + if match.group(2) == 'crossCompile': + substitution = 'CMAKE_CROSSCOMPILING' + + elif match.group(1) == 'tests': + substitution = map_tests(match.group(2)) + + elif match.group(1) == 'input': + substitution = 'INPUT_{}'.format(featureName(match.group(2))) + + elif match.group(1) == 'config': + substitution = map_platform(match.group(2)) + elif match.group(1) == 'module': + substitution = 'TARGET {}'.format(map_qt_library(match.group(2))) + + elif match.group(1) == 'arch': + if match.group(2) == 'i386': + # FIXME: Does this make sense? + substitution = '(TEST_architecture_arch STREQUAL i386)' + elif match.group(2) == 'x86_64': + substitution = '(TEST_architecture_arch STREQUAL x86_64)' + elif match.group(2) == 'arm': + # FIXME: Does this make sense? + substitution = '(TEST_architecture_arch STREQUAL arm)' + elif match.group(2) == 'arm64': + # FIXME: Does this make sense? + substitution = '(TEST_architecture_arch STREQUAL arm64)' + elif match.group(2) == 'mips': + # FIXME: Does this make sense? + substitution = '(TEST_architecture_arch STREQUAL mips)' + + if substitution is None: + print(' XXXX Unknown condition "{}".'.format(match.group(0))) + has_failed = True + else: + mapped_condition += condition[last_pos:match.start(1)] + substitution + last_pos = match.end(2) + + mapped_condition += condition[last_pos:] + + # Space out '(' and ')': + mapped_condition = mapped_condition.replace('(', ' ( ') + mapped_condition = mapped_condition.replace(')', ' ) ') + + # Prettify: + condition = re.sub('\\s+', ' ', mapped_condition) + condition = condition.strip() + + if has_failed: + condition += ' OR FIXME' + + return condition + + +def parseInput(ctx, input, data, cm_fh): + skip_inputs = { + "prefix", "hostprefix", "extprefix", + + "archdatadir", "bindir", "datadir", "docdir", + "examplesdir", "external-hostbindir", "headerdir", + "hostbindir", "hostdatadir", "hostlibdir", + "importdir", "libdir", "libexecdir", + "plugindir", "qmldir", "settingsdir", + "sysconfdir", "testsdir", "translationdir", + + "android-arch", "android-ndk", "android-ndk-host", + "android-ndk-platform", "android-sdk", + "android-toolchain-version", "android-style-assets", + + "appstore-compliant", + + "avx", "avx2", "avx512", "c++std", "ccache", "commercial", + "compile-examples", "confirm-license", + "dbus", + "dbus-runtime", + + "debug", "debug-and-release", + + "developer-build", + + "device", "device-option", + + "f16c", + + "force-asserts", "force-debug-info", "force-pkg-config", + "framework", + + "gc-binaries", + + "gdb-index", + + "gcc-sysroot", + + "gcov", + + "gnumake", + + "gui", + + "harfbuzz", + + "headersclean", + + "incredibuild-xge", + + "libudev", + "ltcg", + "make", + "make-tool", + + "mips_dsp", + "mips_dspr2", + "mp", + + "nomake", + + "opensource", + + "optimize-debug", "optimize-size", "optimized-qmake", "optimized-tools", + + "pch", + + "pkg-config", + + "platform", + + "plugin-manifests", + "profile", + "qreal", + + "reduce-exports", "reduce-relocations", + + "release", + + "rpath", + + "sanitize", + + "sdk", + + "separate-debug-info", + + "shared", + + "silent", + + "qdbus", + + "sse2", + "sse3", + "sse4.1", + "sse4.2", + "ssse3", + "static", + "static-runtime", + "strip", + "syncqt", + "sysroot", + "testcocoon", + "use-gold-linker", + "warnings-are-errors", + "Werror", + "widgets", + "xplatform", + "zlib", + + "doubleconversion", + + "eventfd", + "glib", + "icu", + "inotify", + "journald", + "pcre", + "posix-ipc", + "pps", + "slog2", + "syslog", + + "sqlite", + } + + if input in skip_inputs: + print(' **** Skipping input {}: masked.'.format(input)) + return + + type = data + if isinstance(data, dict): + type = data["type"] + + if type == "boolean": + print(' **** Skipping boolean input {}: masked.'.format(input)) + return + + if type == "enum": + cm_fh.write("# input {}\n".format(input)) + cm_fh.write('set(INPUT_{} "undefined" CACHE STRING "")\n'.format(featureName(input))) + cm_fh.write('set_property(CACHE INPUT_{} PROPERTY STRINGS undefined {})\n\n'.format(featureName(input), " ".join(data["values"]))) + return + + print(' XXXX UNHANDLED INPUT TYPE {} in input description'.format(type)) + return + + +# "tests": { +# "cxx11_future": { +# "label": "C++11 <future>", +# "type": "compile", +# "test": { +# "include": "future", +# "main": [ +# "std::future<int> f = std::async([]() { return 42; });", +# "(void)f.get();" +# ], +# "qmake": "unix:LIBS += -lpthread" +# } +# }, +def parseTest(ctx, test, data, cm_fh): + skip_tests = { + 'c++11', 'c++14', 'c++1y', 'c++1z', + 'c11', 'c99', + 'gc_binaries', + 'posix-iconv', "sun-iconv", + 'precomile_header', + 'reduce_exports', + 'separate_debug_info', # FIXME: see if cmake can do this + 'gc_binaries', + 'libinput_axis_api', + 'xlib', + } + + if test in skip_tests: + print(' **** Skipping features {}: masked.'.format(test)) + return + + if data["type"] == "compile": + knownTests.add(test) + + details = data["test"] + + if isinstance(details, str): + print(' XXXX UNHANDLED TEST SUB-TYPE {} in test description'.format(details)) + return + + head = details.get("head", "") + if isinstance(head, list): + head = "\n".join(head) + + sourceCode = head + '\n' + + include = details.get("include", "") + if isinstance(include, list): + include = '#include <' + '>\n#include <'.join(include) + '>' + elif include: + include = '#include <{}>'.format(include) + + sourceCode += include + '\n' + + tail = details.get("tail", "") + if isinstance(tail, list): + tail = "\n".join(tail) + + sourceCode += tail + '\n' + + sourceCode += "int main(int argc, char **argv)\n" + sourceCode += "{\n" + sourceCode += " (void)argc; (void)argv;\n" + sourceCode += " /* BEGIN TEST: */\n" + + main = details.get("main", "") + if isinstance(main, list): + main = "\n".join(main) + + sourceCode += main + '\n' + + sourceCode += " /* END TEST: */\n" + sourceCode += " return 0;\n" + sourceCode += "}\n" + + sourceCode = sourceCode.replace('"', '\\"') + + librariesCmakeName = "" + qmakeFixme = "" + + cm_fh.write("# {}\n".format(test)) + if "qmake" in details: # We don't really have many so we can just enumerate them all + if details["qmake"] == "unix:LIBS += -lpthread": + librariesCmakeName = format(featureName(test)) + "_TEST_LIBRARIES" + cm_fh.write("if (UNIX)\n") + cm_fh.write(" set(" + librariesCmakeName + " pthread)\n") + cm_fh.write("endif()\n") + elif details["qmake"] == "linux: LIBS += -lpthread -lrt": + librariesCmakeName = format(featureName(test)) + "_TEST_LIBRARIES" + cm_fh.write("if (LINUX)\n") + cm_fh.write(" set(" + librariesCmakeName + " pthread rt)\n") + cm_fh.write("endif()\n") + elif details["qmake"] == "CONFIG += c++11": + # do nothing we're always in c++11 mode + pass + else: + qmakeFixme = "# FIXME: qmake: {}\n".format(details["qmake"]) + + if "use" in data: + if data["use"] == "egl xcb_xlib": + librariesCmakeName = format(featureName(test)) + "_TEST_LIBRARIES" + cm_fh.write("if (HAVE_EGL AND X11_XCB_FOUND AND X11_FOUND)\n") + cm_fh.write(" set(" + librariesCmakeName + " EGL::EGL X11::X11 X11::XCB)\n") + cm_fh.write("endif()\n") + else: + qmakeFixme += "# FIXME: use: {}\n".format(data["use"]) + + cm_fh.write("qt_config_compile_test({}\n".format(featureName(test))) + cm_fh.write(lineify("LABEL", data.get("label", ""))) + if librariesCmakeName != "": + cm_fh.write(lineify("LIBRARIES", "${"+librariesCmakeName+"}")) + cm_fh.write(" CODE\n") + cm_fh.write('"' + sourceCode + '"') + if qmakeFixme != "": + cm_fh.write(qmakeFixme) + cm_fh.write(")\n\n") + + elif data["type"] == "x86Simd": + knownTests.add(test) + + label = data["label"] + + cm_fh.write("# {}\n".format(test)) + cm_fh.write("qt_config_compile_test_x86simd({} \"{}\")\n".format(test, label)) + cm_fh.write("\n") + +# "features": { +# "android-style-assets": { +# "label": "Android Style Assets", +# "condition": "config.android", +# "output": [ "privateFeature" ], +# "comment": "This belongs into gui, but the license check needs it here already." +# }, + else: + print(' XXXX UNHANDLED TEST TYPE {} in test description'.format(data["type"])) + + +def parseFeature(ctx, feature, data, cm_fh): + # This is *before* the feature name gets normalized! So keep - and + chars, etc. + feature_mapping = { + 'alloc_h': None, # handled by alloc target + 'alloc_malloc_h': None, + 'alloc_stdlib_h': None, + 'build_all': None, + 'c11': None, + 'c89': None, + 'c99': None, + 'ccache': None, + 'compiler-flags': None, + 'cross_compile': None, + 'debug_and_release': None, + 'debug': None, + 'dlopen': { + 'condition': 'UNIX', + }, + 'doubleconversion': None, + 'enable_gdb_index': None, + 'enable_new_dtags': None, + 'force_debug_info': None, + 'framework': { + 'condition': 'APPLE AND BUILD_SHARED_LIBS', + }, + 'gc_binaries': None, + 'gcc-sysroot': None, + 'gcov': None, + 'gnu-libiconv': { + 'condition': 'NOT WIN32 AND NOT QNX AND NOT ANDROID AND NOT APPLE AND TEST_posix_iconv AND NOT TEST_iconv_needlib', + 'enable': 'TEST_posix_iconv AND NOT TEST_iconv_needlib', + 'disable': 'NOT TEST_posix_iconv OR TEST_iconv_needlib', + }, + 'GNUmake': None, + 'harfbuzz': { + 'condition': 'HARFBUZZ_FOUND' + }, + 'host-dbus': None, + 'iconv': { + 'condition': 'NOT QT_FEATURE_icu AND QT_FEATURE_textcodec AND ( TEST_posix_iconv OR TEST_sun_iconv )' + }, + 'incredibuild_xge': None, + 'jpeg': { + 'condition': 'QT_FEATURE_imageformatplugin AND JPEG_FOUND' + }, + 'ltcg': None, + 'msvc_mp': None, + 'optimize_debug': None, + 'optimize_size': None, + # special case to enable implicit feature on WIN32, until ANGLE is ported + 'opengl-desktop': { + 'autoDetect': '' + }, + # special case to disable implicit feature on WIN32, until ANGLE is ported + 'opengl-dynamic': { + 'autoDetect': 'OFF' + }, + 'opengles2': { # special case to disable implicit feature on WIN32, until ANGLE is ported + 'condition': 'NOT WIN32 AND ( NOT APPLE_WATCHOS AND NOT QT_FEATURE_opengl_desktop AND GLESv2_FOUND )' + }, + 'pkg-config': None, + 'posix_fallocate': None, # Only needed for sqlite, which we do not want to build + 'posix-libiconv': { + 'condition': 'NOT WIN32 AND NOT QNX AND NOT ANDROID AND NOT APPLE AND TEST_posix_iconv AND TEST_iconv_needlib', + 'enable': 'TEST_posix_iconv AND TEST_iconv_needlib', + 'disable': 'NOT TEST_posix_iconv OR NOT TEST_iconv_needlib', + }, + 'precompile_header': None, + 'profile': None, + 'qmakeargs': None, + 'qpa_default_platform': None, # Not a bool! + 'reduce_relocations': None, + 'release': None, + 'release_tools': None, + 'rpath_dir': None, # rpath related + 'rpath': None, + 'sanitize_address': None, # sanitizer + 'sanitize_memory': None, + 'sanitizer': None, + 'sanitize_thread': None, + 'sanitize_undefined': None, + 'separate_debug_info': None, + 'shared': None, + 'silent': None, + 'sql-sqlite' : { + 'condition': 'QT_FEATURE_datestring AND SQLite3_FOUND', + }, + 'stack-protector-strong': None, + 'static': None, + 'static_runtime': None, + 'stl': None, # Do we really need to test for this in 2018?! + 'strip': None, + 'sun-libiconv': { + 'condition': 'NOT WIN32 AND NOT QNX AND NOT ANDROID AND NOT APPLE AND TEST_sun_iconv', + 'enable': 'TEST_sun_iconv', + 'disable': 'NOT TEST_sun_iconv', + }, + 'system-doubleconversion': None, # No system libraries anymore! + 'system-freetype': None, + 'system-harfbuzz': None, + 'system-jpeg': None, + 'system-pcre2': None, + 'system-png': None, + 'system-sqlite': None, + 'system-xcb': None, + 'system-zlib': None, + 'tiff': { + 'condition': 'QT_FEATURE_imageformatplugin AND TIFF_FOUND' + }, + 'use_gold_linker': None, + 'verifyspec': None, # qmake specific... + 'warnings_are_errors': None, # FIXME: Do we need these? + 'webp': { + 'condition': 'QT_FEATURE_imageformatplugin AND WrapWebP_FOUND' + }, + 'xkbcommon-system': None, # another system library, just named a bit different from the rest + } + + mapping = feature_mapping.get(feature, {}) + + if mapping is None: + print(' **** Skipping features {}: masked.'.format(feature)) + return + + handled = { 'autoDetect', 'comment', 'condition', 'description', 'disable', 'emitIf', 'enable', 'label', 'output', 'purpose', 'section' } + label = mapping.get('label', data.get('label', '')) + purpose = mapping.get('purpose', data.get('purpose', data.get('description', label))) + autoDetect = map_condition(mapping.get('autoDetect', data.get('autoDetect', ''))) + condition = map_condition(mapping.get('condition', data.get('condition', ''))) + output = mapping.get('output', data.get('output', [])) + comment = mapping.get('comment', data.get('comment', '')) + section = mapping.get('section', data.get('section', '')) + enable = map_condition(mapping.get('enable', data.get('enable', ''))) + disable = map_condition(mapping.get('disable', data.get('disable', ''))) + emitIf = map_condition(mapping.get('emitIf', data.get('emitIf', ''))) + + for k in [k for k in data.keys() if k not in handled]: + print(' XXXX UNHANDLED KEY {} in feature description'.format(k)) + + if not output: + # feature that is only used in the conditions of other features + output = ["internalFeature"] + + publicFeature = False # #define QT_FEATURE_featurename in public header + privateFeature = False # #define QT_FEATURE_featurename in private header + negativeFeature = False # #define QT_NO_featurename in public header + internalFeature = False # No custom or QT_FEATURE_ defines + publicDefine = False # #define MY_CUSTOM_DEFINE in public header + + for o in output: + outputType = o + outputArgs = {} + if isinstance(o, dict): + outputType = o['type'] + outputArgs = o + + if outputType in ['varAssign', 'varAppend', 'varRemove', 'publicQtConfig', 'privateConfig', 'publicConfig']: + continue + elif outputType == 'define': + publicDefine = True + elif outputType == 'feature': + negativeFeature = True + elif outputType == 'publicFeature': + publicFeature = True + elif outputType == 'privateFeature': + privateFeature = True + elif outputType == 'internalFeature': + internalFeature = True + else: + print(' XXXX UNHANDLED OUTPUT TYPE {} in feature {}.'.format(outputType, feature)) + continue + + if not any([publicFeature, privateFeature, internalFeature, publicDefine, negativeFeature]): + print(' **** Skipping feature {}: Not relevant for C++.'.format(feature)) + return + + cxxFeature = featureName(feature) + + def writeFeature(name, publicFeature=False, privateFeature=False, labelAppend=''): + if comment: + cm_fh.write('# {}\n'.format(comment)) + + cm_fh.write('qt_feature("{}"'.format(name)) + if publicFeature: + cm_fh.write(' PUBLIC') + if privateFeature: + cm_fh.write(' PRIVATE') + cm_fh.write('\n') + + cm_fh.write(lineify('SECTION', section)) + cm_fh.write(lineify('LABEL', label + labelAppend)) + if purpose != label: + cm_fh.write(lineify('PURPOSE', purpose)) + cm_fh.write(lineify('AUTODETECT', autoDetect, quote=False)) + cm_fh.write(lineify('CONDITION', condition, quote=False)) + cm_fh.write(lineify('ENABLE', enable, quote=False)) + cm_fh.write(lineify('DISABLE', disable, quote=False)) + cm_fh.write(lineify('EMIT_IF', emitIf, quote=False)) + cm_fh.write(')\n') + + # Write qt_feature() calls before any qt_feature_definition() calls + + # Default internal feature case. + featureCalls = {} + featureCalls[cxxFeature] = {'name': cxxFeature, 'labelAppend': ''} + + # Go over all outputs to compute the number of features that have to be declared + for o in output: + outputType = o + name = cxxFeature + + # The label append is to provide a unique label for features that have more than one output + # with different names. + labelAppend = '' + + if isinstance(o, dict): + outputType = o['type'] + if 'name' in o: + name = o['name'] + labelAppend = ': {}'.format(o['name']) + + if outputType not in ['feature', 'publicFeature', 'privateFeature']: + continue + if name not in featureCalls: + featureCalls[name] = {'name': name, 'labelAppend': labelAppend} + + if outputType in ['feature', 'publicFeature']: + featureCalls[name]['publicFeature'] = True + elif outputType == 'privateFeature': + featureCalls[name]['privateFeature'] = True + + # Write the qt_feature() calls from the computed feature map + for _, args in featureCalls.items(): + writeFeature(**args) + + # Write qt_feature_definition() calls + for o in output: + outputType = o + outputArgs = {} + if isinstance(o, dict): + outputType = o['type'] + outputArgs = o + + # Map negative feature to define: + if outputType == 'feature': + outputType = 'define' + outputArgs = {'name': 'QT_NO_{}'.format(cxxFeature.upper()), + 'negative': True, + 'value': 1, + 'type': 'define'} + + if outputType != 'define': + continue + + if outputArgs.get('name') is None: + print(' XXXX DEFINE output without name in feature {}.'.format(feature)) + continue + + cm_fh.write('qt_feature_definition("{}" "{}"'.format(cxxFeature, outputArgs.get('name'))) + if outputArgs.get('negative', False): + cm_fh.write(' NEGATE') + if outputArgs.get('value') is not None: + cm_fh.write(' VALUE "{}"'.format(outputArgs.get('value'))) + cm_fh.write(')\n') + + +def processInputs(ctx, data, cm_fh): + print(' inputs:') + if 'commandline' not in data: + return + + commandLine = data['commandline'] + if "options" not in commandLine: + return + + for input in commandLine['options']: + parseInput(ctx, input, commandLine['options'][input], cm_fh) + + +def processTests(ctx, data, cm_fh): + print(' tests:') + if 'tests' not in data: + return + + for test in data['tests']: + parseTest(ctx, test, data['tests'][test], cm_fh) + + +def processFeatures(ctx, data, cm_fh): + print(' features:') + if 'features' not in data: + return + + for feature in data['features']: + parseFeature(ctx, feature, data['features'][feature], cm_fh) + + +def processLibraries(ctx, data, cm_fh): + cmake_find_packages_set = set() + print(' libraries:') + if 'libraries' not in data: + return + + for lib in data['libraries']: + parseLib(ctx, lib, data, cm_fh, cmake_find_packages_set) + + +def processSubconfigs(dir, ctx, data): + assert ctx is not None + if 'subconfigs' in data: + for subconf in data['subconfigs']: + subconfDir = os.path.join(dir, subconf) + subconfData = readJsonFromDir(subconfDir) + subconfCtx = ctx + processJson(subconfDir, subconfCtx, subconfData) + + +def processJson(dir, ctx, data): + ctx['module'] = data.get('module', 'global') + + ctx = processFiles(ctx, data) + + with open(os.path.join(dir, "configure.cmake"), 'w') as cm_fh: + cm_fh.write("\n\n#### Inputs\n\n") + + processInputs(ctx, data, cm_fh) + + cm_fh.write("\n\n#### Libraries\n\n") + + processLibraries(ctx, data, cm_fh) + + cm_fh.write("\n\n#### Tests\n\n") + + processTests(ctx, data, cm_fh) + + cm_fh.write("\n\n#### Features\n\n") + + processFeatures(ctx, data, cm_fh) + + if ctx.get('module') == 'global': + cm_fh.write('\nqt_extra_definition("QT_VERSION_STR" "\\\"${PROJECT_VERSION}\\\"" PUBLIC)\n') + cm_fh.write('qt_extra_definition("QT_VERSION_MAJOR" ${PROJECT_VERSION_MAJOR} PUBLIC)\n') + cm_fh.write('qt_extra_definition("QT_VERSION_MINOR" ${PROJECT_VERSION_MINOR} PUBLIC)\n') + cm_fh.write('qt_extra_definition("QT_VERSION_PATCH" ${PROJECT_VERSION_PATCH} PUBLIC)\n') + + # do this late: + processSubconfigs(dir, ctx, data) + + +def main(): + if len(sys.argv) != 2: + print("This scripts needs one directory to process!") + quit(1) + + dir = sys.argv[1] + + print("Processing: {}.".format(dir)) + + data = readJsonFromDir(dir) + processJson(dir, {}, data) + + +if __name__ == '__main__': + main() diff --git a/util/cmake/generate_module_map.sh b/util/cmake/generate_module_map.sh new file mode 100755 index 0000000000..1ca0bfc43c --- /dev/null +++ b/util/cmake/generate_module_map.sh @@ -0,0 +1,38 @@ +#!/usr/bin/bash +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +pro_files=$(find . -name \*.pro) + +for f in ${pro_files}; do + if grep "^load(qt_module)" "${f}" > /dev/null ; then + target=$(grep "TARGET" "${f}" | cut -d'=' -f2 | sed -e "s/\s*//g") + module=$(basename ${f}) + echo "'${module%.pro}': '${target}'," + fi +done diff --git a/util/cmake/helper.py b/util/cmake/helper.py new file mode 100644 index 0000000000..e80813a1f7 --- /dev/null +++ b/util/cmake/helper.py @@ -0,0 +1,462 @@ +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +import re +import typing + +class LibraryMapping: + def __init__(self, soName: str, + packageName: typing.Optional[str], + targetName: typing.Optional[str], *, + resultVariable: typing.Optional[str] = None, + extra: typing.List[str] = [], + appendFoundSuffix: bool = True, + emit_if: str = '') -> None: + self.soName = soName + self.packageName = packageName + self.resultVariable = resultVariable + self.appendFoundSuffix = appendFoundSuffix + self.extra = extra + self.targetName = targetName + + # if emit_if is non-empty, the generated find_package call + # for a library will be surrounded by this condition. + self.emit_if = emit_if + + def is_qt(self) -> bool: + return self.packageName == 'Qt' \ + or self.packageName == 'Qt5' \ + or self.packageName == 'Qt6' + +_qt_library_map = [ + # Qt: + LibraryMapping('accessibility_support', 'Qt6', 'Qt::AccessibilitySupport', extra = ['COMPONENTS', 'AccessibilitySupport']), + LibraryMapping('androidextras', 'Qt6', 'Qt::AndroidExtras', extra = ['COMPONENTS', 'AndroidExtras']), + LibraryMapping('animation', 'Qt6', 'Qt::3DAnimation', extra = ['COMPONENTS', '3DAnimation']), + LibraryMapping('application-lib', 'Qt6', 'Qt::AppManApplication', extra = ['COMPONENTS', 'AppManApplication']), + LibraryMapping('bluetooth', 'Qt6', 'Qt::Bluetooth', extra = ['COMPONENTS', 'Bluetooth']), + LibraryMapping('bootstrap', 'Qt6', 'Qt::Bootstrap', extra = ['COMPONENTS', 'Bootstrap']), + # bootstrap-dbus: Not needed in Qt6! + LibraryMapping('client', 'Qt6', 'Qt::WaylandClient', extra = ['COMPONENTS', 'WaylandClient']), + LibraryMapping('clipboard_support', 'Qt6', 'Qt::ClipboardSupport', extra = ['COMPONENTS', 'ClipboardSupport']), + LibraryMapping('common-lib', 'Qt6', 'Qt::AppManCommon', extra = ['COMPONENTS', 'AppManCommon']), + LibraryMapping('compositor', 'Qt6', 'Qt::WaylandCompositor', extra = ['COMPONENTS', 'WaylandCompositor']), + LibraryMapping('concurrent', 'Qt6', 'Qt::Concurrent', extra = ['COMPONENTS', 'Concurrent']), + LibraryMapping('container', 'Qt6', 'Qt::AxContainer', extra = ['COMPONENTS', 'AxContainer']), + LibraryMapping('control', 'Qt6', 'Qt::AxServer', extra = ['COMPONENTS', 'AxServer']), + LibraryMapping('core_headers', 'Qt6', 'Qt::WebEngineCore', extra = ['COMPONENTS', 'WebEngineCore']), + LibraryMapping('core', 'Qt6', 'Qt::Core', extra = ['COMPONENTS', 'Core']), + LibraryMapping('coretest', 'Qt6', 'Qt::3DCoreTest', extra = ['COMPONENTS', '3DCoreTest']), + LibraryMapping('crypto-lib', 'Qt6', 'Qt::AppManCrypto', extra = ['COMPONENTS', 'AppManCrypto']), + LibraryMapping('dbus', 'Qt6', 'Qt::DBus', extra = ['COMPONENTS', 'DBus']), + LibraryMapping('devicediscovery', 'Qt6', 'Qt::DeviceDiscoverySupport', extra = ['COMPONENTS', 'DeviceDiscoverySupport']), + LibraryMapping('devicediscovery_support', 'Qt6', 'Qt::DeviceDiscoverySupport', extra = ['COMPONENTS', 'DeviceDiscoverySupport']), + LibraryMapping('edid', 'Qt6', 'Qt::EdidSupport', extra = ['COMPONENTS', 'EdidSupport']), + LibraryMapping('edid_support', 'Qt6', 'Qt::EdidSupport', extra = ['COMPONENTS', 'EdidSupport']), + LibraryMapping('eglconvenience', 'Qt6', 'Qt::EglSupport', extra = ['COMPONENTS', 'EglSupport']), + LibraryMapping('eglfsdeviceintegration', 'Qt6', 'Qt::EglFSDeviceIntegration', extra = ['COMPONENTS', 'EglFSDeviceIntegration']), + LibraryMapping('eglfs_kms_support', 'Qt6', 'Qt::EglFsKmsSupport', extra = ['COMPONENTS', 'EglFsKmsSupport']), + LibraryMapping('egl_support', 'Qt6', 'Qt::EglSupport', extra = ['COMPONENTS', 'EglSupport']), + # enginio: Not needed in Qt6! + LibraryMapping('eventdispatchers', 'Qt6', 'Qt::EventDispatcherSupport', extra = ['COMPONENTS', 'EventDispatcherSupport']), + LibraryMapping('eventdispatcher_support', 'Qt6', 'Qt::EventDispatcherSupport', extra = ['COMPONENTS', 'EventDispatcherSupport']), + LibraryMapping('extras', 'Qt6', 'Qt::3DExtras', extra = ['COMPONENTS', '3DExtras']), + LibraryMapping('fbconvenience', 'Qt6', 'Qt::FbSupport', extra = ['COMPONENTS', 'FbSupport']), + LibraryMapping('fb_support', 'Qt6', 'Qt::FbSupport', extra = ['COMPONENTS', 'FbSupport']), + LibraryMapping('fontdatabase_support', 'Qt6', 'Qt::FontDatabaseSupport', extra = ['COMPONENTS', 'FontDatabaseSupport']), + LibraryMapping('gamepad', 'Qt6', 'Qt::Gamepad', extra = ['COMPONENTS', 'Gamepad']), + LibraryMapping('global', 'Qt6', 'Qt::Core', extra = ['COMPONENTS', 'Core']), # manually added special case + LibraryMapping('glx_support', 'Qt6', 'Qt::GlxSupport', extra = ['COMPONENTS', 'GlxSupport']), + LibraryMapping('graphics_support', 'Qt6', 'Qt::GraphicsSupport', extra = ['COMPONENTS', 'GraphicsSupport']), + LibraryMapping('gsttools', 'Qt6', 'Qt::MultimediaGstTools', extra = ['COMPONENTS', 'MultimediaGstTools']), + LibraryMapping('gui', 'Qt6', 'Qt::Gui', extra = ['COMPONENTS', 'Gui']), + LibraryMapping('help', 'Qt6', 'Qt::Help', extra = ['COMPONENTS', 'Help']), + LibraryMapping('hunspellinputmethod', 'Qt6', 'Qt::HunspellInputMethod', extra = ['COMPONENTS', 'HunspellInputMethod']), + LibraryMapping('input', 'Qt6', 'Qt::InputSupport', extra = ['COMPONENTS', 'InputSupport']), + LibraryMapping('input_support', 'Qt6', 'Qt::InputSupport', extra = ['COMPONENTS', 'InputSupport']), + LibraryMapping('installer-lib', 'Qt6', 'Qt::AppManInstaller', extra = ['COMPONENTS', 'AppManInstaller']), + LibraryMapping('kmsconvenience', 'Qt6', 'Qt::KmsSupport', extra = ['COMPONENTS', 'KmsSupport']), + LibraryMapping('kms_support', 'Qt6', 'Qt::KmsSupport', extra = ['COMPONENTS', 'KmsSupport']), + LibraryMapping('launcher-lib', 'Qt6', 'Qt::AppManLauncher', extra = ['COMPONENTS', 'AppManLauncher']), + LibraryMapping('lib', 'Qt6', 'Qt::Designer', extra = ['COMPONENTS', 'Designer']), + LibraryMapping('linuxaccessibility_support', 'Qt6', 'Qt::LinuxAccessibilitySupport', extra = ['COMPONENTS', 'LinuxAccessibilitySupport']), + LibraryMapping('location', 'Qt6', 'Qt::Location', extra = ['COMPONENTS', 'Location']), + LibraryMapping('logic', 'Qt6', 'Qt::3DLogic', extra = ['COMPONENTS', '3DLogic']), + LibraryMapping('macextras', 'Qt6', 'Qt::MacExtras', extra = ['COMPONENTS', 'MacExtras']), + LibraryMapping('main-lib', 'Qt6', 'Qt::AppManMain', extra = ['COMPONENTS', 'AppManMain']), + LibraryMapping('manager-lib', 'Qt6', 'Qt::AppManManager', extra = ['COMPONENTS', 'AppManManager']), + LibraryMapping('monitor-lib', 'Qt6', 'Qt::AppManMonitor', extra = ['COMPONENTS', 'AppManMonitor']), + LibraryMapping('multimedia', 'Qt6', 'Qt::Multimedia', extra = ['COMPONENTS', 'Multimedia']), + LibraryMapping('multimediawidgets', 'Qt6', 'Qt::MultimediaWidgets', extra = ['COMPONENTS', 'MultimediaWidgets']), + LibraryMapping('network', 'Qt6', 'Qt::Network', extra = ['COMPONENTS', 'Network']), + LibraryMapping('networkauth', 'Qt6', 'Qt::NetworkAuth', extra = ['COMPONENTS', 'NetworkAuth']), + LibraryMapping('nfc', 'Qt6', 'Qt::Nfc', extra = ['COMPONENTS', 'Nfc']), + LibraryMapping('oauth', 'Qt6', 'Qt::NetworkAuth', extra = ['COMPONENTS', 'NetworkAuth']), + LibraryMapping('openglextensions', 'Qt6', 'Qt::OpenGLExtensions', extra = ['COMPONENTS', 'OpenGLExtensions']), + LibraryMapping('opengl', 'Qt6', 'Qt::OpenGL', extra = ['COMPONENTS', 'OpenGL']), + LibraryMapping('package-lib', 'Qt6', 'Qt::AppManPackage', extra = ['COMPONENTS', 'AppManPackage']), + LibraryMapping('packetprotocol', 'Qt6', 'Qt::PacketProtocol', extra = ['COMPONENTS', 'PacketProtocol']), + LibraryMapping('particles', 'Qt6', 'Qt::QuickParticles', extra = ['COMPONENTS', 'QuickParticles']), + LibraryMapping('platformcompositor', 'Qt6', 'Qt::PlatformCompositorSupport', extra = ['COMPONENTS', 'PlatformCompositorSupport']), + LibraryMapping('platformcompositor_support', 'Qt6', 'Qt::PlatformCompositorSupport', extra = ['COMPONENTS', 'PlatformCompositorSupport']), + LibraryMapping('plugin-interfaces', 'Qt6', 'Qt::AppManPluginInterfaces', extra = ['COMPONENTS', 'AppManPluginInterfaces']), + LibraryMapping('positioning', 'Qt6', 'Qt::Positioning', extra = ['COMPONENTS', 'Positioning']), + LibraryMapping('positioningquick', 'Qt6', 'Qt::PositioningQuick', extra = ['COMPONENTS', 'PositioningQuick']), + LibraryMapping('printsupport', 'Qt6', 'Qt::PrintSupport', extra = ['COMPONENTS', 'PrintSupport']), + LibraryMapping('purchasing', 'Qt6', 'Qt::Purchasing', extra = ['COMPONENTS', 'Purchasing']), + LibraryMapping('qmldebug', 'Qt6', 'Qt::QmlDebug', extra = ['COMPONENTS', 'QmlDebug']), + LibraryMapping('qmldevtools', 'Qt6', 'Qt::QmlDevTools', extra = ['COMPONENTS', 'QmlDevTools']), + LibraryMapping('qml', 'Qt6', 'Qt::Qml', extra = ['COMPONENTS', 'Qml']), + LibraryMapping('qmlmodels', 'Qt6', 'Qt::QmlModels', extra = ['COMPONENTS', 'QmlModels']), + LibraryMapping('qmltest', 'Qt6', 'Qt::QuickTest', extra = ['COMPONENTS', 'QuickTest']), + LibraryMapping('qtmultimediaquicktools', 'Qt6', 'Qt::MultimediaQuick', extra = ['COMPONENTS', 'MultimediaQuick']), + LibraryMapping('quick3danimation', 'Qt6', 'Qt::3DQuickAnimation', extra = ['COMPONENTS', '3DQuickAnimation']), + LibraryMapping('quick3dextras', 'Qt6', 'Qt::3DQuickExtras', extra = ['COMPONENTS', '3DQuickExtras']), + LibraryMapping('quick3dinput', 'Qt6', 'Qt::3DQuickInput', extra = ['COMPONENTS', '3DQuickInput']), + LibraryMapping('quick3d', 'Qt6', 'Qt::3DQuick', extra = ['COMPONENTS', '3DQuick']), + LibraryMapping('quick3drender', 'Qt6', 'Qt::3DQuickRender', extra = ['COMPONENTS', '3DQuickRender']), + LibraryMapping('quick3dscene2d', 'Qt6', 'Qt::3DQuickScene2D', extra = ['COMPONENTS', '3DQuickScene2D']), + LibraryMapping('quickcontrols2', 'Qt6', 'Qt::QuickControls2', extra = ['COMPONENTS', 'QuickControls2']), + LibraryMapping('quick', 'Qt6', 'Qt::Quick', extra = ['COMPONENTS', 'Quick']), + LibraryMapping('quickshapes', 'Qt6', 'Qt::QuickShapes', extra = ['COMPONENTS', 'QuickShapes']), + LibraryMapping('quicktemplates2', 'Qt6', 'Qt::QuickTemplates2', extra = ['COMPONENTS', 'QuickTemplates2']), + LibraryMapping('quickwidgets', 'Qt6', 'Qt::QuickWidgets', extra = ['COMPONENTS', 'QuickWidgets']), + LibraryMapping('render', 'Qt6', 'Qt::3DRender', extra = ['COMPONENTS', '3DRender']), + LibraryMapping('script', 'Qt6', 'Qt::Script', extra = ['COMPONENTS', 'Script']), + LibraryMapping('scripttools', 'Qt6', 'Qt::ScriptTools', extra = ['COMPONENTS', 'ScriptTools']), + LibraryMapping('sensors', 'Qt6', 'Qt::Sensors', extra = ['COMPONENTS', 'Sensors']), + LibraryMapping('serialport', 'Qt6', 'Qt::SerialPort', extra = ['COMPONENTS', 'SerialPort']), + LibraryMapping('services', 'Qt6', 'Qt::ServiceSupport', extra = ['COMPONENTS', 'ServiceSupport']), + LibraryMapping('service_support', 'Qt6', 'Qt::ServiceSupport', extra = ['COMPONENTS', 'ServiceSupport']), + LibraryMapping('sql', 'Qt6', 'Qt::Sql', extra = ['COMPONENTS', 'Sql']), + LibraryMapping('svg', 'Qt6', 'Qt::Svg', extra = ['COMPONENTS', 'Svg']), + LibraryMapping('testlib', 'Qt6', 'Qt::Test', extra = ['COMPONENTS', 'Test']), + LibraryMapping('theme_support', 'Qt6', 'Qt::ThemeSupport', extra = ['COMPONENTS', 'ThemeSupport']), + LibraryMapping('tts', 'Qt6', 'Qt::TextToSpeech', extra = ['COMPONENTS', 'TextToSpeech']), + LibraryMapping('uiplugin', 'Qt6', 'Qt::UiPlugin', extra = ['COMPONENTS', 'UiPlugin']), + LibraryMapping('uitools', 'Qt6', 'Qt::UiTools', extra = ['COMPONENTS', 'UiTools']), + LibraryMapping('virtualkeyboard', 'Qt6', 'Qt::VirtualKeyboard', extra = ['COMPONENTS', 'VirtualKeyboard']), + LibraryMapping('vulkan_support', 'Qt6', 'Qt::VulkanSupport', extra = ['COMPONENTS', 'VulkanSupport']), + LibraryMapping('webchannel', 'Qt6', 'Qt::WebChannel', extra = ['COMPONENTS', 'WebChannel']), + LibraryMapping('webengine', 'Qt6', 'Qt::WebEngine', extra = ['COMPONENTS', 'WebEngine']), + LibraryMapping('webenginewidgets', 'Qt6', 'Qt::WebEngineWidgets', extra = ['COMPONENTS', 'WebEngineWidgets']), + LibraryMapping('websockets', 'Qt6', 'Qt::WebSockets', extra = ['COMPONENTS', 'WebSockets']), + LibraryMapping('webview', 'Qt6', 'Qt::WebView', extra = ['COMPONENTS', 'WebView']), + LibraryMapping('widgets', 'Qt6', 'Qt::Widgets', extra = ['COMPONENTS', 'Widgets']), + LibraryMapping('window-lib', 'Qt6', 'Qt::AppManWindow', extra = ['COMPONENTS', 'AppManWindow']), + LibraryMapping('windowsuiautomation_support', 'Qt6', 'Qt::WindowsUIAutomationSupport', extra = ['COMPONENTS', 'WindowsUIAutomationSupport']), + LibraryMapping('winextras', 'Qt6', 'Qt::WinExtras', extra = ['COMPONENTS', 'WinExtras']), + LibraryMapping('x11extras', 'Qt6', 'Qt::X11Extras', extra = ['COMPONENTS', 'X11Extras']), + LibraryMapping('xcb_qpa_lib', 'Qt6', 'Qt::XcbQpa', extra = ['COMPONENTS', 'XcbQpa']), + LibraryMapping('xkbcommon_support', 'Qt6', 'Qt::XkbCommonSupport', extra = ['COMPONENTS', 'XkbCommonSupport']), + LibraryMapping('xmlpatterns', 'Qt6', 'Qt::XmlPatterns', extra = ['COMPONENTS', 'XmlPatterns']), + LibraryMapping('xml', 'Qt6', 'Qt::Xml', extra = ['COMPONENTS', 'Xml']), + # qtzlib: No longer supported. +] + +# Note that the library map is adjusted dynamically further down. +_library_map = [ + # 3rd party: + LibraryMapping('atspi', 'ATSPI2', 'PkgConfig::ATSPI2'), + LibraryMapping('corewlan', None, None), + LibraryMapping('cups', 'Cups', 'Cups::Cups'), + LibraryMapping('dbus', 'WrapDBus1', 'dbus-1', resultVariable="DBus1"), + LibraryMapping('doubleconversion', None, None), + LibraryMapping('drm', 'Libdrm', 'Libdrm::Libdrm'), + LibraryMapping('egl', 'EGL', 'EGL::EGL'), + LibraryMapping('fontconfig', 'Fontconfig', 'Fontconfig::Fontconfig', resultVariable="FONTCONFIG"), + LibraryMapping('freetype', 'WrapFreetype', 'WrapFreetype::WrapFreetype', extra=['REQUIRED']), + LibraryMapping('gbm', 'gbm', 'gbm::gbm'), + LibraryMapping('glib', 'GLIB2', 'GLIB2::GLIB2'), + LibraryMapping('gnu_iconv', None, None), + LibraryMapping('gtk3', 'GTK3', 'PkgConfig::GTK3'), + LibraryMapping('harfbuzz', 'harfbuzz', 'harfbuzz::harfbuzz'), + LibraryMapping('host_dbus', None, None), + LibraryMapping('icu', 'ICU', 'ICU::i18n ICU::uc ICU::data', extra=['COMPONENTS', 'i18n', 'uc', 'data']), + LibraryMapping('journald', 'Libsystemd', 'PkgConfig::Libsystemd'), + LibraryMapping('jpeg', 'JPEG', 'JPEG::JPEG'), # see also libjpeg + LibraryMapping('libatomic', 'Atomic', 'Atomic'), + LibraryMapping('libdl', None, '${CMAKE_DL_LIBS}'), + LibraryMapping('libinput', 'Libinput', 'Libinput::Libinput'), + LibraryMapping('libjpeg', 'JPEG', 'JPEG::JPEG'), # see also jpeg + LibraryMapping('libpng', 'PNG', 'PNG::PNG'), + LibraryMapping('libproxy', 'Libproxy', 'PkgConfig::Libproxy'), + LibraryMapping('librt', 'WrapRt','WrapRt'), + LibraryMapping('libudev', 'Libudev', 'PkgConfig::Libudev'), + LibraryMapping('lttng-ust', 'LTTngUST', 'LTTng::UST', resultVariable='LTTNGUST'), + LibraryMapping('mtdev', 'Mtdev', 'PkgConfig::Mtdev'), + LibraryMapping('odbc', 'ODBC', 'ODBC::ODBC'), + LibraryMapping('opengl_es2', 'GLESv2', 'GLESv2::GLESv2'), + LibraryMapping('opengl', 'OpenGL', 'OpenGL::GL', resultVariable='OpenGL_OpenGL'), + LibraryMapping('openssl_headers', 'OpenSSL', 'OpenSSL::SSL_nolink', resultVariable='OPENSSL_INCLUDE_DIR', appendFoundSuffix=False), + LibraryMapping('openssl', 'OpenSSL', 'OpenSSL::SSL'), + LibraryMapping('pcre2', 'WrapPCRE2', 'WrapPCRE2::WrapPCRE2', extra = ['REQUIRED']), + LibraryMapping('posix_iconv', None, None), + LibraryMapping('pps', 'PPS', 'PPS::PPS'), + LibraryMapping('psql', 'PostgreSQL', 'PostgreSQL::PostgreSQL'), + LibraryMapping('slog2', 'Slog2', 'Slog2::Slog2'), + LibraryMapping('sqlite2', None, None), # No more sqlite2 support in Qt6! + LibraryMapping('sqlite3', 'SQLite3', 'SQLite::SQLite3'), + LibraryMapping('sun_iconv', None, None), + LibraryMapping('tslib', 'Tslib', 'PkgConfig::Tslib'), + LibraryMapping('udev', 'Libudev', 'PkgConfig::Libudev'), + LibraryMapping('udev', 'Libudev', 'PkgConfig::Libudev'), # see also libudev! + LibraryMapping('vulkan', 'Vulkan', 'Vulkan::Vulkan'), + LibraryMapping('wayland_server', 'Wayland', 'Wayland::Server'), + LibraryMapping('x11sm', 'X11', '${X11_SM_LIB} ${X11_ICE_LIB}', resultVariable="X11_SM"), + LibraryMapping('xcb', 'XCB', 'XCB::XCB', extra = ['1.9']), + LibraryMapping('xcb_glx', 'XCB', 'XCB::GLX', extra = ['COMPONENTS', 'GLX'], resultVariable='XCB_GLX'), + LibraryMapping('xcb_icccm', 'XCB', 'XCB::ICCCM', extra = ['COMPONENTS', 'ICCCM'], resultVariable='XCB_ICCCM'), + LibraryMapping('xcb_image', 'XCB', 'XCB::IMAGE', extra = ['COMPONENTS', 'IMAGE'], resultVariable='XCB_IMAGE'), + LibraryMapping('xcb_keysyms', 'XCB', 'XCB::KEYSYMS', extra = ['COMPONENTS', 'KEYSYMS'], resultVariable='XCB_KEYSYMS'), + LibraryMapping('xcb_randr', 'XCB', 'XCB::RANDR', extra = ['COMPONENTS', 'RANDR'], resultVariable='XCB_RANDR'), + LibraryMapping('xcb_render', 'XCB', 'XCB::RENDER', extra = ['COMPONENTS', 'RENDER'], resultVariable='XCB_RENDER'), + LibraryMapping('xcb_renderutil', 'XCB', 'XCB::RENDERUTIL', extra = ['COMPONENTS', 'RENDERUTIL'], resultVariable='XCB_RENDERUTIL'), + LibraryMapping('xcb_shape', 'XCB', 'XCB::SHAPE', extra = ['COMPONENTS', 'SHAPE'], resultVariable='XCB_SHAPE'), + LibraryMapping('xcb_shm', 'XCB', 'XCB::SHM', extra = ['COMPONENTS', 'SHM'], resultVariable='XCB_SHM'), + LibraryMapping('xcb_sync', 'XCB', 'XCB::SYNC', extra = ['COMPONENTS', 'SYNC'], resultVariable='XCB_SYNC'), + LibraryMapping('xcb_xfixes', 'XCB', 'XCB::XFIXES', extra = ['COMPONENTS', 'XFIXES'], resultVariable='XCB_XFIXES'), + LibraryMapping('xcb_xinerama', 'XCB', 'XCB::XINERAMA', extra = ['COMPONENTS', 'XINERAMA'], resultVariable='XCB_XINERAMA'), + LibraryMapping('xcb_xinput', 'XCB', 'XCB::XINPUT', extra = ['COMPONENTS', 'XINPUT'], resultVariable='XCB_XINPUT'), + LibraryMapping('xcb_xkb', 'XCB', 'XCB::XKB', extra = ['COMPONENTS', 'XKB'], resultVariable='XCB_XKB'), + LibraryMapping('xcb_xlib', 'X11_XCB', 'X11::XCB'), + LibraryMapping('xkbcommon_evdev', 'XKB', 'XKB::XKB', extra = ['0.4.1']), # see also xkbcommon + LibraryMapping('xkbcommon_x11', 'XKB', 'XKB::XKB', extra = ['0.4.1']), # see also xkbcommon + LibraryMapping('xkbcommon', 'XKB', 'XKB::XKB', extra = ['0.4.1']), + LibraryMapping('xlib', 'X11', 'X11::XCB'), # FIXME: Is this correct? + LibraryMapping('xrender', 'XRender', 'PkgConfig::XRender'), + LibraryMapping('zlib', 'ZLIB', 'ZLIB::ZLIB', extra=['REQUIRED']), + LibraryMapping('zstd', 'ZSTD', 'ZSTD::ZSTD'), + LibraryMapping('tiff', 'TIFF', 'TIFF::TIFF'), + LibraryMapping('webp', 'WrapWebP', 'WrapWebP::WrapWebP'), + LibraryMapping('jasper', 'WrapJasper', 'WrapJasper::WrapJasper'), +] + + +def _adjust_library_map(): + # Assign a Linux condition on all x and wayland related packages. + # We don't want to get pages of package not found messages on + # Windows and macOS, and this also improves configure time on + # those platforms. + linux_package_prefixes = ['xcb', 'x11', 'xkb', 'xrender', 'xlib', 'wayland'] + for i, _ in enumerate(_library_map): + if any([_library_map[i].soName.startswith(p) for p in linux_package_prefixes]): + _library_map[i].emit_if = 'config.linux' + + +_adjust_library_map() + + +def find_3rd_party_library_mapping(soName: str) -> typing.Optional[LibraryMapping]: + for i in _library_map: + if i.soName == soName: + return i + return None + + +def find_qt_library_mapping(soName: str) -> typing.Optional[LibraryMapping]: + for i in _qt_library_map: + if i.soName == soName: + return i + return None + + +def find_library_info_for_target(targetName: str) -> typing.Optional[LibraryMapping]: + qt_target = targetName + if targetName.endswith('Private'): + qt_target = qt_target[:-7] + + for i in _qt_library_map: + if i.targetName == qt_target: + return i + + for i in _library_map: + if i.targetName == targetName: + return i + + return None + + +def featureName(input: str) -> str: + replacement_char = '_' + if input.startswith('c++'): + replacement_char = 'x' + return re.sub(r'[^a-zA-Z0-9_]', replacement_char, input) + + +def map_qt_library(lib: str) -> str: + private = False + if lib.endswith('-private'): + private = True + lib = lib[:-8] + mapped = find_qt_library_mapping(lib) + qt_name = lib + if mapped: + assert mapped.targetName # Qt libs must have a target name set + qt_name = mapped.targetName + if private: + qt_name += 'Private' + return qt_name + + +platform_mapping = { + 'win32': 'WIN32', + 'win': 'WIN32', + 'unix': 'UNIX', + 'darwin': 'APPLE', + 'linux': 'LINUX', + 'integrity': 'INTEGRITY', + 'qnx': 'QNX', + 'vxworks': 'VXWORKS', + 'hpux': 'HPUX', + 'nacl': 'NACL', + 'android': 'ANDROID', + 'android-embedded': 'ANDROID_EMBEDDED', + 'uikit': 'APPLE_UIKIT', + 'tvos': 'APPLE_TVOS', + 'watchos': 'APPLE_WATCHOS', + 'winrt': 'WINRT', + 'wasm': 'WASM', + 'msvc': 'MSVC', + 'clang': 'CLANG', + 'gcc': 'GCC', + 'icc': 'ICC', + 'intel_icc': 'ICC', + 'osx': 'APPLE_OSX', + 'ios': 'APPLE_IOS', + 'freebsd': 'FREEBSD', + 'openbsd': 'OPENBSD', + 'netbsd': 'NETBSD', + 'haiku': 'HAIKU', + 'netbsd': 'NETBSD', + 'mac': 'APPLE_OSX', + 'macx': 'APPLE_OSX', + 'macos': 'APPLE_OSX', + 'macx-icc': '(APPLE_OSX AND ICC)', +} + + +def map_platform(platform: str) -> str: + """ Return the qmake platform as cmake platform or the unchanged string. """ + return platform_mapping.get(platform, platform) + + +def is_known_3rd_party_library(lib: str) -> bool: + if lib.endswith('/nolink') or lib.endswith('_nolink'): + lib = lib[:-7] + mapping = find_3rd_party_library_mapping(lib) + + return mapping is not None + + +def map_3rd_party_library(lib: str) -> str: + libpostfix = '' + if lib.endswith('/nolink'): + lib = lib[:-7] + libpostfix = '_nolink' + mapping = find_3rd_party_library_mapping(lib) + if not mapping or not mapping.targetName: + return lib + return mapping.targetName + libpostfix + + +def generate_find_package_info(lib: LibraryMapping, + use_qt_find_package: bool=True, *, + indent: int = 0, + emit_if: str = '') -> str: + isRequired = False + + extra = lib.extra.copy() + + if "REQUIRED" in extra and use_qt_find_package: + isRequired = True + extra.remove("REQUIRED") + + cmake_target_name = lib.targetName + assert(cmake_target_name); + + # _nolink or not does not matter at this point: + if cmake_target_name.endswith('_nolink') or cmake_target_name.endswith('/nolink'): + cmake_target_name = cmake_target_name[:-7] + + if cmake_target_name and use_qt_find_package: + extra += ['PROVIDED_TARGETS', cmake_target_name] + + result = '' + one_ind = ' ' + ind = one_ind * indent + + if use_qt_find_package: + if extra: + result = '{}qt_find_package({} {})\n'.format(ind, lib.packageName, ' '.join(extra)) + else: + result = '{}qt_find_package({})\n'.format(ind, lib.packageName) + + if isRequired: + result += '{}set_package_properties({} PROPERTIES TYPE REQUIRED)\n'.format(ind, lib.packageName) + else: + if extra: + result = '{}find_package({} {})\n'.format(ind, lib.packageName, ' '.join(extra)) + else: + result = '{}find_package({})\n'.format(ind, lib.packageName) + + # If a package should be found only in certain conditions, wrap + # the find_package call within that condition. + if emit_if: + result = "if(({emit_if}) OR QT_FIND_ALL_PACKAGES_ALWAYS)\n" \ + "{ind}{result}endif()\n".format(emit_if=emit_if, + result=result, + ind=one_ind) + + return result + + +def _set_up_py_parsing_nicer_debug_output(pp): + indent = -1 + + def increase_indent(fn): + def wrapper_function(*args): + nonlocal indent + indent += 1 + print("> " * indent, end="") + return fn(*args) + + return wrapper_function + + def decrease_indent(fn): + def wrapper_function(*args): + nonlocal indent + print("> " * indent, end="") + indent -= 1 + return fn(*args) + + return wrapper_function + + pp._defaultStartDebugAction = increase_indent(pp._defaultStartDebugAction) + pp._defaultSuccessDebugAction = decrease_indent(pp._defaultSuccessDebugAction) + pp._defaultExceptionDebugAction = decrease_indent(pp._defaultExceptionDebugAction) diff --git a/util/cmake/json_parser.py b/util/cmake/json_parser.py new file mode 100644 index 0000000000..6ead008f08 --- /dev/null +++ b/util/cmake/json_parser.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +import pyparsing as pp +import json +import re +from helper import _set_up_py_parsing_nicer_debug_output +_set_up_py_parsing_nicer_debug_output(pp) + + +class QMakeSpecificJSONParser: + def __init__(self, *, debug: bool = False) -> None: + self.debug = debug + self.grammar = self.create_py_parsing_grammar() + + def create_py_parsing_grammar(self): + # Keep around all whitespace. + pp.ParserElement.setDefaultWhitespaceChars('') + + def add_element(name: str, value: pp.ParserElement): + nonlocal self + if self.debug: + value.setName(name) + value.setDebug() + return value + + # Our grammar is pretty simple. We want to remove all newlines + # inside quoted strings, to make the quoted strings JSON + # compliant. So our grammar should skip to the first quote while + # keeping everything before it as-is, process the quoted string + # skip to the next quote, and repeat that until the end of the + # file. + + EOF = add_element('EOF', pp.StringEnd()) + SkipToQuote = add_element('SkipToQuote', pp.SkipTo('"')) + SkipToEOF = add_element('SkipToEOF', pp.SkipTo(EOF)) + + def remove_newlines_and_whitespace_in_quoted_string(tokens): + first_string = tokens[0] + replaced_string = re.sub(r'\n[ ]*', ' ', first_string) + return replaced_string + + QuotedString = add_element('QuotedString', pp.QuotedString(quoteChar='"', + multiline=True, + unquoteResults=False)) + QuotedString.setParseAction(remove_newlines_and_whitespace_in_quoted_string) + + QuotedTerm = add_element('QuotedTerm', pp.Optional(SkipToQuote) + QuotedString) + Grammar = add_element('Grammar', pp.OneOrMore(QuotedTerm) + SkipToEOF) + + return Grammar + + def parse_file_using_py_parsing(self, file: str): + print('Pre processing "{}" using py parsing to remove incorrect newlines.'.format(file)) + try: + with open(file, 'r') as file_fd: + contents = file_fd.read() + + parser_result = self.grammar.parseString(contents, parseAll=True) + token_list = parser_result.asList() + joined_string = ''.join(token_list) + + return joined_string + except pp.ParseException as pe: + print(pe.line) + print(' '*(pe.col-1) + '^') + print(pe) + raise pe + + def parse(self, file: str): + pre_processed_string = self.parse_file_using_py_parsing(file) + print('Parsing "{}" using json.loads().'.format(file)) + json_parsed = json.loads(pre_processed_string) + return json_parsed diff --git a/util/cmake/pro2cmake.py b/util/cmake/pro2cmake.py new file mode 100755 index 0000000000..28724b1fc0 --- /dev/null +++ b/util/cmake/pro2cmake.py @@ -0,0 +1,1903 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + + +from __future__ import annotations + +from argparse import ArgumentParser +import copy +import xml.etree.ElementTree as ET +from itertools import chain +import os.path +import re +import io +import typing + +from sympy.logic import (simplify_logic, And, Or, Not,) +import pyparsing as pp +from helper import _set_up_py_parsing_nicer_debug_output +_set_up_py_parsing_nicer_debug_output(pp) + +from helper import map_qt_library, map_3rd_party_library, is_known_3rd_party_library, \ + featureName, map_platform, find_library_info_for_target, generate_find_package_info, \ + LibraryMapping + +from shutil import copyfile +from special_case_helper import SpecialCaseHandler + + +def _parse_commandline(): + parser = ArgumentParser(description='Generate CMakeLists.txt files from .' + 'pro files.') + parser.add_argument('--debug', dest='debug', action='store_true', + help='Turn on all debug output') + parser.add_argument('--debug-parser', dest='debug_parser', + action='store_true', + help='Print debug output from qmake parser.') + parser.add_argument('--debug-parse-result', dest='debug_parse_result', + action='store_true', + help='Dump the qmake parser result.') + parser.add_argument('--debug-parse-dictionary', + dest='debug_parse_dictionary', action='store_true', + help='Dump the qmake parser result as dictionary.') + parser.add_argument('--debug-pro-structure', dest='debug_pro_structure', + action='store_true', + help='Dump the structure of the qmake .pro-file.') + parser.add_argument('--debug-full-pro-structure', + dest='debug_full_pro_structure', action='store_true', + help='Dump the full structure of the qmake .pro-file ' + '(with includes).') + parser.add_argument('--debug-special-case-preservation', + dest='debug_special_case_preservation', action='store_true', + help='Show all git commands and file copies.') + + parser.add_argument('--is-example', action='store_true', + dest="is_example", + help='Treat the input .pro file as an example.') + parser.add_argument('-s', '--skip-special-case-preservation', + dest='skip_special_case_preservation', action='store_true', + help='Skips behavior to reapply ' + 'special case modifications (requires git in PATH)') + parser.add_argument('-k', '--keep-temporary-files', + dest='keep_temporary_files', action='store_true', + help='Don\'t automatically remove CMakeLists.gen.txt and other ' + 'intermediate files.') + + parser.add_argument('files', metavar='<.pro/.pri file>', type=str, + nargs='+', help='The .pro/.pri file to process') + + return parser.parse_args() + + +def process_qrc_file(target: str, filepath: str, base_dir: str = '') -> str: + assert(target) + resource_name = os.path.splitext(os.path.basename(filepath))[0] + base_dir = os.path.join('' if base_dir == '.' else base_dir, os.path.dirname(filepath)) + + tree = ET.parse(filepath) + root = tree.getroot() + assert(root.tag == 'RCC') + + output = '' + + resource_count = 0 + for resource in root: + assert(resource.tag == 'qresource') + lang = resource.get('lang', '') + prefix = resource.get('prefix', '') + + full_resource_name = resource_name + (str(resource_count) if resource_count > 0 else '') + + files: typing.Dict[str, str] = {} + for file in resource: + path = file.text + assert path + + # Get alias: + alias = file.get('alias', '') + files[path] = alias + + sorted_files = sorted(files.keys()) + + assert(sorted_files) + + for source in sorted_files: + alias = files[source] + if alias: + full_source = os.path.join(base_dir, source) + output += 'set_source_files_properties("{}"\n' \ + ' PROPERTIES alias "{}")\n'.format(full_source, alias) + + params = '' + if lang: + params += ' LANG "{}"'.format(lang) + if prefix: + params += ' PREFIX "{}"'.format(prefix) + if base_dir: + params += ' BASE "{}"'.format(base_dir) + output += 'add_qt_resource({} "{}"{} FILES\n {})\n'.format(target, full_resource_name, + params, + '\n '.join(sorted_files)) + + resource_count += 1 + + return output + + +def fixup_linecontinuation(contents: str) -> str: + # Remove all line continuations, aka a backslash followed by + # a newline character with an arbitrary amount of whitespace + # between the backslash and the newline. + # This greatly simplifies the qmake parsing grammar. + contents = re.sub(r'([^\t ])\\[ \t]*\n', '\\1 ', contents) + contents = re.sub(r'\\[ \t]*\n', '', contents) + return contents + + +def fixup_comments(contents: str) -> str: + # Get rid of completely commented out lines. + # So any line which starts with a '#' char and ends with a new line + # will be replaced by a single new line. + # + # This is needed because qmake syntax is weird. In a multi line + # assignment (separated by backslashes and newlines aka + # # \\\n ), if any of the lines are completely commented out, in + # principle the assignment should fail. + # + # It should fail because you would have a new line separating + # the previous value from the next value, and the next value would + # not be interpreted as a value, but as a new token / operation. + # qmake is lenient though, and accepts that, so we need to take + # care of it as well, as if the commented line didn't exist in the + # first place. + + contents = re.sub(r'\n#[^\n]*?\n', '\n', contents, re.DOTALL) + return contents + + +def spaces(indent: int) -> str: + return ' ' * indent + + +def trim_leading_dot(file: str) -> str: + while file.startswith('./'): + file = file[2:] + return file + + +def map_to_file(f: str, scope: Scope, *, is_include: bool = False) -> str: + assert('$$' not in f) + + if f.startswith('${'): # Some cmake variable is prepended + return f + + base_dir = scope.currentdir if is_include else scope.basedir + f = os.path.join(base_dir, f) + + return trim_leading_dot(f) + + +def handle_vpath(source: str, base_dir: str, vpath: typing.List[str]) -> str: + assert('$$' not in source) + + if not source: + return '' + + if not vpath: + return source + + if os.path.exists(os.path.join(base_dir, source)): + return source + + variable_pattern = re.compile(r'\$\{[A-Za-z0-9_]+\}') + match = re.match(variable_pattern, source) + if match: + # a complex, variable based path, skipping validation + # or resolving + return source + + for v in vpath: + fullpath = os.path.join(v, source) + if os.path.exists(fullpath): + return trim_leading_dot(os.path.relpath(fullpath, base_dir)) + + print(' XXXX: Source {}: Not found.'.format(source)) + return '{}-NOTFOUND'.format(source) + + +class Operation: + def __init__(self, value: typing.Union[typing.List[str], str]): + if isinstance(value, list): + self._value = value + else: + self._value = [str(value), ] + + def process(self, key: str, input: typing.List[str], + transformer: typing.Callable[[typing.List[str]], typing.List[str]]) -> typing.List[str]: + assert(False) + + def __repr__(self): + assert(False) + + def _dump(self): + if not self._value: + return '<NOTHING>' + + if not isinstance(self._value, list): + return '<NOT A LIST>' + + result = [] + for i in self._value: + if not i: + result.append('<NONE>') + else: + result.append(str(i)) + return '"' + '", "'.join(result) + '"' + + +class AddOperation(Operation): + def process(self, key: str, input: typing.List[str], + transformer: typing.Callable[[typing.List[str]], typing.List[str]]) -> typing.List[str]: + return input + transformer(self._value) + + def __repr__(self): + return '+({})'.format(self._dump()) + + +class UniqueAddOperation(Operation): + def process(self, key: str, input: typing.List[str], + transformer: typing.Callable[[typing.List[str]], typing.List[str]]) -> typing.List[str]: + result = input + for v in transformer(self._value): + if v not in result: + result.append(v) + return result + + def __repr__(self): + return '*({})'.format(self._dump()) + + +class SetOperation(Operation): + def process(self, key: str, input: typing.List[str], + transformer: typing.Callable[[typing.List[str]], typing.List[str]]) -> typing.List[str]: + values = [] # typing.List[str] + for v in self._value: + if v != '$${}'.format(key): + values.append(v) + else: + values += input + + if transformer: + return list(transformer(values)) + else: + return values + + def __repr__(self): + return '=({})'.format(self._dump()) + + +class RemoveOperation(Operation): + def __init__(self, value): + super().__init__(value) + + def process(self, key: str, input: typing.List[str], + transformer: typing.Callable[[typing.List[str]], typing.List[str]]) -> typing.List[str]: + input_set = set(input) + value_set = set(self._value) + result = [] + + # Add everything that is not going to get removed: + for v in input: + if v not in value_set: + result += [v,] + + # Add everything else with removal marker: + for v in transformer(self._value): + if v not in input_set: + result += ['-{}'.format(v), ] + + return result + + def __repr__(self): + return '-({})'.format(self._dump()) + + +class Scope(object): + + SCOPE_ID: int = 1 + + def __init__(self, *, + parent_scope: typing.Optional[Scope], + file: typing.Optional[str] = None, condition: str = '', + base_dir: str = '', + operations: typing.Mapping[str, typing.List[Operation]] = { + 'QT_SOURCE_TREE': [SetOperation(['${PROJECT_SOURCE_DIR}'])], + 'QT_BUILD_TREE': [SetOperation(['${PROJECT_BUILD_DIR}'])], + }) -> None: + if parent_scope: + parent_scope._add_child(self) + else: + self._parent = None # type: typing.Optional[Scope] + + self._basedir = base_dir + if file: + self._currentdir = os.path.dirname(file) + if not self._currentdir: + self._currentdir = '.' + if not self._basedir: + self._basedir = self._currentdir + + self._scope_id = Scope.SCOPE_ID + Scope.SCOPE_ID += 1 + self._file = file + self._condition = map_condition(condition) + self._children = [] # type: typing.List[Scope] + self._included_children = [] # type: typing.List[Scope] + self._operations = copy.deepcopy(operations) + self._visited_keys = set() # type: typing.Set[str] + self._total_condition = None # type: typing.Optional[str] + + def __repr__(self): + return '{}:{}:{}:{}:{}'.format(self._scope_id, + self._basedir, self._currentdir, + self._file, self._condition or '<TRUE>') + + def reset_visited_keys(self): + self._visited_keys = set() + + def merge(self, other: 'Scope') -> None: + assert self != other + self._included_children.append(other) + + @property + def scope_debug(self) -> bool: + merge = self.get_string('PRO2CMAKE_SCOPE_DEBUG').lower() + return merge == '1' or merge == 'on' or merge == 'yes' or merge == 'true' + + @property + def parent(self) -> typing.Optional[Scope]: + return self._parent + + @property + def basedir(self) -> str: + return self._basedir + + @property + def currentdir(self) -> str: + return self._currentdir + + def can_merge_condition(self): + if self._condition == 'else': + return False + if self._operations: + return False + + child_count = len(self._children) + if child_count == 0 or child_count > 2: + return False + assert child_count != 1 or self._children[0]._condition != 'else' + return child_count == 1 or self._children[1]._condition == 'else' + + def settle_condition(self): + new_children: typing.List[Scope] = [] + for c in self._children: + c.settle_condition() + + if c.can_merge_condition(): + child = c._children[0] + child._condition = '({}) AND ({})'.format(c._condition, child._condition) + new_children += c._children + else: + new_children.append(c) + self._children = new_children + + @staticmethod + def FromDict(parent_scope: typing.Optional['Scope'], + file: str, statements, cond: str = '', base_dir: str = '') -> Scope: + scope = Scope(parent_scope=parent_scope, file=file, condition=cond, base_dir=base_dir) + for statement in statements: + if isinstance(statement, list): # Handle skipped parts... + assert not statement + continue + + operation = statement.get('operation', None) + if operation: + key = statement.get('key', '') + value = statement.get('value', []) + assert key != '' + + if operation == '=': + scope._append_operation(key, SetOperation(value)) + elif operation == '-=': + scope._append_operation(key, RemoveOperation(value)) + elif operation == '+=': + scope._append_operation(key, AddOperation(value)) + elif operation == '*=': + scope._append_operation(key, UniqueAddOperation(value)) + else: + print('Unexpected operation "{}" in scope "{}".' + .format(operation, scope)) + assert(False) + + continue + + condition = statement.get('condition', None) + if condition: + Scope.FromDict(scope, file, + statement.get('statements'), condition, + scope.basedir) + + else_statements = statement.get('else_statements') + if else_statements: + Scope.FromDict(scope, file, else_statements, + 'else', scope.basedir) + continue + + loaded = statement.get('loaded') + if loaded: + scope._append_operation('_LOADED', UniqueAddOperation(loaded)) + continue + + option = statement.get('option', None) + if option: + scope._append_operation('_OPTION', UniqueAddOperation(option)) + continue + + included = statement.get('included', None) + if included: + scope._append_operation('_INCLUDED', + UniqueAddOperation(included)) + continue + + scope.settle_condition() + + if scope.scope_debug: + print('..... [SCOPE_DEBUG]: Created scope {}:'.format(scope)) + scope.dump(indent=1) + print('..... [SCOPE_DEBUG]: <<END OF SCOPE>>') + return scope + + def _append_operation(self, key: str, op: Operation) -> None: + if key in self._operations: + self._operations[key].append(op) + else: + self._operations[key] = [op, ] + + @property + def file(self) -> str: + return self._file or '' + + @property + def generated_cmake_lists_path(self) -> str: + assert self.basedir + return os.path.join(self.basedir, 'CMakeLists.gen.txt') + + @property + def original_cmake_lists_path(self) -> str: + assert self.basedir + return os.path.join(self.basedir, 'CMakeLists.txt') + + @property + def condition(self) -> str: + return self._condition + + @property + def total_condition(self) -> typing.Optional[str]: + return self._total_condition + + @total_condition.setter + def total_condition(self, condition: str) -> None: + self._total_condition = condition + + def _add_child(self, scope: 'Scope') -> None: + scope._parent = self + self._children.append(scope) + + @property + def children(self) -> typing.List['Scope']: + result = list(self._children) + for include_scope in self._included_children: + result += include_scope.children + return result + + def dump(self, *, indent: int = 0) -> None: + ind = ' ' * indent + print('{}Scope "{}":'.format(ind, self)) + if self.total_condition: + print('{} Total condition = {}'.format(ind, self.total_condition)) + print('{} Keys:'.format(ind)) + keys = self._operations.keys() + if not keys: + print('{} -- NONE --'.format(ind)) + else: + for k in sorted(keys): + print('{} {} = "{}"' + .format(ind, k, self._operations.get(k, []))) + print('{} Children:'.format(ind)) + if not self._children: + print('{} -- NONE --'.format(ind)) + else: + for c in self._children: + c.dump(indent=indent + 1) + print('{} Includes:'.format(ind)) + if not self._included_children: + print('{} -- NONE --'.format(ind)) + else: + for c in self._included_children: + c.dump(indent=indent + 1) + + def dump_structure(self, *, type: str = 'ROOT', indent: int = 0) -> None: + print('{}{}: {}'.format(spaces(indent), type, self)) + for i in self._included_children: + i.dump_structure(type='INCL', indent=indent + 1) + for i in self._children: + i.dump_structure(type='CHLD', indent=indent + 1) + + @property + def keys(self): + return self._operations.keys() + + @property + def visited_keys(self): + return self._visited_keys + + def _evalOps(self, key: str, + transformer: typing.Optional[typing.Callable[[Scope, typing.List[str]], typing.List[str]]], + result: typing.List[str], *, inherrit: bool = False) \ + -> typing.List[str]: + self._visited_keys.add(key) + + # Inherrit values from above: + if self._parent and inherrit: + result = self._parent._evalOps(key, transformer, result) + + if transformer: + op_transformer = lambda files: transformer(self, files) + else: + op_transformer = lambda files: files + + for op in self._operations.get(key, []): + result = op.process(key, result, op_transformer) + + for ic in self._included_children: + result = list(ic._evalOps(key, transformer, result)) + + return result + + def get(self, key: str, *, ignore_includes: bool = False, inherrit: bool = False) -> typing.List[str]: + + is_same_path = self.currentdir == self.basedir + + if key == 'PWD': + if is_same_path: + return ['${CMAKE_CURRENT_SOURCE_DIR}'] + else: + return ['${CMAKE_CURRENT_SOURCE_DIR}/' + os.path.relpath(self.currentdir, self.basedir),] + if key == 'OUT_PWD': + if is_same_path: + return ['${CMAKE_CURRENT_BINARY_DIR}'] + else: + return ['${CMAKE_CURRENT_BINARY_DIR}/' + os.path.relpath(self.currentdir, self.basedir),] + + return self._evalOps(key, None, [], inherrit=inherrit) + + def get_string(self, key: str, default: str = '') -> str: + v = self.get(key) + if len(v) == 0: + return default + assert len(v) == 1 + return v[0] + + def _map_files(self, files: typing.List[str], *, + use_vpath: bool = True, is_include: bool = False) -> typing.List[str]: + + expanded_files = [] # type: typing.List[str] + for f in files: + r = self._expand_value(f) + expanded_files += r + + mapped_files = list(map(lambda f: map_to_file(f, self, is_include=is_include), expanded_files)) + + if use_vpath: + result = list(map(lambda f: handle_vpath(f, self.basedir, self.get('VPATH', inherrit=True)), mapped_files)) + else: + result = mapped_files + + # strip ${CMAKE_CURRENT_SOURCE_DIR}: + result = list(map(lambda f: f[28:] if f.startswith('${CMAKE_CURRENT_SOURCE_DIR}/') else f, result)) + + # strip leading ./: + result = list(map(lambda f: trim_leading_dot(f), result)) + + return result + + def get_files(self, key: str, *, use_vpath: bool = False, + is_include: bool = False) -> typing.List[str]: + transformer = lambda scope, files: scope._map_files(files, use_vpath=use_vpath, is_include=is_include) + return list(self._evalOps(key, transformer, [])) + + def _expand_value(self, value: str) -> typing.List[str]: + result = value + pattern = re.compile(r'\$\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?') + match = re.search(pattern, result) + while match: + old_result = result + if match.group(0) == value: + return self.get(match.group(1)) + + replacement = self.get(match.group(1)) + replacement_str = replacement[0] if replacement else '' + result = result[:match.start()] \ + + replacement_str \ + + result[match.end():] + + if result == old_result: + return [result,] # Do not go into infinite loop + + match = re.search(pattern, result) + return [result,] + + def expand(self, key: str) -> typing.List[str]: + value = self.get(key) + result: typing.List[str] = [] + assert isinstance(value, list) + for v in value: + result += self._expand_value(v) + return result + + def expandString(self, key: str) -> str: + result = self._expand_value(self.get_string(key)) + assert len(result) == 1 + return result[0] + + @property + def TEMPLATE(self) -> str: + return self.get_string('TEMPLATE', 'app') + + def _rawTemplate(self) -> str: + return self.get_string('TEMPLATE') + + @property + def TARGET(self) -> str: + return self.get_string('TARGET') \ + or os.path.splitext(os.path.basename(self.file))[0] + + @property + def _INCLUDED(self) -> typing.List[str]: + return self.get('_INCLUDED') + + +class QmakeParser: + def __init__(self, *, debug: bool = False) -> None: + self.debug = debug + self._Grammar = self._generate_grammar() + + def _generate_grammar(self): + # Define grammar: + pp.ParserElement.setDefaultWhitespaceChars(' \t') + + def add_element(name: str, value: pp.ParserElement): + nonlocal self + if self.debug: + value.setName(name) + value.setDebug() + return value + + EOL = add_element('EOL', pp.Suppress(pp.LineEnd())) + Else = add_element('Else', pp.Keyword('else')) + Identifier = add_element('Identifier', pp.Word(pp.alphas + '_', + bodyChars=pp.alphanums+'_-./')) + BracedValue = add_element('BracedValue', + pp.nestedExpr( + ignoreExpr=pp.quotedString | + pp.QuotedString(quoteChar='$(', + endQuoteChar=')', + escQuote='\\', + unquoteResults=False) + ).setParseAction(lambda s, l, t: ['(', *t[0], ')'])) + + Substitution \ + = add_element('Substitution', + pp.Combine(pp.Literal('$') + + (((pp.Literal('$') + Identifier + + pp.Optional(pp.nestedExpr())) + | (pp.Literal('(') + Identifier + pp.Literal(')')) + | (pp.Literal('{') + Identifier + pp.Literal('}')) + | (pp.Literal('$') + pp.Literal('{') + + Identifier + pp.Optional(pp.nestedExpr()) + + pp.Literal('}')) + | (pp.Literal('$') + pp.Literal('[') + Identifier + + pp.Literal(']')) + )))) + LiteralValuePart = add_element('LiteralValuePart', + pp.Word(pp.printables, excludeChars='$#{}()')) + SubstitutionValue \ + = add_element('SubstitutionValue', + pp.Combine(pp.OneOrMore(Substitution + | LiteralValuePart + | pp.Literal('$')))) + Value \ + = add_element('Value', + pp.NotAny(Else | pp.Literal('}') | EOL) \ + + (pp.QuotedString(quoteChar='"', escChar='\\') + | SubstitutionValue + | BracedValue)) + + Values = add_element('Values', pp.ZeroOrMore(Value)('value')) + + Op = add_element('OP', + pp.Literal('=') | pp.Literal('-=') | pp.Literal('+=') \ + | pp.Literal('*=')) + + Key = add_element('Key', Identifier) + + Operation = add_element('Operation', Key('key') + Op('operation') + Values('value')) + CallArgs = add_element('CallArgs', pp.nestedExpr()) + + def parse_call_args(results): + out = '' + for item in chain(*results): + if isinstance(item, str): + out += item + else: + out += "(" + parse_call_args(item) + ")" + return out + + CallArgs.setParseAction(parse_call_args) + + Load = add_element('Load', pp.Keyword('load') + CallArgs('loaded')) + Include = add_element('Include', pp.Keyword('include') + CallArgs('included')) + Option = add_element('Option', pp.Keyword('option') + CallArgs('option')) + + # ignore the whole thing... + DefineTestDefinition = add_element( + 'DefineTestDefinition', + pp.Suppress(pp.Keyword('defineTest') + CallArgs + + pp.nestedExpr(opener='{', closer='}', ignoreExpr=pp.LineEnd()))) + + # ignore the whole thing... + ForLoop = add_element( + 'ForLoop', + pp.Suppress(pp.Keyword('for') + CallArgs + + pp.nestedExpr(opener='{', closer='}', ignoreExpr=pp.LineEnd()))) + + # ignore the whole thing... + ForLoopSingleLine = add_element( + 'ForLoopSingleLine', + pp.Suppress(pp.Keyword('for') + CallArgs + pp.Literal(':') + pp.SkipTo(EOL))) + + # ignore the whole thing... + FunctionCall = add_element('FunctionCall', pp.Suppress(Identifier + pp.nestedExpr())) + + Scope = add_element('Scope', pp.Forward()) + + Statement = add_element('Statement', + pp.Group(Load | Include | Option | ForLoop | ForLoopSingleLine + | DefineTestDefinition | FunctionCall | Operation)) + StatementLine = add_element('StatementLine', Statement + (EOL | pp.FollowedBy('}'))) + StatementGroup = add_element('StatementGroup', + pp.ZeroOrMore(StatementLine | Scope | pp.Suppress(EOL))) + + Block = add_element('Block', + pp.Suppress('{') + pp.Optional(EOL) + + StatementGroup + pp.Optional(EOL) + + pp.Suppress('}') + pp.Optional(EOL)) + + ConditionEnd = add_element('ConditionEnd', + pp.FollowedBy((pp.Optional(pp.White()) + + (pp.Literal(':') + | pp.Literal('{') + | pp.Literal('|'))))) + + ConditionPart1 = add_element('ConditionPart1', + (pp.Optional('!') + Identifier + pp.Optional(BracedValue))) + ConditionPart2 = add_element('ConditionPart2', pp.CharsNotIn('#{}|:=\\\n')) + ConditionPart = add_element( + 'ConditionPart', + (ConditionPart1 ^ ConditionPart2) + ConditionEnd) + + ConditionOp = add_element('ConditionOp', pp.Literal('|') ^ pp.Literal(':')) + ConditionWhiteSpace = add_element('ConditionWhiteSpace', + pp.Suppress(pp.Optional(pp.White(' ')))) + + ConditionRepeated = add_element('ConditionRepeated', + pp.ZeroOrMore(ConditionOp + + ConditionWhiteSpace + ConditionPart)) + + Condition = add_element('Condition', pp.Combine(ConditionPart + ConditionRepeated)) + Condition.setParseAction(lambda x: ' '.join(x).strip().replace(':', ' && ').strip(' && ')) + + # Weird thing like write_file(a)|error() where error() is the alternative condition + # which happens to be a function call. In this case there is no scope, but our code expects + # a scope with a list of statements, so create a fake empty statement. + ConditionEndingInFunctionCall = add_element( + 'ConditionEndingInFunctionCall', pp.Suppress(ConditionOp) + FunctionCall + + pp.Empty().setParseAction(lambda x: [[]]) + .setResultsName('statements')) + + SingleLineScope = add_element('SingleLineScope', + pp.Suppress(pp.Literal(':')) + + pp.Group(Block | (Statement + EOL))('statements')) + MultiLineScope = add_element('MultiLineScope', Block('statements')) + + SingleLineElse = add_element('SingleLineElse', + pp.Suppress(pp.Literal(':')) + + (Scope | Block | (Statement + pp.Optional(EOL)))) + MultiLineElse = add_element('MultiLineElse', Block) + ElseBranch = add_element('ElseBranch', pp.Suppress(Else) + (SingleLineElse | MultiLineElse)) + + # Scope is already add_element'ed in the forward declaration above. + Scope <<= \ + pp.Group(Condition('condition') + + (SingleLineScope | MultiLineScope | ConditionEndingInFunctionCall) + + pp.Optional(ElseBranch)('else_statements')) + + Grammar = StatementGroup('statements') + Grammar.ignore(pp.pythonStyleComment()) + + return Grammar + + def parseFile(self, file: str): + print('Parsing \"{}\"...'.format(file)) + try: + with open(file, 'r') as file_fd: + contents = file_fd.read() + + old_contents = contents + contents = fixup_comments(contents) + contents = fixup_linecontinuation(contents) + + if old_contents != contents: + print('Warning: Fixed line continuation in .pro-file!\n' + ' Position information in Parsing output might be wrong!') + result = self._Grammar.parseString(contents, parseAll=True) + except pp.ParseException as pe: + print(pe.line) + print(' '*(pe.col-1) + '^') + print(pe) + raise pe + return result + + +def parseProFile(file: str, *, debug=False): + parser = QmakeParser(debug=debug) + return parser.parseFile(file) + + +def map_condition(condition: str) -> str: + # Some hardcoded cases that are too bothersome to generalize. + condition = re.sub(r'^qtConfig\(opengl\(es1\|es2\)\?\)$', + r'QT_FEATURE_opengl OR QT_FEATURE_opengles2 OR QT_FEATURE_opengles3', + condition) + condition = re.sub(r'^qtConfig\(opengl\.\*\)$', r'QT_FEATURE_opengl', condition) + condition = re.sub(r'^win\*$', r'win', condition) + + def gcc_version_handler(match_obj: re.Match): + operator = match_obj.group(1) + version_type = match_obj.group(2) + if operator == 'equals': + operator = 'STREQUAL' + elif operator == 'greaterThan': + operator = 'STRGREATER' + elif operator == 'lessThan': + operator = 'STRLESS' + + version = match_obj.group(3) + return '(QT_COMPILER_VERSION_{} {} {})'.format(version_type, operator, version) + + # TODO: Possibly fix for other compilers. + pattern = r'(equals|greaterThan|lessThan)\(QT_GCC_([A-Z]+)_VERSION,[ ]*([0-9]+)\)' + condition = re.sub(pattern, gcc_version_handler, condition) + + # TODO: the current if(...) replacement makes the parentheses + # unbalanced when there are nested expressions. + # Need to fix this either with pypi regex recursive regexps, + # using pyparsing, or some other proper means of handling + # balanced parentheses. + condition = re.sub(r'\bif\s*\((.*?)\)', r'\1', condition) + + condition = re.sub(r'\bisEmpty\s*\((.*?)\)', r'\1_ISEMPTY', condition) + condition = re.sub(r'\bcontains\s*\((.*?),\s*"?(.*?)"?\)', + r'\1___contains___\2', condition) + condition = re.sub(r'\bequals\s*\((.*?),\s*"?(.*?)"?\)', + r'\1___equals___\2', condition) + condition = re.sub(r'\bisEqual\s*\((.*?),\s*"?(.*?)"?\)', + r'\1___equals___\2', condition) + condition = re.sub(r'\s*==\s*', '___STREQUAL___', condition) + condition = re.sub(r'\bexists\s*\((.*?)\)', r'EXISTS \1', condition) + + pattern = r'CONFIG\((debug|release),debug\|release\)' + match_result = re.match(pattern, condition) + if match_result: + build_type = match_result.group(1) + if build_type == 'debug': + build_type = 'Debug' + elif build_type == 'release': + build_type = 'Release' + condition = re.sub(pattern, '(CMAKE_BUILD_TYPE STREQUAL {})'.format(build_type), condition) + + condition = condition.replace('*', '_x_') + condition = condition.replace('.$$', '__ss_') + condition = condition.replace('$$', '_ss_') + + condition = condition.replace('!', 'NOT ') + condition = condition.replace('&&', ' AND ') + condition = condition.replace('|', ' OR ') + + cmake_condition = '' + for part in condition.split(): + # some features contain e.g. linux, that should not be + # turned upper case + feature = re.match(r"(qtConfig|qtHaveModule)\(([a-zA-Z0-9_-]+)\)", + part) + if feature: + if (feature.group(1) == "qtHaveModule"): + part = 'TARGET {}'.format(map_qt_library(feature.group(2))) + else: + feature_name = featureName(feature.group(2)) + if feature_name.startswith('system_') and is_known_3rd_party_library(feature_name[7:]): + part = 'ON' + elif feature == 'dlopen': + part = 'ON' + else: + part = 'QT_FEATURE_' + feature_name + else: + part = map_platform(part) + + part = part.replace('true', 'ON') + part = part.replace('false', 'OFF') + cmake_condition += ' ' + part + return cmake_condition.strip() + + +def handle_subdir(scope: Scope, cm_fh: typing.IO[str], *, + indent: int = 0, is_example: bool=False) -> None: + ind = ' ' * indent + for sd in scope.get_files('SUBDIRS'): + if os.path.isdir(sd): + cm_fh.write('{}add_subdirectory({})\n'.format(ind, sd)) + elif os.path.isfile(sd): + subdir_result = parseProFile(sd, debug=False) + subdir_scope \ + = Scope.FromDict(scope, sd, + subdir_result.asDict().get('statements'), + '', scope.basedir) + + do_include(subdir_scope) + cmakeify_scope(subdir_scope, cm_fh, indent=indent, is_example=is_example) + elif sd.startswith('-'): + cm_fh.write('{}### remove_subdirectory' + '("{}")\n'.format(ind, sd[1:])) + else: + print(' XXXX: SUBDIR {} in {}: Not found.'.format(sd, scope)) + + for c in scope.children: + cond = c.condition + if cond == 'else': + cm_fh.write('\n{}else()\n'.format(ind)) + elif cond: + cm_fh.write('\n{}if({})\n'.format(ind, cond)) + + handle_subdir(c, cm_fh, indent=indent + 1, is_example=is_example) + + if cond: + cm_fh.write('{}endif()\n'.format(ind)) + + +def sort_sources(sources: typing.List[str]) -> typing.List[str]: + to_sort = {} # type: typing.Dict[str, typing.List[str]] + for s in sources: + if s is None: + continue + + dir = os.path.dirname(s) + base = os.path.splitext(os.path.basename(s))[0] + if base.endswith('_p'): + base = base[:-2] + sort_name = os.path.join(dir, base) + + array = to_sort.get(sort_name, []) + array.append(s) + + to_sort[sort_name] = array + + lines = [] + for k in sorted(to_sort.keys()): + lines.append(' '.join(sorted(to_sort[k]))) + + return lines + + +def _map_libraries_to_cmake(libraries: typing.List[str], + known_libraries: typing.Set[str]) -> typing.List[str]: + result = [] # type: typing.List[str] + is_framework = False + + for l in libraries: + if l == '-framework': + is_framework = True + continue + if is_framework: + l = '${FW%s}' % l + if l.startswith('-l'): + l = l[2:] + + if l.startswith('-'): + l = '# Remove: {}'.format(l[1:]) + else: + l = map_3rd_party_library(l) + + if not l or l in result or l in known_libraries: + continue + + result.append(l) + is_framework = False + + return result + + +def extract_cmake_libraries(scope: Scope, *, known_libraries: typing.Set[str]=set()) \ + -> typing.Tuple[typing.List[str], typing.List[str]]: + public_dependencies = [] # type: typing.List[str] + private_dependencies = [] # type: typing.List[str] + + for key in ['QMAKE_USE', 'LIBS',]: + public_dependencies += scope.expand(key) + for key in ['QMAKE_USE_PRIVATE', 'QMAKE_USE_FOR_PRIVATE', 'LIBS_PRIVATE',]: + private_dependencies += scope.expand(key) + + for key in ['QT_FOR_PRIVATE',]: + private_dependencies += [map_qt_library(q) for q in scope.expand(key)] + + for key in ['QT',]: + # Qt public libs: These may include FooPrivate in which case we get + # a private dependency on FooPrivate as well as a public dependency on Foo + for lib in scope.expand(key): + mapped_lib = map_qt_library(lib) + + if mapped_lib.endswith('Private'): + private_dependencies.append(mapped_lib) + public_dependencies.append(mapped_lib[:-7]) + else: + public_dependencies.append(mapped_lib) + + return (_map_libraries_to_cmake(public_dependencies, known_libraries), + _map_libraries_to_cmake(private_dependencies, known_libraries)) + + +def write_header(cm_fh: typing.IO[str], name: str, + typename: str, *, indent: int = 0): + cm_fh.write('{}###########################################' + '##########################\n'.format(spaces(indent))) + cm_fh.write('{}## {} {}:\n'.format(spaces(indent), name, typename)) + cm_fh.write('{}###########################################' + '##########################\n\n'.format(spaces(indent))) + + +def write_scope_header(cm_fh: typing.IO[str], *, indent: int = 0): + cm_fh.write('\n{}## Scopes:\n'.format(spaces(indent))) + cm_fh.write('{}###########################################' + '##########################\n'.format(spaces(indent))) + + +def write_list(cm_fh: typing.IO[str], entries: typing.List[str], + cmake_parameter: str, + indent: int = 0, *, + header: str = '', footer: str = ''): + if not entries: + return + + ind = spaces(indent) + extra_indent = '' + + if header: + cm_fh.write('{}{}'.format(ind, header)) + extra_indent += ' ' + if cmake_parameter: + cm_fh.write('{}{}{}\n'.format(ind, extra_indent, cmake_parameter)) + extra_indent += ' ' + for s in sort_sources(entries): + cm_fh.write('{}{}{}\n'.format(ind, extra_indent, s)) + if footer: + cm_fh.write('{}{}\n'.format(ind, footer)) + + +def write_source_file_list(cm_fh: typing.IO[str], scope, cmake_parameter: str, + keys: typing.List[str], indent: int = 0, *, + header: str = '', footer: str = ''): + # collect sources + sources: typing.List[str] = [] + for key in keys: + sources += scope.get_files(key, use_vpath=True) + + write_list(cm_fh, sources, cmake_parameter, indent, header=header, footer=footer) + + +def write_all_source_file_lists(cm_fh: typing.IO[str], scope: Scope, header: str, *, + indent: int = 0, footer: str = '', + extra_keys: typing.Optional[typing.List[str]] = None): + if extra_keys is None: + extra_keys = [] + write_source_file_list(cm_fh, scope, header, + ['SOURCES', 'HEADERS', 'OBJECTIVE_SOURCES', 'NO_PCH_SOURCES', 'FORMS'] + extra_keys, + indent, footer=footer) + + +def write_defines(cm_fh: typing.IO[str], scope: Scope, cmake_parameter: str, *, + indent: int = 0, footer: str = ''): + defines = scope.expand('DEFINES') + defines += [d[2:] for d in scope.expand('QMAKE_CXXFLAGS') if d.startswith('-D')] + defines = [d.replace('=\\\\\\"$$PWD/\\\\\\"', + '="${CMAKE_CURRENT_SOURCE_DIR}/"') for d in defines] + + write_list(cm_fh, defines, cmake_parameter, indent, footer=footer) + + +def write_include_paths(cm_fh: typing.IO[str], scope: Scope, cmake_parameter: str, *, + indent: int = 0, footer: str = ''): + includes = [i.rstrip('/') or ('/') for i in scope.get_files('INCLUDEPATH')] + + write_list(cm_fh, includes, cmake_parameter, indent, footer=footer) + + +def write_compile_options(cm_fh: typing.IO[str], scope: Scope, cmake_parameter: str, *, + indent: int = 0, footer: str = ''): + compile_options = [d for d in scope.expand('QMAKE_CXXFLAGS') if not d.startswith('-D')] + + write_list(cm_fh, compile_options, cmake_parameter, indent, footer=footer) + + +def write_library_section(cm_fh: typing.IO[str], scope: Scope, *, + indent: int = 0, known_libraries: typing.Set[str]=set()): + (public_dependencies, private_dependencies) \ + = extract_cmake_libraries(scope, known_libraries=known_libraries) + + write_list(cm_fh, private_dependencies, 'LIBRARIES', indent + 1) + write_list(cm_fh, public_dependencies, 'PUBLIC_LIBRARIES', indent + 1) + + +def write_autogen_section(cm_fh: typing.IO[str], scope: Scope, *, + indent: int = 0): + forms = scope.get_files('FORMS') + if forms: + write_list(cm_fh, ['uic'], 'ENABLE_AUTOGEN_TOOLS', indent) + + +def write_sources_section(cm_fh: typing.IO[str], scope: Scope, *, + indent: int = 0, known_libraries=set()): + ind = spaces(indent) + + # mark RESOURCES as visited: + scope.get('RESOURCES') + + write_all_source_file_lists(cm_fh, scope, 'SOURCES', indent=indent + 1) + + write_source_file_list(cm_fh, scope, 'DBUS_ADAPTOR_SOURCES', ['DBUS_ADAPTORS',], indent + 1) + dbus_adaptor_flags = scope.expand('QDBUSXML2CPP_ADAPTOR_HEADER_FLAGS') + if dbus_adaptor_flags: + cm_fh.write('{} DBUS_ADAPTOR_FLAGS\n'.format(ind)) + cm_fh.write('{} "{}"\n'.format(ind, '" "'.join(dbus_adaptor_flags))) + + write_source_file_list(cm_fh, scope, 'DBUS_INTERFACE_SOURCES', ['DBUS_INTERFACES',], indent + 1) + dbus_interface_flags = scope.expand('QDBUSXML2CPP_INTERFACE_HEADER_FLAGS') + if dbus_interface_flags: + cm_fh.write('{} DBUS_INTERFACE_FLAGS\n'.format(ind)) + cm_fh.write('{} "{}"\n'.format(ind, '" "'.join(dbus_interface_flags))) + + write_defines(cm_fh, scope, 'DEFINES', indent=indent + 1) + + write_include_paths(cm_fh, scope, 'INCLUDE_DIRECTORIES', indent=indent + 1) + + write_library_section(cm_fh, scope, indent=indent, known_libraries=known_libraries) + + write_compile_options(cm_fh, scope, 'COMPILE_OPTIONS', indent=indent + 1) + + write_autogen_section(cm_fh, scope, indent=indent + 1) + + link_options = scope.get('QMAKE_LFLAGS') + if link_options: + cm_fh.write('{} LINK_OPTIONS\n'.format(ind)) + for lo in link_options: + cm_fh.write('{} "{}"\n'.format(ind, lo)) + + moc_options = scope.get('QMAKE_MOC_OPTIONS') + if moc_options: + cm_fh.write('{} MOC_OPTIONS\n'.format(ind)) + for mo in moc_options: + cm_fh.write('{} "{}"\n'.format(ind, mo)) + + +def is_simple_condition(condition: str) -> bool: + return ' ' not in condition \ + or (condition.startswith('NOT ') and ' ' not in condition[4:]) + + +def write_ignored_keys(scope: Scope, indent: str) -> str: + result = '' + ignored_keys = scope.keys - scope.visited_keys + for k in sorted(ignored_keys): + if k == '_INCLUDED' or k == 'TARGET' or k == 'QMAKE_DOCS' or k == 'QT_SOURCE_TREE' \ + or k == 'QT_BUILD_TREE' or k == 'TRACEPOINT_PROVIDER': + # All these keys are actually reported already + continue + values = scope.get(k) + value_string = '<EMPTY>' if not values \ + else '"' + '" "'.join(scope.get(k)) + '"' + result += '{}# {} = {}\n'.format(indent, k, value_string) + + if result: + result = '\n#### Keys ignored in scope {}:\n{}'.format(scope, result) + + return result + + +def _iterate_expr_tree(expr, op, matches): + assert expr.func == op + keepers = () + for arg in expr.args: + if arg in matches: + matches = tuple(x for x in matches if x != arg) + elif arg == op: + (matches, extra_keepers) = _iterate_expr_tree(arg, op, matches) + keepers = (*keepers, *extra_keepers) + else: + keepers = (*keepers, arg) + return matches, keepers + + +def _simplify_expressions(expr, op, matches, replacement): + for arg in expr.args: + expr = expr.subs(arg, _simplify_expressions(arg, op, matches, + replacement)) + + if expr.func == op: + (to_match, keepers) = tuple(_iterate_expr_tree(expr, op, matches)) + if len(to_match) == 0: + # build expression with keepers and replacement: + if keepers: + start = replacement + current_expr = None + last_expr = keepers[-1] + for repl_arg in keepers[:-1]: + current_expr = op(start, repl_arg) + start = current_expr + top_expr = op(start, last_expr) + else: + top_expr = replacement + + expr = expr.subs(expr, top_expr) + + return expr + + +def _simplify_flavors_in_condition(base: str, flavors, expr): + ''' Simplify conditions based on the knownledge of which flavors + belong to which OS. ''' + base_expr = simplify_logic(base) + false_expr = simplify_logic('false') + for flavor in flavors: + flavor_expr = simplify_logic(flavor) + expr = _simplify_expressions(expr, And, (base_expr, flavor_expr,), + flavor_expr) + expr = _simplify_expressions(expr, Or, (base_expr, flavor_expr), + base_expr) + expr = _simplify_expressions(expr, And, (Not(base_expr), flavor_expr,), + false_expr) + return expr + + +def _simplify_os_families(expr, family_members, other_family_members): + for family in family_members: + for other in other_family_members: + if other in family_members: + continue # skip those in the sub-family + + f_expr = simplify_logic(family) + o_expr = simplify_logic(other) + + expr = _simplify_expressions(expr, And, (f_expr, Not(o_expr)), f_expr) + expr = _simplify_expressions(expr, And, (Not(f_expr), o_expr), o_expr) + expr = _simplify_expressions(expr, And, (f_expr, o_expr), simplify_logic('false')) + return expr + + +def _recursive_simplify(expr): + ''' Simplify the expression as much as possible based on + domain knowledge. ''' + input_expr = expr + + # Simplify even further, based on domain knowledge: + windowses = ('WIN32', 'WINRT') + apples = ('APPLE_OSX', 'APPLE_UIKIT', 'APPLE_IOS', + 'APPLE_TVOS', 'APPLE_WATCHOS',) + bsds = ('FREEBSD', 'OPENBSD', 'NETBSD',) + androids = ('ANDROID', 'ANDROID_EMBEDDED') + unixes = ('APPLE', *apples, 'BSD', *bsds, 'LINUX', + *androids, 'HAIKU', + 'INTEGRITY', 'VXWORKS', 'QNX', 'WASM') + + unix_expr = simplify_logic('UNIX') + win_expr = simplify_logic('WIN32') + false_expr = simplify_logic('false') + true_expr = simplify_logic('true') + + expr = expr.subs(Not(unix_expr), win_expr) # NOT UNIX -> WIN32 + expr = expr.subs(Not(win_expr), unix_expr) # NOT WIN32 -> UNIX + + # UNIX [OR foo ]OR WIN32 -> ON [OR foo] + expr = _simplify_expressions(expr, Or, (unix_expr, win_expr,), true_expr) + # UNIX [AND foo ]AND WIN32 -> OFF [AND foo] + expr = _simplify_expressions(expr, And, (unix_expr, win_expr,), false_expr) + + expr = _simplify_flavors_in_condition('WIN32', ('WINRT',), expr) + expr = _simplify_flavors_in_condition('APPLE', apples, expr) + expr = _simplify_flavors_in_condition('BSD', bsds, expr) + expr = _simplify_flavors_in_condition('UNIX', unixes, expr) + expr = _simplify_flavors_in_condition('ANDROID', ('ANDROID_EMBEDDED',), expr) + + # Simplify families of OSes against other families: + expr = _simplify_os_families(expr, ('WIN32', 'WINRT'), unixes) + expr = _simplify_os_families(expr, androids, unixes) + expr = _simplify_os_families(expr, ('BSD', *bsds), unixes) + + for family in ('HAIKU', 'QNX', 'INTEGRITY', 'LINUX', 'VXWORKS'): + expr = _simplify_os_families(expr, (family,), unixes) + + # Now simplify further: + expr = simplify_logic(expr) + + while expr != input_expr: + input_expr = expr + expr = _recursive_simplify(expr) + + return expr + + +def simplify_condition(condition: str) -> str: + input_condition = condition.strip() + + # Map to sympy syntax: + condition = ' ' + input_condition + ' ' + condition = condition.replace('(', ' ( ') + condition = condition.replace(')', ' ) ') + + tmp = '' + while tmp != condition: + tmp = condition + + condition = condition.replace(' NOT ', ' ~ ') + condition = condition.replace(' AND ', ' & ') + condition = condition.replace(' OR ', ' | ') + condition = condition.replace(' ON ', ' true ') + condition = condition.replace(' OFF ', ' false ') + + try: + # Generate and simplify condition using sympy: + condition_expr = simplify_logic(condition) + condition = str(_recursive_simplify(condition_expr)) + + # Map back to CMake syntax: + condition = condition.replace('~', 'NOT ') + condition = condition.replace('&', 'AND') + condition = condition.replace('|', 'OR') + condition = condition.replace('True', 'ON') + condition = condition.replace('False', 'OFF') + except: + # sympy did not like our input, so leave this condition alone: + condition = input_condition + + return condition or 'ON' + + +def recursive_evaluate_scope(scope: Scope, parent_condition: str = '', + previous_condition: str = '') -> str: + current_condition = scope.condition + total_condition = current_condition + if total_condition == 'else': + assert previous_condition, \ + "Else branch without previous condition in: %s" % scope.file + total_condition = 'NOT ({})'.format(previous_condition) + if parent_condition: + if not total_condition: + total_condition = parent_condition + else: + total_condition = '({}) AND ({})'.format(parent_condition, + total_condition) + + scope.total_condition = simplify_condition(total_condition) + + prev_condition = '' + for c in scope.children: + prev_condition = recursive_evaluate_scope(c, total_condition, + prev_condition) + + return current_condition + + +def map_to_cmake_condition(condition: typing.Optional[str]) -> str: + condition = re.sub(r'\bQT_ARCH___equals___([a-zA-Z_0-9]*)', + r'(TEST_architecture_arch STREQUAL "\1")', condition or '') + condition = re.sub(r'\bQT_ARCH___contains___([a-zA-Z_0-9]*)', + r'(TEST_architecture_arch STREQUAL "\1")', condition or '') + return condition + + +def write_resources(cm_fh: typing.IO[str], target: str, scope: Scope, indent: int = 0): + vpath = scope.expand('VPATH') + + # Handle QRC files by turning them into add_qt_resource: + resources = scope.get_files('RESOURCES') + qrc_output = '' + if resources: + qrc_only = True + for r in resources: + if r.endswith('.qrc'): + qrc_output += process_qrc_file(target, r, scope.basedir) + else: + qrc_only = False + + if not qrc_only: + print(' XXXX Ignoring non-QRC file resources.') + + if qrc_output: + cm_fh.write('\n# Resources:\n') + for line in qrc_output.split('\n'): + cm_fh.write(' ' * indent + line + '\n') + + +def write_extend_target(cm_fh: typing.IO[str], target: str, + scope: Scope, indent: int = 0): + ind = spaces(indent) + extend_qt_io_string = io.StringIO() + write_sources_section(extend_qt_io_string, scope) + extend_qt_string = extend_qt_io_string.getvalue() + + extend_scope = '\n{}extend_target({} CONDITION {}\n' \ + '{}{})\n'.format(ind, target, + map_to_cmake_condition(scope.total_condition), + extend_qt_string, ind) + + if not extend_qt_string: + extend_scope = '' # Nothing to report, so don't! + + cm_fh.write(extend_scope) + + write_resources(cm_fh, target, scope, indent) + + +def flatten_scopes(scope: Scope) -> typing.List[Scope]: + result = [scope] # type: typing.List[Scope] + for c in scope.children: + result += flatten_scopes(c) + return result + + +def merge_scopes(scopes: typing.List[Scope]) -> typing.List[Scope]: + result = [] # type: typing.List[Scope] + + # Merge scopes with their parents: + known_scopes = {} # type: typing.Mapping[str, Scope] + for scope in scopes: + total_condition = scope.total_condition + assert total_condition + if total_condition == 'OFF': + # ignore this scope entirely! + pass + elif total_condition in known_scopes: + known_scopes[total_condition].merge(scope) + else: + # Keep everything else: + result.append(scope) + known_scopes[total_condition] = scope + + return result + + +def write_simd_part(cm_fh: typing.IO[str], target: str, scope: Scope, indent: int = 0): + simd_options = [ 'sse2', 'sse3', 'ssse3', 'sse4_1', 'sse4_2', 'aesni', 'shani', 'avx', 'avx2', + 'avx512f', 'avx512cd', 'avx512er', 'avx512pf', 'avx512dq', 'avx512bw', + 'avx512vl', 'avx512ifma', 'avx512vbmi', 'f16c', 'rdrnd', 'neon', 'mips_dsp', + 'mips_dspr2', + 'arch_haswell', 'avx512common', 'avx512core']; + for simd in simd_options: + SIMD = simd.upper(); + write_source_file_list(cm_fh, scope, 'SOURCES', + ['{}_HEADERS'.format(SIMD), + '{}_SOURCES'.format(SIMD), + '{}_C_SOURCES'.format(SIMD), + '{}_ASM'.format(SIMD)], + indent, + header = 'add_qt_simd_part({} SIMD {}\n'.format(target, simd), + footer = ')\n\n') + + +def write_main_part(cm_fh: typing.IO[str], name: str, typename: str, + cmake_function: str, scope: Scope, *, + extra_lines: typing.List[str] = [], + indent: int = 0, extra_keys: typing.List[str], + **kwargs: typing.Any): + # Evaluate total condition of all scopes: + recursive_evaluate_scope(scope) + + if 'exceptions' in scope.get('CONFIG'): + extra_lines.append('EXCEPTIONS') + + # Get a flat list of all scopes but the main one: + scopes = flatten_scopes(scope) + total_scopes = len(scopes) + # Merge scopes based on their conditions: + scopes = merge_scopes(scopes) + + assert len(scopes) + assert scopes[0].total_condition == 'ON' + + scopes[0].reset_visited_keys() + for k in extra_keys: + scopes[0].get(k) + + # Now write out the scopes: + write_header(cm_fh, name, typename, indent=indent) + + cm_fh.write('{}{}({}\n'.format(spaces(indent), cmake_function, name)) + for extra_line in extra_lines: + cm_fh.write('{} {}\n'.format(spaces(indent), extra_line)) + + write_sources_section(cm_fh, scopes[0], indent=indent, **kwargs) + + # Footer: + cm_fh.write('{})\n'.format(spaces(indent))) + + write_resources(cm_fh, name, scope, indent) + + write_simd_part(cm_fh, name, scope, indent) + + ignored_keys_report = write_ignored_keys(scopes[0], spaces(indent)) + if ignored_keys_report: + cm_fh.write(ignored_keys_report) + + + # Scopes: + if len(scopes) == 1: + return + + write_scope_header(cm_fh, indent=indent) + + for c in scopes[1:]: + c.reset_visited_keys() + write_extend_target(cm_fh, name, c, indent=indent) + ignored_keys_report = write_ignored_keys(c, spaces(indent)) + if ignored_keys_report: + cm_fh.write(ignored_keys_report) + + +def write_module(cm_fh: typing.IO[str], scope: Scope, *, + indent: int = 0) -> None: + module_name = scope.TARGET + if not module_name.startswith('Qt'): + print('XXXXXX Module name {} does not start with Qt!'.format(module_name)) + + extra = [] + + # A module should be static when 'static' is in CONFIG + # or when option(host_build) is used, as described in qt_module.prf. + is_static = 'static' in scope.get('CONFIG') or 'host_build' in scope.get('_OPTION') + + if is_static: + extra.append('STATIC') + if 'internal_module' in scope.get('CONFIG'): + extra.append('INTERNAL_MODULE') + if 'no_module_headers' in scope.get('CONFIG'): + extra.append('NO_MODULE_HEADERS') + if 'minimal_syncqt' in scope.get('CONFIG'): + extra.append('NO_SYNC_QT') + + module_config = scope.get("MODULE_CONFIG") + if len(module_config): + extra.append('QMAKE_MODULE_CONFIG {}'.format(" ".join(module_config))) + + module_plugin_types = scope.get_files('MODULE_PLUGIN_TYPES') + if module_plugin_types: + extra.append('PLUGIN_TYPES {}'.format(" ".join(module_plugin_types))) + + write_main_part(cm_fh, module_name[2:], 'Module', 'add_qt_module', scope, + extra_lines=extra, indent=indent, + known_libraries={}, extra_keys=[]) + + if 'qt_tracepoints' in scope.get('CONFIG'): + tracepoints = scope.get_files('TRACEPOINT_PROVIDER') + cm_fh.write('\n\n{}qt_create_tracepoints({} {})\n' + .format(spaces(indent), module_name[2:], ' '.join(tracepoints))) + + +def write_tool(cm_fh: typing.IO[str], scope: Scope, *, + indent: int = 0) -> None: + tool_name = scope.TARGET + + extra = ['BOOTSTRAP'] if 'force_bootstrap' in scope.get('CONFIG') else [] + + write_main_part(cm_fh, tool_name, 'Tool', 'add_qt_tool', scope, + indent=indent, known_libraries={'Qt::Core', }, + extra_lines=extra, extra_keys=['CONFIG']) + + +def write_test(cm_fh: typing.IO[str], scope: Scope, *, + indent: int = 0) -> None: + test_name = scope.TARGET + assert test_name + + write_main_part(cm_fh, test_name, 'Test', 'add_qt_test', scope, + indent=indent, known_libraries={'Qt::Core', 'Qt::Test',}, + extra_keys=[]) + + +def write_binary(cm_fh: typing.IO[str], scope: Scope, + gui: bool = False, *, indent: int = 0) -> None: + binary_name = scope.TARGET + assert binary_name + + extra = ['GUI',] if gui else[] + + target_path = scope.get_string('target.path') + if target_path: + target_path = target_path.replace('$$[QT_INSTALL_EXAMPLES]', '${INSTALL_EXAMPLESDIR}') + extra.append('OUTPUT_DIRECTORY "{}"'.format(target_path)) + if 'target' in scope.get('INSTALLS'): + extra.append('INSTALL_DIRECTORY "{}"'.format(target_path)) + + write_main_part(cm_fh, binary_name, 'Binary', 'add_qt_executable', scope, + extra_lines=extra, indent=indent, + known_libraries={'Qt::Core', }, extra_keys=['target.path', 'INSTALLS']) + + +def write_find_package_section(cm_fh: typing.IO[str], + public_libs: typing.List[str], + private_libs: typing.List[str], *, indent: int=0): + packages = [] # type: typing.List[LibraryMapping] + all_libs = public_libs + private_libs + + for l in all_libs: + info = find_library_info_for_target(l) + if info and info not in packages: + packages.append(info) + + ind = spaces(indent) + + for p in packages: + cm_fh.write(generate_find_package_info(p, use_qt_find_package=False, indent=indent)) + + if packages: + cm_fh.write('\n') + + +def write_example(cm_fh: typing.IO[str], scope: Scope, + gui: bool = False, *, indent: int = 0) -> None: + binary_name = scope.TARGET + assert binary_name + + cm_fh.write('cmake_minimum_required(VERSION 3.14)\n' + + 'project({} LANGUAGES CXX)\n\n'.format(binary_name) + + 'set(CMAKE_INCLUDE_CURRENT_DIR ON)\n\n' + + 'set(CMAKE_AUTOMOC ON)\n' + + 'set(CMAKE_AUTORCC ON)\n' + + 'set(CMAKE_AUTOUIC ON)\n\n' + + 'set(INSTALL_EXAMPLEDIR "examples")\n\n') + + (public_libs, private_libs) = extract_cmake_libraries(scope) + write_find_package_section(cm_fh, public_libs, private_libs, indent=indent) + + add_executable = 'add_{}executable({}'.format("qt_gui_" if gui else "", binary_name); + + write_all_source_file_lists(cm_fh, scope, add_executable, indent=0, extra_keys=['RESOURCES']) + + cm_fh.write(')\n') + + write_include_paths(cm_fh, scope, 'target_include_directories({} PUBLIC'.format(binary_name), + indent=0, footer=')') + write_defines(cm_fh, scope, 'target_compile_definitions({} PUBLIC'.format(binary_name), + indent=0, footer=')') + write_list(cm_fh, private_libs, '', indent=indent, + header='target_link_libraries({} PRIVATE\n'.format(binary_name), footer=')') + write_list(cm_fh, public_libs, '', indent=indent, + header='target_link_libraries({} PUBLIC\n'.format(binary_name), footer=')') + write_compile_options(cm_fh, scope, 'target_compile_options({}'.format(binary_name), + indent=0, footer=')') + + cm_fh.write('\ninstall(TARGETS {}\n'.format(binary_name) + + ' RUNTIME DESTINATION "${INSTALL_EXAMPLEDIR}"\n' + + ' BUNDLE DESTINATION "${INSTALL_EXAMPLEDIR}"\n' + + ' LIBRARY DESTINATION "${INSTALL_EXAMPLEDIR}"\n' + + ')\n') + + +def write_plugin(cm_fh, scope, *, indent: int = 0): + plugin_name = scope.TARGET + assert plugin_name + + extra = [] + + plugin_type = scope.get_string('PLUGIN_TYPE') + if plugin_type: + extra.append('TYPE {}'.format(plugin_type)) + + plugin_class_name = scope.get_string('PLUGIN_CLASS_NAME') + if plugin_class_name: + extra.append('CLASS_NAME {}'.format(plugin_class_name)) + + write_main_part(cm_fh, plugin_name, 'Plugin', 'add_qt_plugin', scope, + indent=indent, extra_lines=extra, known_libraries={}, extra_keys=[]) + + +def handle_app_or_lib(scope: Scope, cm_fh: typing.IO[str], *, + indent: int = 0, is_example: bool=False) -> None: + assert scope.TEMPLATE in ('app', 'lib') + + is_lib = scope.TEMPLATE == 'lib' + is_plugin = any('qt_plugin' == s for s in scope.get('_LOADED')) + + if is_lib or 'qt_module' in scope.get('_LOADED'): + assert not is_example + write_module(cm_fh, scope, indent=indent) + elif is_plugin: + assert not is_example + write_plugin(cm_fh, scope, indent=indent) + elif 'qt_tool' in scope.get('_LOADED'): + assert not is_example + write_tool(cm_fh, scope, indent=indent) + else: + if 'testcase' in scope.get('CONFIG') \ + or 'testlib' in scope.get('CONFIG'): + assert not is_example + write_test(cm_fh, scope, indent=indent) + else: + config = scope.get('CONFIG') + gui = all(val not in config for val in ['console', 'cmdline']) + if is_example: + write_example(cm_fh, scope, gui, indent=indent) + else: + write_binary(cm_fh, scope, gui, indent=indent) + + ind = spaces(indent) + write_source_file_list(cm_fh, scope, '', + ['QMAKE_DOCS',], + indent, + header = 'add_qt_docs(\n', + footer = ')\n') + + +def cmakeify_scope(scope: Scope, cm_fh: typing.IO[str], *, + indent: int = 0, is_example: bool=False) -> None: + template = scope.TEMPLATE + if template == 'subdirs': + handle_subdir(scope, cm_fh, indent=indent, is_example=is_example) + elif template in ('app', 'lib'): + handle_app_or_lib(scope, cm_fh, indent=indent, is_example=is_example) + else: + print(' XXXX: {}: Template type {} not yet supported.' + .format(scope.file, template)) + + +def generate_new_cmakelists(scope: Scope, *, is_example: bool=False) -> None: + print('Generating CMakeLists.gen.txt') + with open(scope.generated_cmake_lists_path, 'w') as cm_fh: + assert scope.file + cm_fh.write('# Generated from {}.\n\n' + .format(os.path.basename(scope.file))) + cmakeify_scope(scope, cm_fh, is_example=is_example) + + +def do_include(scope: Scope, *, debug: bool = False) -> None: + for c in scope.children: + do_include(c) + + for include_file in scope.get_files('_INCLUDED', is_include=True): + if not include_file: + continue + if not os.path.isfile(include_file): + print(' XXXX: Failed to include {}.'.format(include_file)) + continue + + include_result = parseProFile(include_file, debug=debug) + include_scope \ + = Scope.FromDict(None, include_file, + include_result.asDict().get('statements'), + '', scope.basedir) # This scope will be merged into scope! + + do_include(include_scope) + + scope.merge(include_scope) + + +def copy_generated_file_to_final_location(scope: Scope, keep_temporary_files=False) -> None: + print('Copying {} to {}'.format(scope.generated_cmake_lists_path, + scope.original_cmake_lists_path)) + copyfile(scope.generated_cmake_lists_path, scope.original_cmake_lists_path) + if not keep_temporary_files: + os.remove(scope.generated_cmake_lists_path) + + +def main() -> None: + args = _parse_commandline() + + debug_parsing = args.debug_parser or args.debug + + backup_current_dir = os.getcwd() + + for file in args.files: + new_current_dir = os.path.dirname(file) + file_relative_path = os.path.basename(file) + if new_current_dir: + os.chdir(new_current_dir) + + parseresult = parseProFile(file_relative_path, debug=debug_parsing) + + if args.debug_parse_result or args.debug: + print('\n\n#### Parser result:') + print(parseresult) + print('\n#### End of parser result.\n') + if args.debug_parse_dictionary or args.debug: + print('\n\n####Parser result dictionary:') + print(parseresult.asDict()) + print('\n#### End of parser result dictionary.\n') + + file_scope = Scope.FromDict(None, file_relative_path, + parseresult.asDict().get('statements')) + + if args.debug_pro_structure or args.debug: + print('\n\n#### .pro/.pri file structure:') + file_scope.dump() + print('\n#### End of .pro/.pri file structure.\n') + + do_include(file_scope, debug=debug_parsing) + + if args.debug_full_pro_structure or args.debug: + print('\n\n#### Full .pro/.pri file structure:') + file_scope.dump() + print('\n#### End of full .pro/.pri file structure.\n') + + generate_new_cmakelists(file_scope, is_example=args.is_example) + + copy_generated_file = True + if not args.skip_special_case_preservation: + debug_special_case = args.debug_special_case_preservation or args.debug + handler = SpecialCaseHandler(file_scope.original_cmake_lists_path, + file_scope.generated_cmake_lists_path, + file_scope.basedir, + keep_temporary_files=args.keep_temporary_files, + debug=debug_special_case) + + copy_generated_file = handler.handle_special_cases() + + if copy_generated_file: + copy_generated_file_to_final_location(file_scope, + keep_temporary_files=args.keep_temporary_files) + os.chdir(backup_current_dir) + + +if __name__ == '__main__': + main() diff --git a/util/cmake/pro_conversion_rate.py b/util/cmake/pro_conversion_rate.py new file mode 100755 index 0000000000..740e834ca5 --- /dev/null +++ b/util/cmake/pro_conversion_rate.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from __future__ import annotations + +""" +This utility script shows statistics about +converted .pro -> CMakeLists.txt files. + +To execute: python3 pro_conversion_rate.py <src dir> +where <src dir> can be any qt source directory. For better statistics, +specify a module root source dir (like ./qtbase or ./qtsvg). + +""" + +from argparse import ArgumentParser + +import os +import typing +from timeit import default_timer + + +def _parse_commandline(): + parser = ArgumentParser(description='Find pro files for which there are no CMakeLists.txt.') + parser.add_argument('source_directory', metavar='<src dir>', type=str, + help='The source directory') + + return parser.parse_args() + + +class Blacklist: + """ Class to check if a certain dir_name / dir_path is blacklisted """ + + def __init__(self, names: typing.List[str], path_parts: typing.List[str]): + self.names = names + self.path_parts = path_parts + + # The lookup algorithm + self.lookup = self.is_blacklisted_part + self.tree = None + + try: + # If package is available, use Aho-Corasick algorithm, + from ahocorapy.keywordtree import KeywordTree + self.tree = KeywordTree(case_insensitive=True) + + for p in self.path_parts: + self.tree.add(p) + self.tree.finalize() + + self.lookup = self.is_blacklisted_part_aho + except ImportError: + pass + + def is_blacklisted(self, dir_name: str, dir_path: str) -> bool: + # First check if exact dir name is blacklisted. + if dir_name in self.names: + return True + + # Check if a path part is blacklisted (e.g. util/cmake) + return self.lookup(dir_path) + + def is_blacklisted_part(self, dir_path: str) -> bool: + if any(part in dir_path for part in self.path_parts): + return True + return False + + def is_blacklisted_part_aho(self, dir_path: str) -> bool: + return self.tree.search(dir_path) is not None + + +def recursive_scan(path: str, extension: str, result_paths: typing.List[str], blacklist: Blacklist): + """ Find files ending with a certain extension, filtering out blacklisted entries """ + try: + for entry in os.scandir(path): + entry: os.DirEntry = entry + + if entry.is_file() and entry.path.endswith(extension): + result_paths.append(entry.path) + elif entry.is_dir(): + if blacklist.is_blacklisted(entry.name, entry.path): + continue + recursive_scan(entry.path, extension, result_paths, blacklist) + except Exception as e: + print(e) + + +def check_for_cmake_project(pro_path: str) -> bool: + pro_dir_name = os.path.dirname(pro_path) + cmake_project_path = os.path.join(pro_dir_name, "CMakeLists.txt") + return os.path.exists(cmake_project_path) + + +def compute_stats(src_path: str, pros_with_missing_project: typing.List[str], + total_pros: int, existing_pros: int, missing_pros: int) -> dict: + stats = {} + stats['total projects'] = {'label': 'Total pro files found', + 'value': total_pros} + stats['existing projects'] = {'label': 'Existing CMakeLists.txt files found', + 'value': existing_pros} + stats['missing projects'] = {'label': 'Missing CMakeLists.txt files found', + 'value': missing_pros} + stats['missing examples'] = {'label': 'Missing examples', 'value': 0} + stats['missing tests'] = {'label': 'Missing tests', 'value': 0} + stats['missing src'] = {'label': 'Missing src/**/**', 'value': 0} + stats['missing plugins'] = {'label': 'Missing plugins', 'value': 0} + + for p in pros_with_missing_project: + rel_path = os.path.relpath(p, src_path) + if rel_path.startswith("examples"): + stats['missing examples']['value'] += 1 + elif rel_path.startswith("tests"): + stats['missing tests']['value'] += 1 + elif rel_path.startswith(os.path.join("src", "plugins")): + stats['missing plugins']['value'] += 1 + elif rel_path.startswith("src"): + stats['missing src']['value'] += 1 + + for stat in stats: + if stats[stat]['value'] > 0: + stats[stat]['percentage'] = round(stats[stat]['value'] * 100 / total_pros, 2) + return stats + + +def print_stats(src_path: str, pros_with_missing_project: typing.List[str], stats: dict, + scan_time: float, script_time: float): + + if stats['total projects']['value'] == 0: + print("No .pro files found. Did you specify a correct source path?") + return + + if stats['total projects']['value'] == stats['existing projects']['value']: + print("All projects were converted.") + else: + print("Missing CMakeLists.txt files for the following projects: \n") + + for p in pros_with_missing_project: + rel_path = os.path.relpath(p, src_path) + print(rel_path) + + print("\nStatistics: \n") + + for stat in stats: + if stats[stat]['value'] > 0: + print("{:<40}: {} ({}%)".format(stats[stat]['label'], + stats[stat]['value'], + stats[stat]['percentage'])) + + print("\n{:<40}: {:.10f} seconds".format("Scan time", scan_time)) + print("{:<40}: {:.10f} seconds".format("Total script time", script_time)) + + +def main(): + args = _parse_commandline() + src_path = os.path.abspath(args.source_directory) + pro_paths = [] + + extension = ".pro" + + blacklist_names = ["config.tests", "doc", "3rdparty", "angle"] + blacklist_path_parts = [ + os.path.join("util", "cmake") + ] + + script_start_time = default_timer() + blacklist = Blacklist(blacklist_names, blacklist_path_parts) + + scan_time_start = default_timer() + recursive_scan(src_path, extension, pro_paths, blacklist) + scan_time_end = default_timer() + scan_time = scan_time_end - scan_time_start + + total_pros = len(pro_paths) + + pros_with_missing_project = [] + for pro_path in pro_paths: + if not check_for_cmake_project(pro_path): + pros_with_missing_project.append(pro_path) + + missing_pros = len(pros_with_missing_project) + existing_pros = total_pros - missing_pros + + stats = compute_stats(src_path, pros_with_missing_project, total_pros, existing_pros, + missing_pros) + script_end_time = default_timer() + script_time = script_end_time - script_start_time + + print_stats(src_path, pros_with_missing_project, stats, scan_time, script_time) + + +if __name__ == '__main__': + main() diff --git a/util/cmake/run_pro2cmake.py b/util/cmake/run_pro2cmake.py new file mode 100755 index 0000000000..bc64fb3fbb --- /dev/null +++ b/util/cmake/run_pro2cmake.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +import glob +import os +import subprocess +import concurrent.futures +import typing +import argparse +from argparse import ArgumentParser + + +def parse_command_line(): + parser = ArgumentParser(description='Run pro2cmake on all .pro files recursively in given path.') + parser.add_argument('--only-existing', dest='only_existing', action='store_true', + help='Run pro2cmake only on .pro files that already have a CMakeLists.txt.') + parser.add_argument('--only-qtbase-main-modules', dest='only_qtbase_main_modules', action='store_true', + help='Run pro2cmake only on the main modules in qtbase.') + parser.add_argument('path', metavar='<path>', type=str, + help='The path where to look for .pro files.') + + return parser.parse_args() + + +def find_all_pro_files(base_path: str, args: argparse.Namespace): + + def sorter(pro_file: str) -> str: + """ Sorter that tries to prioritize main pro files in a directory. """ + pro_file_without_suffix = pro_file.rsplit('/', 1)[-1][:-4] + dir_name = os.path.dirname(pro_file) + if dir_name.endswith('/' + pro_file_without_suffix): + return dir_name + return dir_name + "/__" + pro_file + + all_files = [] + previous_dir_name: str = None + + print('Finding .pro files.') + glob_result = glob.glob(os.path.join(base_path, '**/*.pro'), recursive=True) + + def cmake_lists_exists_filter(path): + path_dir_name = os.path.dirname(path) + if os.path.exists(os.path.join(path_dir_name, 'CMakeLists.txt')): + return True + return False + + def qtbase_main_modules_filter(path): + main_modules = [ + 'corelib', + 'network', + 'gui', + 'widgets', + 'testlib', + 'printsupport', + 'opengl', + 'sql', + 'dbus', + 'concurrent', + 'xml', + ] + path_suffixes = ['src/{}/{}.pro'.format(m, m, '.pro') for m in main_modules] + + for path_suffix in path_suffixes: + if path.endswith(path_suffix): + return True + return False + + filter_result = glob_result + filter_func = None + if args.only_existing: + filter_func = cmake_lists_exists_filter + elif args.only_qtbase_main_modules: + filter_func = qtbase_main_modules_filter + + if filter_func: + print('Filtering.') + filter_result = [p for p in filter_result if filter_func(p)] + + for pro_file in sorted(filter_result, key=sorter): + dir_name = os.path.dirname(pro_file) + if dir_name == previous_dir_name: + print("Skipping:", pro_file) + else: + all_files.append(pro_file) + previous_dir_name = dir_name + return all_files + + +def run(all_files: typing.List[str], pro2cmake: str, args: argparse.Namespace) -> typing.List[str]: + failed_files = [] + files_count = len(all_files) + workers = (os.cpu_count() or 1) + + if args.only_qtbase_main_modules: + # qtbase main modules take longer than usual to process. + workers = 2 + + with concurrent.futures.ThreadPoolExecutor(max_workers=workers, initializer=os.nice, initargs=(10,)) as pool: + print('Firing up thread pool executor.') + + def _process_a_file(data: typing.Tuple[str, int, int]) -> typing.Tuple[int, str, str]: + filename, index, total = data + result = subprocess.run((pro2cmake, os.path.basename(filename)), + cwd=os.path.dirname(filename), + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout = 'Converted[{}/{}]: {}\n'.format(index, total, filename) + return result.returncode, filename, stdout + result.stdout.decode() + + for return_code, filename, stdout in pool.map(_process_a_file, + zip(all_files, + range(1, files_count + 1), + (files_count for _ in all_files))): + if return_code: + failed_files.append(filename) + print(stdout) + + return failed_files + + +def main() -> None: + args = parse_command_line() + + script_path = os.path.dirname(os.path.abspath(__file__)) + pro2cmake = os.path.join(script_path, 'pro2cmake.py') + base_path = args.path + + all_files = find_all_pro_files(base_path, args) + files_count = len(all_files) + failed_files = run(all_files, pro2cmake, args) + if len(all_files) == 0: + print('No files found.') + + if failed_files: + print('The following files were not successfully ' + 'converted ({} of {}):'.format(len(failed_files), files_count)) + for f in failed_files: + print(' "{}"'.format(f)) + + +if __name__ == '__main__': + main() diff --git a/util/cmake/special_case_helper.py b/util/cmake/special_case_helper.py new file mode 100644 index 0000000000..b9cb93dce0 --- /dev/null +++ b/util/cmake/special_case_helper.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +""" +This is a helper script that takes care of reapplying special case +modifications when regenerating a CMakeLists.txt file using +pro2cmake.py. + +It has two modes of operation: +1) Dumb "special case" block removal and re-application. +2) Smart "special case" diff application, using a previously generated + "clean" CMakeLists.txt as a source. "clean" in this case means a + generated file which has no "special case" modifications. + +Both modes use a temporary git repository to compute and reapply +"special case" diffs. + +For the first mode to work, the developer has to mark changes +with "# special case" markers on every line they want to keep. Or +enclose blocks of code they want to keep between "# special case begin" +and "# special case end" markers. + +For example: + +SOURCES + foo.cpp + bar.cpp # special case + +SOURCES + foo1.cpp + foo2.cpp + # special case begin + foo3.cpp + foo4.cpp + # special case end + +The second mode, as mentioned, requires a previous "clean" +CMakeLists.txt file. + +The script can then compute the exact diff between +a "clean" and "modified" (with special cases) file, and reapply that +diff to a newly generated "CMakeLists.txt" file. + +This implies that we always have to keep a "clean" file alongside the +"modified" project file for each project (corelib, gui, etc.) So we +have to commit both files to the repository. + +If there is no such "clean" file, we can use the first operation mode +to generate one. After that, we only have to use the second operation +mode for the project file in question. + +When the script is used, the developer only has to take care of fixing +the newly generated "modified" file. The "clean" file is automatically +handled and git add'ed by the script, and will be committed together +with the "modified" file. + + +""" + +import re +import os +import subprocess +import filecmp +import time +import typing +import stat + +from shutil import copyfile +from shutil import rmtree + + +def remove_special_cases(original: str) -> str: + # Remove content between the following markers + # '# special case begin' and '# special case end'. + # This also remove the markers. + replaced = re.sub(r'\n[^#\n]*?#[^\n]*?special case begin.*?#[^\n]*special case end[^\n]*?\n', + '\n', + original, + 0, + re.DOTALL) + + # Remove individual lines that have the "# special case" marker. + replaced = re.sub(r'\n.*#.*special case[^\n]*\n', '\n', replaced) + return replaced + + +def read_content_from_file(file_path: str) -> str: + with open(file_path, 'r') as file_fd: + content = file_fd.read() + return content + + +def write_content_to_file(file_path: str, content: str) -> None: + with open(file_path, 'w') as file_fd: + file_fd.write(content) + + +def resolve_simple_git_conflicts(file_path: str, debug=False) -> None: + content = read_content_from_file(file_path) + # If the conflict represents the addition of a new content hunk, + # keep the content and remove the conflict markers. + if debug: + print('Resolving simple conflicts automatically.') + replaced = re.sub(r'\n<<<<<<< HEAD\n=======(.+?)>>>>>>> master\n', r'\1', content, 0, re.DOTALL) + write_content_to_file(file_path, replaced) + + +def copyfile_log(src: str, dst: str, debug=False): + if debug: + print('Copying {} to {}.'.format(src, dst)) + copyfile(src, dst) + + +def check_if_git_in_path() -> bool: + is_win = os.name == 'nt' + for path in os.environ['PATH'].split(os.pathsep): + git_path = os.path.join(path, 'git') + if is_win: + git_path += '.exe' + if os.path.isfile(git_path) and os.access(git_path, os.X_OK): + return True + return False + + +def run_process_quiet(args_string: str, debug=False) -> bool: + if debug: + print('Running command: "{}\"'.format(args_string)) + args_list = args_string.split() + try: + subprocess.run(args_list, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + # git merge with conflicts returns with exit code 1, but that's not + # an error for us. + if 'git merge' not in args_string: + print('Error while running: "{}"\n{}'.format(args_string, e.stdout)) + return False + return True + + +def does_file_have_conflict_markers(file_path: str, debug=False) -> bool: + if debug: + print('Checking if {} has no leftover conflict markers.'.format(file_path)) + content_actual = read_content_from_file(file_path) + if '<<<<<<< HEAD' in content_actual: + print('Conflict markers found in {}. ' + 'Please remove or solve them first.'.format(file_path)) + return True + return False + + +def create_file_with_no_special_cases(original_file_path: str, no_special_cases_file_path: str, debug=False): + """ + Reads content of original CMakeLists.txt, removes all content + between "# special case" markers or lines, saves the result into a + new file. + """ + content_actual = read_content_from_file(original_file_path) + if debug: + print('Removing special case blocks from {}.'.format(original_file_path)) + content_no_special_cases = remove_special_cases(content_actual) + + if debug: + print('Saving original contents of {} ' + 'with removed special case blocks to {}'.format(original_file_path, + no_special_cases_file_path)) + write_content_to_file(no_special_cases_file_path, content_no_special_cases) + + +def rm_tree_on_error_handler(func: typing.Callable[..., None], + path: str, exception_info: tuple): + # If the path is read only, try to make it writable, and try + # to remove the path again. + if not os.access(path, os.W_OK): + os.chmod(path, stat.S_IWRITE) + func(path) + else: + print('Error while trying to remove path: {}. Exception: {}'.format(path, exception_info)) + + +class SpecialCaseHandler(object): + + def __init__(self, + original_file_path: str, + generated_file_path: str, + base_dir: str, + keep_temporary_files=False, + debug=False) -> None: + self.base_dir = base_dir + self.original_file_path = original_file_path + self.generated_file_path = generated_file_path + self.keep_temporary_files = keep_temporary_files + self.use_heuristic = False + self.debug = debug + + @property + def prev_file_path(self) -> str: + return os.path.join(self.base_dir, '.prev_CMakeLists.txt') + + @property + def post_merge_file_path(self) -> str: + return os.path.join(self.base_dir, 'CMakeLists-post-merge.txt') + + @property + def no_special_file_path(self) -> str: + return os.path.join(self.base_dir, 'CMakeLists.no-special.txt') + + def apply_git_merge_magic(self, no_special_cases_file_path: str) -> None: + # Create new folder for temporary repo, and ch dir into it. + repo = os.path.join(self.base_dir, 'tmp_repo') + repo_absolute_path = os.path.abspath(repo) + txt = 'CMakeLists.txt' + + try: + os.mkdir(repo) + current_dir = os.getcwd() + os.chdir(repo) + except Exception as e: + print('Failed to create temporary directory for temporary git repo. Exception: {}' + .format(e)) + raise e + + generated_file_path = os.path.join("..", self.generated_file_path) + original_file_path = os.path.join("..", self.original_file_path) + no_special_cases_file_path = os.path.join("..", no_special_cases_file_path) + post_merge_file_path = os.path.join("..", self.post_merge_file_path) + + try: + # Create new repo with the "clean" CMakeLists.txt file. + run_process_quiet('git init .', debug=self.debug) + run_process_quiet('git config user.name fake', debug=self.debug) + run_process_quiet('git config user.email fake@fake', debug=self.debug) + copyfile_log(no_special_cases_file_path, txt, debug=self.debug) + run_process_quiet('git add {}'.format(txt), debug=self.debug) + run_process_quiet('git commit -m no_special', debug=self.debug) + run_process_quiet('git checkout -b no_special', debug=self.debug) + + # Copy the original "modified" file (with the special cases) + # and make a new commit. + run_process_quiet('git checkout -b original', debug=self.debug) + copyfile_log(original_file_path, txt, debug=self.debug) + run_process_quiet('git add {}'.format(txt), debug=self.debug) + run_process_quiet('git commit -m original', debug=self.debug) + + # Checkout the commit with "clean" file again, and create a + # new branch. + run_process_quiet('git checkout no_special', debug=self.debug) + run_process_quiet('git checkout -b newly_generated', debug=self.debug) + + # Copy the new "modified" file and make a commit. + copyfile_log(generated_file_path, txt, debug=self.debug) + run_process_quiet('git add {}'.format(txt), debug=self.debug) + run_process_quiet('git commit -m newly_generated', debug=self.debug) + + # Merge the "old" branch with modifications into the "new" + # branch with the newly generated file. + run_process_quiet('git merge original', debug=self.debug) + + # Resolve some simple conflicts (just remove the markers) + # for cases that don't need intervention. + resolve_simple_git_conflicts(txt, debug=self.debug) + + # Copy the resulting file from the merge. + copyfile_log(txt, post_merge_file_path) + except Exception as e: + print('Git merge conflict resolution process failed. Exception: {}'.format(e)) + raise e + finally: + os.chdir(current_dir) + + # Remove the temporary repo. + try: + if not self.keep_temporary_files: + rmtree(repo_absolute_path, onerror=rm_tree_on_error_handler) + except Exception as e: + print('Error removing temporary repo. Exception: {}'.format(e)) + + def save_next_clean_file(self): + files_are_equivalent = filecmp.cmp(self.generated_file_path, self.post_merge_file_path) + + if not files_are_equivalent: + # Before overriding the generated file with the post + # merge result, save the new "clean" file for future + # regenerations. + copyfile_log(self.generated_file_path, self.prev_file_path, debug=self.debug) + + # Attempt to git add until we succeed. It can fail when + # run_pro2cmake executes pro2cmake in multiple threads, and git + # has acquired the index lock. + success = False + failed_once = False + i = 0 + while not success and i < 20: + success = run_process_quiet("git add {}".format(self.prev_file_path), + debug=self.debug) + if not success: + failed_once = True + i += 1 + time.sleep(0.1) + + if failed_once and not success: + print('Retrying git add, the index.lock was probably acquired.') + if failed_once and success: + print('git add succeeded.') + elif failed_once and not success: + print('git add failed. Make sure to git add {} yourself.'.format( + self.prev_file_path)) + + def handle_special_cases_helper(self) -> bool: + """ + Uses git to reapply special case modifications to the "new" + generated CMakeLists.gen.txt file. + + If use_heuristic is True, a new file is created from the + original file, with special cases removed. + + If use_heuristic is False, an existing "clean" file with no + special cases is used from a previous conversion. The "clean" + file is expected to be in the same folder as the original one. + """ + try: + if does_file_have_conflict_markers(self.original_file_path): + return False + + if self.use_heuristic: + create_file_with_no_special_cases(self.original_file_path, + self.no_special_file_path) + no_special_cases_file_path = self.no_special_file_path + else: + no_special_cases_file_path = self.prev_file_path + + if self.debug: + print('Using git to reapply special case modifications to newly generated {} ' + 'file'.format(self.generated_file_path)) + + self.apply_git_merge_magic(no_special_cases_file_path) + self.save_next_clean_file() + + copyfile_log(self.post_merge_file_path, self.generated_file_path) + if not self.keep_temporary_files: + os.remove(self.post_merge_file_path) + + print('Special case reapplication using git is complete. ' + 'Make sure to fix remaining conflict markers.') + + except Exception as e: + print('Error occurred while trying to reapply special case modifications: {}'.format(e)) + return False + finally: + if not self.keep_temporary_files and self.use_heuristic: + os.remove(self.no_special_file_path) + + return True + + def handle_special_cases(self) -> bool: + original_file_exists = os.path.isfile(self.original_file_path) + prev_file_exists = os.path.isfile(self.prev_file_path) + self.use_heuristic = not prev_file_exists + + git_available = check_if_git_in_path() + keep_special_cases = original_file_exists and git_available + + if not git_available: + print('You need to have git in PATH in order to reapply the special ' + 'case modifications.') + + copy_generated_file = True + + if keep_special_cases: + copy_generated_file = self.handle_special_cases_helper() + + return copy_generated_file diff --git a/util/cmake/tests/__init__.py b/util/cmake/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/util/cmake/tests/__init__.py diff --git a/util/cmake/tests/data/comment_scope.pro b/util/cmake/tests/data/comment_scope.pro new file mode 100644 index 0000000000..be43cad37d --- /dev/null +++ b/util/cmake/tests/data/comment_scope.pro @@ -0,0 +1,6 @@ +# QtCore can't be compiled with -Wl,-no-undefined because it uses the "environ" +# variable and on FreeBSD and OpenBSD, this variable is in the final executable itself. +# OpenBSD 6.0 will include environ in libc. +freebsd|openbsd: QMAKE_LFLAGS_NOUNDEF = + +include(animation/animation.pri) diff --git a/util/cmake/tests/data/complex_assign.pro b/util/cmake/tests/data/complex_assign.pro new file mode 100644 index 0000000000..d251afcdd5 --- /dev/null +++ b/util/cmake/tests/data/complex_assign.pro @@ -0,0 +1,2 @@ +qmake-clean.commands += (cd qmake && $(MAKE) clean ":-(==)-:" '(Foo)' ) + diff --git a/util/cmake/tests/data/complex_condition.pro b/util/cmake/tests/data/complex_condition.pro new file mode 100644 index 0000000000..bc3369bd63 --- /dev/null +++ b/util/cmake/tests/data/complex_condition.pro @@ -0,0 +1,4 @@ +!system("dbus-send --session --type=signal / local.AutotestCheck.Hello >$$QMAKE_SYSTEM_NULL_DEVICE 2>&1") { + SOURCES = dbus.cpp +} + diff --git a/util/cmake/tests/data/complex_values.pro b/util/cmake/tests/data/complex_values.pro new file mode 100644 index 0000000000..4d747c1dd7 --- /dev/null +++ b/util/cmake/tests/data/complex_values.pro @@ -0,0 +1,22 @@ +linux:!static { + precompile_header { + # we'll get an error if we just use SOURCES += + no_pch_assembler.commands = $$QMAKE_CC -c $(CFLAGS) $(INCPATH) ${QMAKE_FILE_IN} -o ${QMAKE_FILE_OUT} + no_pch_assembler.dependency_type = TYPE_C + no_pch_assembler.output = ${QMAKE_VAR_OBJECTS_DIR}${QMAKE_FILE_BASE}$${first(QMAKE_EXT_OBJ)} + no_pch_assembler.input = NO_PCH_ASM + no_pch_assembler.name = compiling[no_pch] ${QMAKE_FILE_IN} + silent: no_pch_assembler.commands = @echo compiling[no_pch] ${QMAKE_FILE_IN} && $$no_pch_assembler.commands + CMAKE_ANGLE_GLES2_IMPLIB_RELEASE = libGLESv2.$${QMAKE_EXTENSION_STATICLIB} + HOST_BINS = $$[QT_HOST_BINS] + CMAKE_HOST_DATA_DIR = $$[QT_HOST_DATA/src]/ + TR_EXCLUDE += ../3rdparty/* + + QMAKE_EXTRA_COMPILERS += no_pch_assembler + NO_PCH_ASM += global/minimum-linux.S + } else { + SOURCES += global/minimum-linux.S + } + HEADERS += global/minimum-linux_p.h +} + diff --git a/util/cmake/tests/data/condition_without_scope.pro b/util/cmake/tests/data/condition_without_scope.pro new file mode 100644 index 0000000000..2aa1237c12 --- /dev/null +++ b/util/cmake/tests/data/condition_without_scope.pro @@ -0,0 +1,2 @@ +write_file("a", contents)|error() + diff --git a/util/cmake/tests/data/contains_scope.pro b/util/cmake/tests/data/contains_scope.pro new file mode 100644 index 0000000000..0f51350a45 --- /dev/null +++ b/util/cmake/tests/data/contains_scope.pro @@ -0,0 +1,4 @@ +contains(DEFINES,QT_EVAL):include(eval.pri) + +HOST_BINS = $$[QT_HOST_BINS] + diff --git a/util/cmake/tests/data/definetest.pro b/util/cmake/tests/data/definetest.pro new file mode 100644 index 0000000000..76b63d239f --- /dev/null +++ b/util/cmake/tests/data/definetest.pro @@ -0,0 +1,6 @@ +defineTest(pathIsAbsolute) { + p = $$clean_path($$1) + !isEmpty(p):isEqual(p, $$absolute_path($$p)): return(true) + return(false) +} + diff --git a/util/cmake/tests/data/else.pro b/util/cmake/tests/data/else.pro new file mode 100644 index 0000000000..bbf9c5ac9f --- /dev/null +++ b/util/cmake/tests/data/else.pro @@ -0,0 +1,6 @@ + +linux { + SOURCES += a.cpp +} else { + SOURCES += b.cpp +} diff --git a/util/cmake/tests/data/else2.pro b/util/cmake/tests/data/else2.pro new file mode 100644 index 0000000000..f2ef36ec28 --- /dev/null +++ b/util/cmake/tests/data/else2.pro @@ -0,0 +1,4 @@ + +osx: A = 1 +else: win32: B = 2 +else: C = 3 diff --git a/util/cmake/tests/data/else3.pro b/util/cmake/tests/data/else3.pro new file mode 100644 index 0000000000..0de9c2c1d9 --- /dev/null +++ b/util/cmake/tests/data/else3.pro @@ -0,0 +1,7 @@ +qtConfig(timezone) { + A = 1 +} else:win32 { + B = 2 +} else { + C = 3 +} diff --git a/util/cmake/tests/data/else4.pro b/util/cmake/tests/data/else4.pro new file mode 100644 index 0000000000..9ed676ccfa --- /dev/null +++ b/util/cmake/tests/data/else4.pro @@ -0,0 +1,6 @@ +qtConfig(timezone) { + A = 1 +} else:win32: B = 2 +else { + C = 3 +} diff --git a/util/cmake/tests/data/else5.pro b/util/cmake/tests/data/else5.pro new file mode 100644 index 0000000000..3de76af50a --- /dev/null +++ b/util/cmake/tests/data/else5.pro @@ -0,0 +1,10 @@ +# comments +qtConfig(timezone) { # bar + A = 1 +} else:win32 { + B = 2 # foo +} else { C = 3 +# baz + # foobar +} +# endcomment diff --git a/util/cmake/tests/data/else6.pro b/util/cmake/tests/data/else6.pro new file mode 100644 index 0000000000..9eaa834a19 --- /dev/null +++ b/util/cmake/tests/data/else6.pro @@ -0,0 +1,11 @@ +qtConfig(timezone) \ +{ + A = \ +1 +} \ +else:win32: \ +B = 2 +else: \ + C \ += 3 + diff --git a/util/cmake/tests/data/else7.pro b/util/cmake/tests/data/else7.pro new file mode 100644 index 0000000000..e663b1c05e --- /dev/null +++ b/util/cmake/tests/data/else7.pro @@ -0,0 +1,2 @@ +msvc:equals(QT_ARCH, i386): QMAKE_LFLAGS += /BASE:0x65000000 + diff --git a/util/cmake/tests/data/else8.pro b/util/cmake/tests/data/else8.pro new file mode 100644 index 0000000000..6d4d5f01ed --- /dev/null +++ b/util/cmake/tests/data/else8.pro @@ -0,0 +1,5 @@ +qtConfig(timezone) { A = 1 } else:win32: {\ +B = 2 \ +} else: \ + C \ += 3 \ diff --git a/util/cmake/tests/data/escaped_value.pro b/util/cmake/tests/data/escaped_value.pro new file mode 100644 index 0000000000..7c95b1fc30 --- /dev/null +++ b/util/cmake/tests/data/escaped_value.pro @@ -0,0 +1,2 @@ +MODULE_AUX_INCLUDES = \ + \$\$QT_MODULE_INCLUDE_BASE/QtANGLE diff --git a/util/cmake/tests/data/for.pro b/util/cmake/tests/data/for.pro new file mode 100644 index 0000000000..5751432980 --- /dev/null +++ b/util/cmake/tests/data/for.pro @@ -0,0 +1,11 @@ +SOURCES = main.cpp +for (config, SIMD) { + uc = $$upper($$config) + DEFINES += QT_COMPILER_SUPPORTS_$${uc} + + add_cflags { + cflags = QMAKE_CFLAGS_$${uc} + !defined($$cflags, var): error("This compiler does not support $${uc}") + QMAKE_CXXFLAGS += $$eval($$cflags) + } +} diff --git a/util/cmake/tests/data/function_if.pro b/util/cmake/tests/data/function_if.pro new file mode 100644 index 0000000000..9af018f864 --- /dev/null +++ b/util/cmake/tests/data/function_if.pro @@ -0,0 +1,4 @@ +pathIsAbsolute($$CMAKE_HOST_DATA_DIR) { + CMAKE_HOST_DATA_DIR = $$[QT_HOST_DATA/src]/ +} + diff --git a/util/cmake/tests/data/include.pro b/util/cmake/tests/data/include.pro new file mode 100644 index 0000000000..22d8a40919 --- /dev/null +++ b/util/cmake/tests/data/include.pro @@ -0,0 +1,3 @@ +A = 42 +include(foo) # load foo +B=23 diff --git a/util/cmake/tests/data/lc.pro b/util/cmake/tests/data/lc.pro new file mode 100644 index 0000000000..def80e7c95 --- /dev/null +++ b/util/cmake/tests/data/lc.pro @@ -0,0 +1,10 @@ +TEMPLATE=subdirs +SUBDIRS=\ + qmacstyle \ + qstyle \ + qstyleoption \ + qstylesheetstyle \ + +!qtConfig(private_tests): SUBDIRS -= \ + qstylesheetstyle \ + diff --git a/util/cmake/tests/data/lc_with_comment.pro b/util/cmake/tests/data/lc_with_comment.pro new file mode 100644 index 0000000000..176913dfc8 --- /dev/null +++ b/util/cmake/tests/data/lc_with_comment.pro @@ -0,0 +1,22 @@ +SUBDIRS = \ +# dds \ + tga \ + wbmp + +MYVAR = foo # comment +MYVAR = foo2# comment +MYVAR = foo3# comment # + +MYVAR = foo4# comment # + +## +# +# +## + + # + # +# + # # + +MYVAR = foo5# comment # # diff --git a/util/cmake/tests/data/load.pro b/util/cmake/tests/data/load.pro new file mode 100644 index 0000000000..c9717e9832 --- /dev/null +++ b/util/cmake/tests/data/load.pro @@ -0,0 +1,3 @@ +A = 42 +load(foo)# load foo +B=23 diff --git a/util/cmake/tests/data/multi_condition_divided_by_lc.pro b/util/cmake/tests/data/multi_condition_divided_by_lc.pro new file mode 100644 index 0000000000..23254231df --- /dev/null +++ b/util/cmake/tests/data/multi_condition_divided_by_lc.pro @@ -0,0 +1,3 @@ +equals(a): \ + greaterThan(a):flags += 1 + diff --git a/util/cmake/tests/data/multiline_assign.pro b/util/cmake/tests/data/multiline_assign.pro new file mode 100644 index 0000000000..42a3d0a674 --- /dev/null +++ b/util/cmake/tests/data/multiline_assign.pro @@ -0,0 +1,4 @@ +A = 42 \ + 43 \ + 44 +B=23 diff --git a/util/cmake/tests/data/nested_function_calls.pro b/util/cmake/tests/data/nested_function_calls.pro new file mode 100644 index 0000000000..5ecc53f1cc --- /dev/null +++ b/util/cmake/tests/data/nested_function_calls.pro @@ -0,0 +1,2 @@ +requires(qtConfig(dlopen)) + diff --git a/util/cmake/tests/data/quoted.pro b/util/cmake/tests/data/quoted.pro new file mode 100644 index 0000000000..61682aa0d0 --- /dev/null +++ b/util/cmake/tests/data/quoted.pro @@ -0,0 +1,5 @@ +if(linux*|hurd*):!cross_compile:!static:!*-armcc* { + prog=$$quote(if (/program interpreter: (.*)]/) { print $1; }) + DEFINES += ELF_INTERPRETER=\\\"$$system(LC_ALL=C readelf -l /bin/ls | perl -n -e \'$$prog\')\\\" +} + diff --git a/util/cmake/tests/data/single_line_for.pro b/util/cmake/tests/data/single_line_for.pro new file mode 100644 index 0000000000..806d08a49c --- /dev/null +++ b/util/cmake/tests/data/single_line_for.pro @@ -0,0 +1,4 @@ +for(d, sd): \ + exists($$d/$${d}.pro): \ + SUBDIRS += $$d + diff --git a/util/cmake/tests/data/sql.pro b/util/cmake/tests/data/sql.pro new file mode 100644 index 0000000000..a9d7fc7c5a --- /dev/null +++ b/util/cmake/tests/data/sql.pro @@ -0,0 +1,3 @@ +TEMPLATE = subdirs +SUBDIRS = \ + kernel \ diff --git a/util/cmake/tests/data/standardpaths.pro b/util/cmake/tests/data/standardpaths.pro new file mode 100644 index 0000000000..4b45788e4f --- /dev/null +++ b/util/cmake/tests/data/standardpaths.pro @@ -0,0 +1,17 @@ +win32 { + !winrt { + SOURCES +=io/qstandardpaths_win.cpp + } else { + SOURCES +=io/qstandardpaths_winrt.cpp + } +} else:unix { + mac { + OBJECTIVE_SOURCES += io/qstandardpaths_mac.mm + } else:android:!android-embedded { + SOURCES += io/qstandardpaths_android.cpp + } else:haiku { + SOURCES += io/qstandardpaths_haiku.cpp + } else { + SOURCES += io/qstandardpaths_unix.cpp + } +} diff --git a/util/cmake/tests/data/unset.pro b/util/cmake/tests/data/unset.pro new file mode 100644 index 0000000000..7ffb0582f1 --- /dev/null +++ b/util/cmake/tests/data/unset.pro @@ -0,0 +1,2 @@ +unset(f16c_cxx) + diff --git a/util/cmake/tests/test_lc_fixup.py b/util/cmake/tests/test_lc_fixup.py new file mode 100755 index 0000000000..841e11615e --- /dev/null +++ b/util/cmake/tests/test_lc_fixup.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from pro2cmake import fixup_linecontinuation + + +def test_no_change(): + input = "test \\\nline2\n line3" + output = "test line2\n line3" + result = fixup_linecontinuation(input) + assert output == result + + +def test_fix(): + input = "test \\\t\nline2\\\n line3\\ \nline4 \\ \t\nline5\\\n\n\n" + output = "test line2 line3 line4 line5 \n\n" + result = fixup_linecontinuation(input) + assert output == result + + diff --git a/util/cmake/tests/test_logic_mapping.py b/util/cmake/tests/test_logic_mapping.py new file mode 100755 index 0000000000..c477aa8351 --- /dev/null +++ b/util/cmake/tests/test_logic_mapping.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from pro2cmake import simplify_condition + + +def validate_simplify(input: str, expected: str) -> None: + output = simplify_condition(input) + assert output == expected + + +def validate_simplify_unchanged(input: str) -> None: + validate_simplify(input, input) + + +def test_simplify_on(): + validate_simplify_unchanged('ON') + + +def test_simplify_off(): + validate_simplify_unchanged('OFF') + + +def test_simplify_not_on(): + validate_simplify('NOT ON', 'OFF') + + +def test_simplify_not_off(): + validate_simplify('NOT OFF', 'ON') + + +def test_simplify_isEmpty(): + validate_simplify_unchanged('isEmpty(foo)') + + +def test_simplify_not_isEmpty(): + validate_simplify_unchanged('NOT isEmpty(foo)') + + +def test_simplify_simple_and(): + validate_simplify_unchanged('QT_FEATURE_bar AND QT_FEATURE_foo') + + +def test_simplify_simple_or(): + validate_simplify_unchanged('QT_FEATURE_bar OR QT_FEATURE_foo') + + +def test_simplify_simple_not(): + validate_simplify_unchanged('NOT QT_FEATURE_foo') + + +def test_simplify_simple_and_reorder(): + validate_simplify('QT_FEATURE_foo AND QT_FEATURE_bar', 'QT_FEATURE_bar AND QT_FEATURE_foo') + + +def test_simplify_simple_or_reorder(): + validate_simplify('QT_FEATURE_foo OR QT_FEATURE_bar', 'QT_FEATURE_bar OR QT_FEATURE_foo') + + +def test_simplify_unix_or_win32(): + validate_simplify('WIN32 OR UNIX', 'ON') + + +def test_simplify_unix_or_win32_or_foobar_or_barfoo(): + validate_simplify('WIN32 OR UNIX OR foobar OR barfoo', 'ON') + + +def test_simplify_not_not_bar(): + validate_simplify(' NOT NOT bar ', 'bar') + + +def test_simplify_not_unix(): + validate_simplify('NOT UNIX', 'WIN32') + + +def test_simplify_not_win32(): + validate_simplify('NOT WIN32', 'UNIX') + + +def test_simplify_unix_and_win32(): + validate_simplify('WIN32 AND UNIX', 'OFF') + + +def test_simplify_unix_or_win32(): + validate_simplify('WIN32 OR UNIX', 'ON') + + +def test_simplify_unix_and_win32_or_foobar_or_barfoo(): + validate_simplify('WIN32 AND foobar AND UNIX AND barfoo', 'OFF') + + +def test_simplify_watchos_and_win32(): + validate_simplify('APPLE_WATCHOS AND WIN32', 'OFF') + + +def test_simplify_win32_and_watchos(): + validate_simplify('WIN32 AND APPLE_WATCHOS', 'OFF') + + +def test_simplify_apple_and_appleosx(): + validate_simplify('APPLE AND APPLE_OSX', 'APPLE_OSX') + + +def test_simplify_apple_or_appleosx(): + validate_simplify('APPLE OR APPLE_OSX', 'APPLE') + + +def test_simplify_apple_or_appleosx_level1(): + validate_simplify('foobar AND (APPLE OR APPLE_OSX )', 'APPLE AND foobar') + + +def test_simplify_apple_or_appleosx_level1_double(): + validate_simplify('foobar AND (APPLE OR APPLE_OSX )', 'APPLE AND foobar') + + +def test_simplify_apple_or_appleosx_level1_double_with_extra_spaces(): + validate_simplify('foobar AND (APPLE OR APPLE_OSX ) ' + 'AND ( APPLE_OSX OR APPLE )', 'APPLE AND foobar') + + +def test_simplify_apple_or_appleosx_level2(): + validate_simplify('foobar AND ( ( APPLE OR APPLE_WATCHOS ) ' + 'OR APPLE_OSX ) AND ( APPLE_OSX OR APPLE ) ' + 'AND ( (WIN32 OR WINRT) OR UNIX) ', 'APPLE AND foobar') + + +def test_simplify_not_apple_and_appleosx(): + validate_simplify('NOT APPLE AND APPLE_OSX', 'OFF') + + +def test_simplify_unix_and_bar_or_win32(): + validate_simplify('WIN32 AND bar AND UNIX', 'OFF') + + +def test_simplify_unix_or_bar_or_win32(): + validate_simplify('WIN32 OR bar OR UNIX', 'ON') + + +def test_simplify_complex_true(): + validate_simplify('WIN32 OR ( APPLE OR UNIX)', 'ON') + + +def test_simplify_apple_unix_freebsd(): + validate_simplify('( APPLE OR ( UNIX OR FREEBSD ))', 'UNIX') + + +def test_simplify_apple_unix_freebsd_foobar(): + validate_simplify('( APPLE OR ( UNIX OR FREEBSD ) OR foobar)', + 'UNIX OR foobar') + + +def test_simplify_complex_false(): + validate_simplify('WIN32 AND foobar AND ( ' + 'APPLE OR ( UNIX OR FREEBSD ))', + 'OFF') + + +def test_simplify_android_not_apple(): + validate_simplify('ANDROID AND NOT ANDROID_EMBEDDED AND NOT APPLE_OSX', + 'ANDROID AND NOT ANDROID_EMBEDDED') diff --git a/util/cmake/tests/test_operations.py b/util/cmake/tests/test_operations.py new file mode 100755 index 0000000000..c1e5f1b250 --- /dev/null +++ b/util/cmake/tests/test_operations.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from pro2cmake import AddOperation, SetOperation, UniqueAddOperation, RemoveOperation + +def test_add_operation(): + op = AddOperation(['bar', 'buz']) + + result = op.process(['foo', 'bar'], ['foo', 'bar'], lambda x: x) + assert ['foo', 'bar', 'bar', 'buz'] == result + + +def test_uniqueadd_operation(): + op = UniqueAddOperation(['bar', 'buz']) + + result = op.process(['foo', 'bar'], ['foo', 'bar'], lambda x: x) + assert ['foo', 'bar', 'buz'] == result + + +def test_set_operation(): + op = SetOperation(['bar', 'buz']) + + result = op.process(['foo', 'bar'], ['foo', 'bar'], lambda x: x) + assert ['bar', 'buz'] == result + + +def test_remove_operation(): + op = RemoveOperation(['bar', 'buz']) + + result = op.process(['foo', 'bar'], ['foo', 'bar'], lambda x: x) + assert ['foo', '-buz'] == result diff --git a/util/cmake/tests/test_parsing.py b/util/cmake/tests/test_parsing.py new file mode 100755 index 0000000000..f924b13913 --- /dev/null +++ b/util/cmake/tests/test_parsing.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +import os +from pro2cmake import QmakeParser + + +_tests_path = os.path.dirname(os.path.abspath(__file__)) + + +def validate_op(key, op, value, to_validate): + assert key == to_validate['key'] + assert op == to_validate['operation'] + assert value == to_validate.get('value', None) + + +def validate_single_op(key, op, value, to_validate): + assert len(to_validate) == 1 + validate_op(key, op, value, to_validate[0]) + + +def evaluate_condition(to_validate): + assert 'condition' in to_validate + assert 'statements' in to_validate + + return (to_validate['condition'], + to_validate['statements'], + to_validate.get('else_statements', {})) + + +def validate_default_else_test(file_name): + result = parse_file(file_name) + assert len(result) == 1 + + (cond, if_branch, else_branch) = evaluate_condition(result[0]) + assert cond == 'qtConfig(timezone)' + validate_single_op('A', '=', ['1'], if_branch) + + assert len(else_branch) == 1 + (cond2, if2_branch, else2_branch) = evaluate_condition(else_branch[0]) + assert cond2 == 'win32' + validate_single_op('B', '=', ['2'], if2_branch) + validate_single_op('C', '=', ['3'], else2_branch) + + +def parse_file(file): + p = QmakeParser(debug=True) + result = p.parseFile(file) + + print('\n\n#### Parser result:') + print(result) + print('\n#### End of parser result.\n') + + print('\n\n####Parser result dictionary:') + print(result.asDict()) + print('\n#### End of parser result dictionary.\n') + + result_dictionary = result.asDict() + + assert len(result_dictionary) == 1 + + return result_dictionary['statements'] + + +def test_else(): + result = parse_file(_tests_path + '/data/else.pro') + assert len(result) == 1 + + (cond, if_branch, else_branch) = evaluate_condition(result[0]) + + assert cond == 'linux' + validate_single_op('SOURCES', '+=', ['a.cpp'], if_branch) + validate_single_op('SOURCES', '+=', ['b.cpp'], else_branch) + + +def test_else2(): + result = parse_file(_tests_path + '/data/else2.pro') + assert len(result) == 1 + + (cond, if_branch, else_branch) = evaluate_condition(result[0]) + assert cond == 'osx' + validate_single_op('A', '=', ['1'], if_branch) + + assert len(else_branch) == 1 + (cond2, if2_branch, else2_branch) = evaluate_condition(else_branch[0]) + assert cond2 == 'win32' + validate_single_op('B', '=', ['2'], if2_branch) + + validate_single_op('C', '=', ['3'], else2_branch) + + +def test_else3(): + validate_default_else_test(_tests_path + '/data/else3.pro') + + +def test_else4(): + validate_default_else_test(_tests_path + '/data/else4.pro') + + +def test_else5(): + validate_default_else_test(_tests_path + '/data/else5.pro') + + +def test_else6(): + validate_default_else_test(_tests_path + '/data/else6.pro') + + +def test_else7(): + result = parse_file(_tests_path + '/data/else7.pro') + assert len(result) == 1 + + +def test_else8(): + validate_default_else_test(_tests_path + '/data/else8.pro') + + +def test_multiline_assign(): + result = parse_file(_tests_path + '/data/multiline_assign.pro') + assert len(result) == 2 + validate_op('A', '=', ['42', '43', '44'], result[0]) + validate_op('B', '=', ['23'], result[1]) + + +def test_include(): + result = parse_file(_tests_path + '/data/include.pro') + assert len(result) == 3 + validate_op('A', '=', ['42'], result[0]) + include = result[1] + assert len(include) == 1 + assert include.get('included', '') == 'foo' + validate_op('B', '=', ['23'], result[2]) + + +def test_load(): + result = parse_file(_tests_path + '/data/load.pro') + assert len(result) == 3 + validate_op('A', '=', ['42'], result[0]) + load = result[1] + assert len(load) == 1 + assert load.get('loaded', '') == 'foo' + validate_op('B', '=', ['23'], result[2]) + + +def test_definetest(): + result = parse_file(_tests_path + '/data/definetest.pro') + assert len(result) == 1 + assert result[0] == [] + + +def test_for(): + result = parse_file(_tests_path + '/data/for.pro') + assert len(result) == 2 + validate_op('SOURCES', '=', ['main.cpp'], result[0]) + assert result[1] == [] + + +def test_single_line_for(): + result = parse_file(_tests_path + '/data/single_line_for.pro') + assert len(result) == 1 + assert result[0] == [] + + +def test_unset(): + result = parse_file(_tests_path + '/data/unset.pro') + assert len(result) == 1 + assert result[0] == [] + + +def test_quoted(): + result = parse_file(_tests_path + '/data/quoted.pro') + assert len(result) == 1 + + +def test_complex_values(): + result = parse_file(_tests_path + '/data/complex_values.pro') + assert len(result) == 1 + + +def test_function_if(): + result = parse_file(_tests_path + '/data/function_if.pro') + assert len(result) == 1 + + +def test_realworld_standardpaths(): + result = parse_file(_tests_path + '/data/standardpaths.pro') + + (cond, if_branch, else_branch) = evaluate_condition(result[0]) + assert cond == 'win32' + assert len(if_branch) == 1 + assert len(else_branch) == 1 + + # win32: + (cond1, if_branch1, else_branch1) = evaluate_condition(if_branch[0]) + assert cond1 == '!winrt' + assert len(if_branch1) == 1 + validate_op('SOURCES', '+=', ['io/qstandardpaths_win.cpp'], if_branch1[0]) + assert len(else_branch1) == 1 + validate_op('SOURCES', '+=', ['io/qstandardpaths_winrt.cpp'], else_branch1[0]) + + # unix: + (cond2, if_branch2, else_branch2) = evaluate_condition(else_branch[0]) + assert cond2 == 'unix' + assert len(if_branch2) == 1 + assert len(else_branch2) == 0 + + # mac / else: + (cond3, if_branch3, else_branch3) = evaluate_condition(if_branch2[0]) + assert cond3 == 'mac' + assert len(if_branch3) == 1 + validate_op('OBJECTIVE_SOURCES', '+=', ['io/qstandardpaths_mac.mm'], if_branch3[0]) + assert len(else_branch3) == 1 + + # android / else: + (cond4, if_branch4, else_branch4) = evaluate_condition(else_branch3[0]) + assert cond4 == 'android && !android-embedded' + assert len(if_branch4) == 1 + validate_op('SOURCES', '+=', ['io/qstandardpaths_android.cpp'], if_branch4[0]) + assert len(else_branch4) == 1 + + # haiku / else: + (cond5, if_branch5, else_branch5) = evaluate_condition(else_branch4[0]) + assert cond5 == 'haiku' + assert len(if_branch5) == 1 + validate_op('SOURCES', '+=', ['io/qstandardpaths_haiku.cpp'], if_branch5[0]) + assert len(else_branch5) == 1 + validate_op('SOURCES', '+=', ['io/qstandardpaths_unix.cpp'], else_branch5[0]) + + +def test_realworld_comment_scope(): + result = parse_file(_tests_path + '/data/comment_scope.pro') + assert len(result) == 2 + (cond, if_branch, else_branch) = evaluate_condition(result[0]) + assert cond == 'freebsd|openbsd' + assert len(if_branch) == 1 + validate_op('QMAKE_LFLAGS_NOUNDEF', '=', None, if_branch[0]) + + assert result[1].get('included', '') == 'animation/animation.pri' + + +def test_realworld_contains_scope(): + result = parse_file(_tests_path + '/data/contains_scope.pro') + assert len(result) == 2 + + +def test_realworld_complex_assign(): + result = parse_file(_tests_path + '/data/complex_assign.pro') + assert len(result) == 1 + validate_op('qmake-clean.commands', '+=', '( cd qmake && $(MAKE) clean ":-(==)-:" \'(Foo)\' )'.split(), + result[0]) + + +def test_realworld_complex_condition(): + result = parse_file(_tests_path + '/data/complex_condition.pro') + assert len(result) == 1 + (cond, if_branch, else_branch) = evaluate_condition(result[0]) + assert cond == '!system("dbus-send --session --type=signal / ' \ + 'local.AutotestCheck.Hello >$$QMAKE_SYSTEM_NULL_DEVICE ' \ + '2>&1")' + assert len(if_branch) == 1 + validate_op('SOURCES', '=', ['dbus.cpp'], if_branch[0]) + + assert len(else_branch) == 0 + + +def test_realworld_sql(): + result = parse_file(_tests_path + '/data/sql.pro') + assert len(result) == 2 + validate_op('TEMPLATE', '=', ['subdirs'], result[0]) + validate_op('SUBDIRS', '=', ['kernel'], result[1]) + + +def test_realworld_qtconfig(): + result = parse_file(_tests_path + '/data/escaped_value.pro') + assert len(result) == 1 + validate_op('MODULE_AUX_INCLUDES', '=', ['\\$\\$QT_MODULE_INCLUDE_BASE/QtANGLE'], result[0]) + + +def test_realworld_lc(): + result = parse_file(_tests_path + '/data/lc.pro') + assert len(result) == 3 + + +def test_realworld_lc_with_comment_in_between(): + result = parse_file(_tests_path + '/data/lc_with_comment.pro') + + my_var = result[1]['value'][0] + assert my_var == 'foo' + + my_var = result[2]['value'][0] + assert my_var == 'foo2' + + my_var = result[3]['value'][0] + assert my_var == 'foo3' + + my_var = result[4]['value'][0] + assert my_var == 'foo4' + + my_var = result[5]['value'][0] + assert my_var == 'foo5' + + sub_dirs = result[0]['value'] + assert sub_dirs[0] == 'tga' + assert sub_dirs[1] == 'wbmp' + assert len(result) == 6 + + +def test_condition_without_scope(): + result = parse_file(_tests_path + '/data/condition_without_scope.pro') + assert len(result) == 1 + + +def test_multi_condition_divided_by_lc(): + result = parse_file(_tests_path + '/data/multi_condition_divided_by_lc.pro') + assert len(result) == 1 + + +def test_nested_function_calls(): + result = parse_file(_tests_path + '/data/nested_function_calls.pro') + assert len(result) == 1 diff --git a/util/cmake/tests/test_scope_handling.py b/util/cmake/tests/test_scope_handling.py new file mode 100755 index 0000000000..c0b553fabd --- /dev/null +++ b/util/cmake/tests/test_scope_handling.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2018 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +from pro2cmake import Scope, SetOperation, merge_scopes, recursive_evaluate_scope + +import pytest +import typing + +ScopeList = typing.List[Scope] + +def _map_to_operation(**kwargs): + result = {} # type: typing.Mapping[str, typing.List[SetOperation]] + for (key, value) in kwargs.items(): + result[key] = [SetOperation([value])] + return result + + +def _new_scope(*, parent_scope=None, condition='', **kwargs) -> Scope: + return Scope(parent_scope=parent_scope, + file='file1', condition=condition, operations=_map_to_operation(**kwargs)) + + +def _evaluate_scopes(scopes: ScopeList) -> ScopeList: + for s in scopes: + if not s.parent: + recursive_evaluate_scope(s) + return scopes + + +def _validate(input_scopes: ScopeList, output_scopes: ScopeList): + merged_scopes = merge_scopes(input_scopes) + assert merged_scopes == output_scopes + + +def test_evaluate_one_scope(): + scope = _new_scope(condition='QT_FEATURE_foo', test1='bar') + + input_scope = scope + recursive_evaluate_scope(scope) + assert scope == input_scope + + +def test_evaluate_child_scope(): + scope = _new_scope(condition='QT_FEATURE_foo', test1='bar') + _new_scope(parent_scope=scope, condition='QT_FEATURE_bar', test2='bar') + + input_scope = scope + recursive_evaluate_scope(scope) + + assert scope.total_condition == 'QT_FEATURE_foo' + assert len(scope.children) == 1 + assert scope.get_string('test1') == 'bar' + assert scope.get_string('test2', 'not found') == 'not found' + + child = scope.children[0] + assert child.total_condition == 'QT_FEATURE_bar AND QT_FEATURE_foo' + assert child.get_string('test1', 'not found') == 'not found' + assert child.get_string('test2') == 'bar' + + +def test_evaluate_two_child_scopes(): + scope = _new_scope(condition='QT_FEATURE_foo', test1='bar') + _new_scope(parent_scope=scope, condition='QT_FEATURE_bar', test2='bar') + _new_scope(parent_scope=scope, condition='QT_FEATURE_buz', test3='buz') + + input_scope = scope + recursive_evaluate_scope(scope) + + assert scope.total_condition == 'QT_FEATURE_foo' + assert len(scope.children) == 2 + assert scope.get_string('test1') == 'bar' + assert scope.get_string('test2', 'not found') == 'not found' + assert scope.get_string('test3', 'not found') == 'not found' + + child1 = scope.children[0] + assert child1.total_condition == 'QT_FEATURE_bar AND QT_FEATURE_foo' + assert child1.get_string('test1', 'not found') == 'not found' + assert child1.get_string('test2') == 'bar' + assert child1.get_string('test3', 'not found') == 'not found' + + child2 = scope.children[1] + assert child2.total_condition == 'QT_FEATURE_buz AND QT_FEATURE_foo' + assert child2.get_string('test1', 'not found') == 'not found' + assert child2.get_string('test2') == '' + assert child2.get_string('test3', 'not found') == 'buz' + + +def test_evaluate_else_child_scopes(): + scope = _new_scope(condition='QT_FEATURE_foo', test1='bar') + _new_scope(parent_scope=scope, condition='QT_FEATURE_bar', test2='bar') + _new_scope(parent_scope=scope, condition='else', test3='buz') + + input_scope = scope + recursive_evaluate_scope(scope) + + assert scope.total_condition == 'QT_FEATURE_foo' + assert len(scope.children) == 2 + assert scope.get_string('test1') == 'bar' + assert scope.get_string('test2', 'not found') == 'not found' + assert scope.get_string('test3', 'not found') == 'not found' + + child1 = scope.children[0] + assert child1.total_condition == 'QT_FEATURE_bar AND QT_FEATURE_foo' + assert child1.get_string('test1', 'not found') == 'not found' + assert child1.get_string('test2') == 'bar' + assert child1.get_string('test3', 'not found') == 'not found' + + child2 = scope.children[1] + assert child2.total_condition == 'QT_FEATURE_foo AND NOT QT_FEATURE_bar' + assert child2.get_string('test1', 'not found') == 'not found' + assert child2.get_string('test2') == '' + assert child2.get_string('test3', 'not found') == 'buz' + + +def test_evaluate_invalid_else_child_scopes(): + scope = _new_scope(condition='QT_FEATURE_foo', test1='bar') + _new_scope(parent_scope=scope, condition='else', test3='buz') + _new_scope(parent_scope=scope, condition='QT_FEATURE_bar', test2='bar') + + input_scope = scope + with pytest.raises(AssertionError): + recursive_evaluate_scope(scope) + + +def test_merge_empty_scope_list(): + _validate([], []) + + +def test_merge_one_scope(): + scopes = [_new_scope(test='foo')] + + recursive_evaluate_scope(scopes[0]) + + _validate(scopes, scopes) + + +def test_merge_one_on_scope(): + scopes = [_new_scope(condition='ON', test='foo')] + + recursive_evaluate_scope(scopes[0]) + + _validate(scopes, scopes) + + +def test_merge_one_off_scope(): + scopes = [_new_scope(condition='OFF', test='foo')] + + recursive_evaluate_scope(scopes[0]) + + _validate(scopes, []) + + +def test_merge_one_conditioned_scope(): + scopes = [_new_scope(condition='QT_FEATURE_foo', test='foo')] + + recursive_evaluate_scope(scopes[0]) + + _validate(scopes, scopes) + + +def test_merge_two_scopes_with_same_condition(): + scopes = [_new_scope(condition='QT_FEATURE_bar', test='foo'), + _new_scope(condition='QT_FEATURE_bar', test2='bar')] + + recursive_evaluate_scope(scopes[0]) + recursive_evaluate_scope(scopes[1]) + + result = merge_scopes(scopes) + + assert len(result) == 1 + r0 = result[0] + assert r0.total_condition == 'QT_FEATURE_bar' + assert r0.get_string('test') == 'foo' + assert r0.get_string('test2') == 'bar' + + +def test_merge_three_scopes_two_with_same_condition(): + scopes = [_new_scope(condition='QT_FEATURE_bar', test='foo'), + _new_scope(condition='QT_FEATURE_baz', test1='buz'), + _new_scope(condition='QT_FEATURE_bar', test2='bar')] + + recursive_evaluate_scope(scopes[0]) + recursive_evaluate_scope(scopes[1]) + recursive_evaluate_scope(scopes[2]) + + result = merge_scopes(scopes) + + assert len(result) == 2 + r0 = result[0] + assert r0.total_condition == 'QT_FEATURE_bar' + assert r0.get_string('test') == 'foo' + assert r0.get_string('test2') == 'bar' + + assert result[1] == scopes[1] + + +def test_merge_two_unrelated_on_off_scopes(): + scopes = [_new_scope(condition='ON', test='foo'), + _new_scope(condition='OFF', test2='bar')] + + recursive_evaluate_scope(scopes[0]) + recursive_evaluate_scope(scopes[1]) + + _validate(scopes, [scopes[0]]) + + +def test_merge_two_unrelated_on_off_scopes(): + scopes = [_new_scope(condition='OFF', test='foo'), + _new_scope(condition='ON', test2='bar')] + + recursive_evaluate_scope(scopes[0]) + recursive_evaluate_scope(scopes[1]) + + _validate(scopes, [scopes[1]]) + + +def test_merge_parent_child_scopes_with_different_conditions(): + scope = _new_scope(condition='FOO', test1='parent') + scopes = [scope, _new_scope(parent_scope=scope, condition='bar', test2='child')] + + recursive_evaluate_scope(scope) + + _validate(scopes, scopes) + + +def test_merge_parent_child_scopes_with_same_conditions(): + scope = _new_scope(condition='FOO AND bar', test1='parent') + scopes = [scope, _new_scope(parent_scope=scope, condition='FOO AND bar', test2='child')] + + recursive_evaluate_scope(scope) + + result = merge_scopes(scopes) + + assert len(result) == 1 + r0 = result[0] + assert r0.parent == None + assert r0.total_condition == 'FOO AND bar' + assert r0.get_string('test1') == 'parent' + assert r0.get_string('test2') == 'child' + + +def test_merge_parent_child_scopes_with_on_child_condition(): + scope = _new_scope(condition='FOO AND bar', test1='parent') + scopes = [scope, _new_scope(parent_scope=scope, condition='ON', test2='child')] + + recursive_evaluate_scope(scope) + + result = merge_scopes(scopes) + + assert len(result) == 1 + r0 = result[0] + assert r0.parent == None + assert r0.total_condition == 'FOO AND bar' + assert r0.get_string('test1') == 'parent' + assert r0.get_string('test2') == 'child' + + +# Real world examples: + +# qstandardpaths selection: + +def test_qstandardpaths_scopes(): + # top level: + scope1 = _new_scope(condition='ON', scope_id=1) + + # win32 { + scope2 = _new_scope(parent_scope=scope1, condition='WIN32') + # !winrt { + # SOURCES += io/qstandardpaths_win.cpp + scope3 = _new_scope(parent_scope=scope2, condition='NOT WINRT', + SOURCES='qsp_win.cpp') + # } else { + # SOURCES += io/qstandardpaths_winrt.cpp + scope4 = _new_scope(parent_scope=scope2, condition='else', + SOURCES='qsp_winrt.cpp') + # } + # else: unix { + scope5 = _new_scope(parent_scope=scope1, condition='else') + scope6 = _new_scope(parent_scope=scope5, condition='UNIX') + # mac { + # OBJECTIVE_SOURCES += io/qstandardpaths_mac.mm + scope7 = _new_scope(parent_scope=scope6, condition='APPLE_OSX', SOURCES='qsp_mac.mm') + # } else:android:!android-embedded { + # SOURCES += io/qstandardpaths_android.cpp + scope8 = _new_scope(parent_scope=scope6, condition='else') + scope9 = _new_scope(parent_scope=scope8, + condition='ANDROID AND NOT ANDROID_EMBEDDED', + SOURCES='qsp_android.cpp') + # } else:haiku { + # SOURCES += io/qstandardpaths_haiku.cpp + scope10 = _new_scope(parent_scope=scope8, condition='else') + scope11 = _new_scope(parent_scope=scope10, condition='HAIKU', SOURCES='qsp_haiku.cpp') + # } else { + # SOURCES +=io/qstandardpaths_unix.cpp + scope12 = _new_scope(parent_scope=scope10, condition='else', SOURCES='qsp_unix.cpp') + # } + # } + + recursive_evaluate_scope(scope1) + + assert scope1.total_condition == 'ON' + assert scope2.total_condition == 'WIN32' + assert scope3.total_condition == 'WIN32 AND NOT WINRT' + assert scope4.total_condition == 'WINRT' + assert scope5.total_condition == 'UNIX' + assert scope6.total_condition == 'UNIX' + assert scope7.total_condition == 'APPLE_OSX' + assert scope8.total_condition == 'UNIX AND NOT APPLE_OSX' + assert scope9.total_condition == 'ANDROID AND NOT ANDROID_EMBEDDED' + assert scope10.total_condition == 'UNIX AND NOT APPLE_OSX AND (ANDROID_EMBEDDED OR NOT ANDROID)' + assert scope11.total_condition == 'HAIKU AND (ANDROID_EMBEDDED OR NOT ANDROID)' + assert scope12.total_condition == 'UNIX AND NOT APPLE_OSX AND NOT HAIKU AND (ANDROID_EMBEDDED OR NOT ANDROID)' |