diff options
Diffstat (limited to 'tools')
35 files changed, 7873 insertions, 84 deletions
diff --git a/tools/checklibs.py b/tools/checklibs.py new file mode 100644 index 000000000..9a53beade --- /dev/null +++ b/tools/checklibs.py @@ -0,0 +1,355 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#!/usr/bin/env python +# +# checklibs.py +# +# Check Mach-O dependencies. +# +# See http://www.entropy.ch/blog/Developer/2011/03/05/2011-Update-to-checklibs-Script-for-dynamic-library-dependencies.html +# +# Written by Marc Liyanage <http://www.entropy.ch> +# +# + +import collections +import optparse +import re +import subprocess +import sys +from pathlib import Path +from pprint import pprint + + +class MachOFile: + + def __init__(self, image_path, arch, parent = None): + self.image_path = image_path + self._dependencies = [] + self._cache = dict(paths = {}, order = []) + self.arch = arch + self.parent = parent + self.header_info = {} + self.load_info() + self.add_to_cache() + + def load_info(self): + if not self.image_path.exists(): + return + self.load_header() + self.load_rpaths() + + def load_header(self): + # Get the mach-o header info, we're interested in the file type + # (executable, dylib) + cmd = 'otool -arch {0} -h "{1}"' + output = self.shell(cmd, [self.arch, self.image_path.resolved_path], + fatal = True) + if not output: + print("Unable to load mach header for {} ({}), architecture " + "mismatch? Use --arch option to pick architecture".format( + self.image_path.resolved_path, self.arch), file=sys.stderr) + exit() + (keys, values) = output.splitlines()[2:] + self.header_info = dict(zip(keys.split(), values.split())) + + def load_rpaths(self): + output = self.shell('otool -arch {0} -l "{1}"', + [self.arch, self.image_path.resolved_path], fatal = True) + # skip file name on first line + load_commands = re.split('Load command (\d+)', output)[1:] + self._rpaths = [] + load_commands = collections.deque(load_commands) + while load_commands: + load_commands.popleft() # command index + command = load_commands.popleft().strip().splitlines() + if command[0].find('LC_RPATH') == -1: + continue + + path = re.findall('path (.+) \(offset \d+\)$', command[2])[0] + image_path = self.image_path_for_recorded_path(path) + image_path.rpath_source = self + self._rpaths.append(image_path) + + def ancestors(self): + ancestors = [] + parent = self.parent + while parent: + ancestors.append(parent) + parent = parent.parent + + return ancestors + + def self_and_ancestors(self): + return [self] + self.ancestors() + + def rpaths(self): + return self._rpaths + + def all_rpaths(self): + rpaths = [] + for image in self.self_and_ancestors(): + rpaths.extend(image.rpaths()) + return rpaths + + def root(self): + if not self.parent: + return self + return self.ancestors()[-1] + + def executable_path(self): + root = self.root() + if root.is_executable(): + return root.image_path + return None + + def filetype(self): + return long(self.header_info.get('filetype', 0)) + + def is_dylib(self): + return self.filetype() == MachOFile.MH_DYLIB + + def is_executable(self): + return self.filetype() == MachOFile.MH_EXECUTE + + def all_dependencies(self): + self.walk_dependencies() + return self.cache()['order'] + + def walk_dependencies(self, known = {}): + if known.get(self.image_path.resolved_path): + return + + known[self.image_path.resolved_path] = self + + for item in self.dependencies(): + item.walk_dependencies(known) + + def dependencies(self): + if not self.image_path.exists(): + return [] + + if self._dependencies: + return self._dependencies + + output = self.shell('otool -arch {0} -L "{1}"', + [self.arch, self.image_path.resolved_path], fatal = True) + output = [line.strip() for line in output.splitlines()] + del(output[0]) + if self.is_dylib(): + # In the case of dylibs, the first line is the id line + del(output[0]) + + self._dependencies = [] + for line in output: + match = re.match('^(.+)\s+(\(.+)\)$', line) + if not match: + continue + recorded_path = match.group(1) + image_path = self.image_path_for_recorded_path(recorded_path) + image = self.lookup_or_make_item(image_path) + self._dependencies.append(image) + + return self._dependencies + + # The root item holds the cache, all lower-level requests bubble up + # the parent chain + def cache(self): + if self.parent: + return self.parent.cache() + return self._cache + + def add_to_cache(self): + cache = self.cache() + cache['paths'][self.image_path.resolved_path] = self + cache['order'].append(self) + + def cached_item_for_path(self, path): + if not path: + return None + return self.cache()['paths'].get(path) + + def lookup_or_make_item(self, image_path): + image = self.cached_item_for_path(image_path.resolved_path) + if not image: # cache miss + image = MachOFile(image_path, self.arch, parent = self) + return image + + def image_path_for_recorded_path(self, recorded_path): + path = ImagePath(None, recorded_path) + + # handle @executable_path + if recorded_path.startswith(ImagePath.EXECUTABLE_PATH_TOKEN): + executable_image_path = self.executable_path() + if executable_image_path: + path.resolved_path = Path( + recorded_path.replace( + ImagePath.EXECUTABLE_PATH_TOKEN, + Path(executable_image_path.resolved_path).parent)) + + # handle @loader_path + elif recorded_path.startswith(ImagePath.LOADER_PATH_TOKEN): + path.resolved_path = Path(recorded_path.replace( + ImagePath.LOADER_PATH_TOKEN, + Path(self.image_path.resolved_path).parent)) + + # handle @rpath + elif recorded_path.startswith(ImagePath.RPATH_TOKEN): + for rpath in self.all_rpaths(): + resolved_path = Path(recorded_path.replace( + ImagePath.RPATH_TOKEN, rpath.resolved_path)) + if resolved_path.exists(): + path.resolved_path = resolved_path + path.rpath_source = rpath.rpath_source + break + + # handle absolute path + elif recorded_path.startswith('/'): + path.resolved_path = recorded_path + + return path + + def __repr__(self): + return str(self.image_path) + + def dump(self): + print(self.image_path) + for dependency in self.dependencies(): + print('\t{0}'.format(dependency)) + + @staticmethod + def shell(cmd_format, args, fatal = False): + cmd = cmd_format.format(*args) + popen = subprocess.Popen(cmd, shell = True, stdout = subprocess.PIPE) + output = popen.communicate()[0] + if popen.returncode and fatal: + print("Nonzero exit status for shell command '{}'".format(cmd), + file=sys.stderr) + sys.exit(1) + + return output + + @classmethod + def architectures_for_image_at_path(cls, path): + output = cls.shell('file "{}"', [path]) + file_architectures = re.findall(r' executable (\w+)', output) + ordering = 'x86_64 i386'.split() + file_architectures = sorted(file_architectures, lambda a, b: cmp( + ordering.index(a), ordering.index(b))) + return file_architectures + + MH_EXECUTE = 0x2 + MH_DYLIB = 0x6 + MH_BUNDLE = 0x8 + + +# ANSI terminal coloring sequences +class Color: + HEADER = '\033[95m' + BLUE = '\033[94m' + GREEN = '\033[92m' + RED = '\033[91m' + ENDC = '\033[0m' + + @staticmethod + def red(string): + return Color.wrap(string, Color.RED) + + @staticmethod + def blue(string): + return Color.wrap(string, Color.BLUE) + + @staticmethod + def wrap(string, color): + return Color.HEADER + color + string + Color.ENDC + + +# This class holds path information for a mach-0 image file. +# It holds the path as it was recorded in the loading binary as well as +# the effective, resolved file system path. +# The former can contain @-replacement tokens. +# In the case where the recorded path contains an @rpath token that was +# resolved successfully, we also capture the path of the binary that +# supplied the rpath value that was used. +# That path itself can contain replacement tokens such as @loader_path. +class ImagePath: + + def __init__(self, resolved_path, recorded_path = None): + self.recorded_path = recorded_path + self.resolved_path = resolved_path + self.rpath_source = None + + def __repr__(self): + description = None + + if self.resolved_equals_recorded() or self.recorded_path == None: + description = self.resolved_path + else: + description = '{0} ({1})'.format(self.resolved_path, + self.recorded_path) + + if (not self.is_system_location()) and (not self.uses_dyld_token()): + description = Color.blue(description) + + if self.rpath_source: + description += ' (rpath source: {0})'.format( + self.rpath_source.image_path.resolved_path) + + if not self.exists(): + description += Color.red(' (missing)') + + return description + + def exists(self): + return self.resolved_path and Path(self.resolved_path).exists() + + def resolved_equals_recorded(self): + return (self.resolved_path and self.recorded_path and + self.resolved_path == self.recorded_path) + + def uses_dyld_token(self): + return self.recorded_path and self.recorded_path.startswith('@') + + def is_system_location(self): + system_prefixes = ['/System/Library', '/usr/lib'] + for prefix in system_prefixes: + if self.resolved_path and self.resolved_path.startswith(prefix): + return True + + EXECUTABLE_PATH_TOKEN = '@executable_path' + LOADER_PATH_TOKEN = '@loader_path' + RPATH_TOKEN = '@rpath' + + +# Command line driver +parser = optparse.OptionParser( + usage = "Usage: %prog [options] path_to_mach_o_file") +parser.add_option( + "--arch", dest = "arch", help = "architecture", metavar = "ARCH") +parser.add_option( + "--all", dest = "include_system_libraries", + help = "Include system frameworks and libraries", action="store_true") +(options, args) = parser.parse_args() + +if len(args) < 1: + parser.print_help() + sys.exit(1) + +archs = MachOFile.architectures_for_image_at_path(args[0]) +if archs and not options.arch: + print('Analyzing architecture {}, override with --arch if needed'.format( + archs[0]), file=sys.stderr) + options.arch = archs[0] + +toplevel_image = MachOFile(ImagePath(args[0]), options.arch) + +for dependency in toplevel_image.all_dependencies(): + if (dependency.image_path.exists() and + (not options.include_system_libraries) and + dependency.image_path.is_system_location()): + continue + + dependency.dump() + print("\n") + diff --git a/tools/create_changelog.py b/tools/create_changelog.py index 12c67d62f..6c24f417f 100644 --- a/tools/create_changelog.py +++ b/tools/create_changelog.py @@ -1,51 +1,16 @@ -############################################################################# -## -## Copyright (C) 2019 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import re +import os import sys -from textwrap import dedent +import textwrap from argparse import ArgumentParser, Namespace, RawTextHelpFormatter -from subprocess import check_output, Popen, PIPE -from typing import Dict, List +from pathlib import Path +from subprocess import PIPE, Popen, check_output +from typing import Dict, List, Tuple -content = """ -Qt for Python @VERSION is a @TYPE release. +content_header = """Qt for Python @VERSION is a @TYPE release. For more details, refer to the online documentation included in this distribution. The documentation is also available online: @@ -61,26 +26,65 @@ Each of these identifiers can be entered in the bug tracker to obtain more information about a particular change. **************************************************************************** -* PySide2 * +* PySide6 * **************************************************************************** +""" -@PYSIDE - -**************************************************************************** -* Shiboken2 * +shiboken_header = """**************************************************************************** +* Shiboken6 * **************************************************************************** +""" + +description = """ +PySide6 changelog tool -@SHIBOKEN +Example usage: +tools/create_changelog.py -v -r 6.5.3 """ +def change_log(version: list) -> Path: + """Return path of the changelog of the version.""" + name = f"changes-{version[0]}.{version[1]}.{version[2]}" + return Path(__file__).parents[1] / "doc" / "changelogs" / name + + +def is_lts_version(version: list) -> bool: + return version[0] == 5 or version[1] in (2, 5) + + +def version_tag(version: list) -> str: + """Return the version tag.""" + tag = f"v{version[0]}.{version[1]}.{version[2]}" + return tag + "-lts" if is_lts_version(version) else tag + + +def revision_range(version: list) -> str: + """Determine a git revision_range from the version. Either log from + the previous version tag or since the last update to the changelog.""" + changelog = change_log(version) + if changelog.is_file(): + output = check_output(["git", "log", "-n", "1", "--format=%H", + os.fspath(changelog)]) + if output: + return output.strip().decode("utf-8") + "..HEAD" + + last_version = version.copy() + if version[2] == 0: + adjust_idx = 0 if version[1] == 0 else 1 + else: + adjust_idx = 2 + last_version[adjust_idx] -= 1 + return version_tag(last_version) + "..HEAD" + + def parse_options() -> Namespace: tag_msg = ("Tags, branches, or SHA to compare\n" "e.g.: v5.12.1..5.12\n" " v5.12.0..v5.12.1\n" " cebc32a5..5.12") - options = ArgumentParser(description="PySide2 changelog tool", + options = ArgumentParser(description=description, formatter_class=RawTextHelpFormatter) options.add_argument("-d", "--directory", @@ -89,8 +93,7 @@ def parse_options() -> Namespace: options.add_argument("-v", "--versions", type=str, - help=tag_msg, - required=True) + help=tag_msg) options.add_argument("-r", "--release", type=str, @@ -99,18 +102,52 @@ def parse_options() -> Namespace: options.add_argument("-t", "--type", type=str, - help="Release type: bug-fix, minor, or major", - default="bug-fix") + help="Release type: bug-fix, minor, or major") + + options.add_argument("-e", + "--exclude", + action="store_true", + help="Exclude commits with a 'Pick-to' line", + default=False) args = options.parse_args() + + release_version = list(int(v) for v in args.release.split(".")) + if len(release_version) != 3: + print("Error: --release must be of form major.minor.patch") + sys.exit(-1) + + # Some auto-detection smartness + if not args.type: + if release_version[2] == 0: + args.type = "major" if release_version[1] == 0 else "minor" + else: + args.type = "bug-fix" + # For major/minor releases, skip all fixes with "Pick-to: " since they + # appear in bug-fix releases. + if args.type != "bug-fix": + args.exclude = True + print(f'Assuming "{args.type}" version', file=sys.stderr) + if args.type not in ("bug-fix", "minor", "major"): - print("Error:" + print("Error: " "-y/--type needs to be: bug-fix (default), minor, or major") sys.exit(-1) + if not args.versions: + args.versions = revision_range(release_version) + print(f"Assuming range {args.versions}", file=sys.stderr) + + args.release_version = release_version return args +def format_text(text: str) -> str: + """Format an entry with a leading dash, 80 columns""" + return textwrap.fill(text, width=77, initial_indent=" - ", + subsequent_indent=" ") + + def check_tag(tag: str) -> bool: output = False @@ -137,22 +174,48 @@ def check_tag(tag: str) -> bool: def get_commit_content(sha: str) -> str: - command= "git log {} -n 1 --pretty=format:%s%n%n%b".format(sha) - print("{}: {}".format(get_commit_content.__name__, command), file=sys.stderr) + command = "git log {} -n 1 --pretty=format:%s%n%n%b".format(sha) + print("{}: {}".format(get_commit_content.__name__, command), + file=sys.stderr) out, err = Popen(command, stdout=PIPE, shell=True).communicate() + if err: + print(err, file=sys.stderr) return out.decode("utf-8") -def git_command(versions: List[str], pattern: str): +def git_get_sha1s(versions: List[str], pattern: str): + """Return a list of SHA1s matching a pattern""" command = "git rev-list --reverse --grep '^{}'".format(pattern) command += " {}..{}".format(versions[0], versions[1]) command += " | git cat-file --batch" - command += " | grep -o -E \"^[0-9a-f]{40}\"" + command += " | grep -o -E \"^[0-9a-f]{40} commit\"" + command += " | awk '{print $1}'" print("{}: {}".format(git_command.__name__, command), file=sys.stderr) out_sha1, err = Popen(command, stdout=PIPE, shell=True).communicate() - sha1_list = [s.decode("utf-8") for s in out_sha1.splitlines()] + if err: + print(err, file=sys.stderr) - for sha in sha1_list: + pick_to_sha1 = [] + + if exclude_pick_to: + # if '-e', we exclude all the 'Pick-to' changes + command = "git rev-list --reverse --grep '^Pick-to:'" + command += " {}..{}".format(versions[0], versions[1]) + command += " | git cat-file --batch" + command += " | grep -o -E \"^[0-9a-f]{40} commit\"" + command += " | awk '{print $1}'" + print("{}: {}".format(git_command.__name__, command), file=sys.stderr) + out_e_sha1, err = Popen(command, stdout=PIPE, shell=True).communicate() + if err: + print(err, file=sys.stderr) + pick_to_sha1 = out_e_sha1.splitlines() + + return [s.decode("utf-8") for s in out_sha1.splitlines() if s not in pick_to_sha1] + + +def git_command(versions: List[str], pattern: str): + task_number_re = re.compile(r'^.*-(\d+)\s*$') + for sha in git_get_sha1s(versions, pattern): content = get_commit_content(sha).splitlines() # First line is title title = content[0] @@ -165,39 +228,105 @@ def git_command(versions: List[str], pattern: str): if not task: continue + task_number = -1 + task_number_match = task_number_re.match(task) + if task_number_match: + task_number = int(task_number_match.group(1)) + entry = {"title": title, "task": task, "task-number": task_number} if "shiboken" in title: - if sha not in shiboken2_commits: - shiboken2_commits[sha] = {"title": title, "task": task} + if sha not in shiboken6_commits: + shiboken6_commits[sha] = entry else: - if sha not in pyside2_commits: - pyside2_commits[sha] = {"title": title, "task": task} + if sha not in pyside6_commits: + pyside6_commits[sha] = entry def create_fixes_log(versions: List[str]) -> None: - git_command(versions, "Fixes") + git_command(versions, "Fixes: ") def create_task_log(versions: List[str]) -> None: - git_command(versions, "Task-number") + git_command(versions, "Task-number: ") + + +def extract_change_log(commit_message: List[str]) -> Tuple[str, int, str]: + """Extract a tuple of (component, task-number, change log paragraph) + from a commit message of the form [ChangeLog][shiboken6] description...""" + result = '' + component = 'pyside' + within_changelog = False + task_nr = '' + for line in commit_message: + if within_changelog: + if line: + result += ' ' + line.strip() + else: + within_changelog = False + else: + if line.startswith('[ChangeLog]'): + log_line = line[11:] + if log_line.startswith('['): + end = log_line.find(']') + if end > 0: + component = log_line[1:end] + log_line = log_line[end + 1:] + result = log_line.strip() + within_changelog = True + elif line.startswith("Fixes: ") or line.startswith("Task-number: "): + task_nr = line.split(":")[1].strip() + + task_nr_int = -1 + if task_nr: + result = f"[{task_nr}] {result}" + dash = task_nr.find('-') # "PYSIDE-627" + task_nr_int = int(task_nr[dash + 1:]) + + return (component, task_nr_int, format_text(result)) + + +def create_change_log(versions: List[str]) -> None: + for sha in git_get_sha1s(versions, r"\[ChangeLog\]"): + change_log = extract_change_log(get_commit_content(sha).splitlines()) + component, task_nr, text = change_log + if component.startswith('shiboken'): + shiboken6_changelogs.append((task_nr, text)) + else: + pyside6_changelogs.append((task_nr, text)) + + +def format_commit_msg(entry: Dict[str, str]) -> str: + task = entry["task"].replace("Fixes: ", "").replace("Task-number: ", "") + title = entry["title"] + if title.startswith("shiboken6: "): + title = title[11:] + elif title.startswith("PySide6: "): + title = title[9:] + return format_text(f"[{task}] {title}") def gen_list(d: Dict[str, Dict[str, str]]) -> str: - if d: - return "".join(" - [{}] {}\n".format(v["task"], v["title"]) - for _, v in d.items()) - else: - return " - No changes" + return "\n".join(format_commit_msg(v) + for _, v in d.items()) + def sort_dict(d: Dict[str, Dict[str, str]]) -> Dict[str, Dict[str, str]]: - return dict(sorted(d.items(), - key=lambda kv: "{:5d}".format( - int(kv[1]['task'].replace("PYSIDE-", ""))))) + return dict(sorted(d.items(), key=lambda kv: kv[1]['task-number'])) + + +def sort_changelog(c: List[Tuple[int, str]]) -> List[Tuple[int, str]]: + return sorted(c, key=lambda task_text_tuple: task_text_tuple[0]) + if __name__ == "__main__": args = parse_options() - pyside2_commits: Dict[str, Dict[str, str]] = {} - shiboken2_commits: Dict[str, Dict[str, str]] = {} + pyside6_commits: Dict[str, Dict[str, str]] = {} + shiboken6_commits: Dict[str, Dict[str, str]] = {} + # Changelogs are tuples of task number/formatted text + pyside6_changelogs: List[Tuple[int, str]] = [] + shiboken6_changelogs: List[Tuple[int, str]] = [] + + exclude_pick_to = args.exclude # Getting commits information directory = args.directory if args.directory else "." @@ -206,14 +335,25 @@ if __name__ == "__main__": if check_tag(versions[0]) and check_tag(versions[1]): create_fixes_log(versions) create_task_log(versions) + create_change_log(versions) # Sort commits - pyside2_commits = sort_dict(pyside2_commits) - shiboken2_commits = sort_dict(shiboken2_commits) + pyside6_commits = sort_dict(pyside6_commits) + shiboken6_commits = sort_dict(shiboken6_commits) + pyside6_changelogs = sort_changelog(pyside6_changelogs) + shiboken6_changelogs = sort_changelog(shiboken6_changelogs) # Generate message - print(content - .replace("@VERSION", args.release) - .replace("@TYPE", args.type) - .replace("@PYSIDE", gen_list(pyside2_commits)) - .replace("@SHIBOKEN", gen_list(shiboken2_commits))) + print(content_header.replace("@VERSION", args.release). + replace("@TYPE", args.type)) + for c in pyside6_changelogs: + print(c[1]) + print(gen_list(pyside6_commits)) + if not pyside6_changelogs and not pyside6_commits: + print(" - No changes") + print(shiboken_header) + for c in shiboken6_changelogs: + print(c[1]) + print(gen_list(shiboken6_commits)) + if not shiboken6_changelogs and not shiboken6_commits: + print(" - No changes") diff --git a/tools/cross_compile_android/android_utilities.py b/tools/cross_compile_android/android_utilities.py new file mode 100644 index 000000000..3d93abec2 --- /dev/null +++ b/tools/cross_compile_android/android_utilities.py @@ -0,0 +1,297 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import logging +import shutil +import re +import os +import stat +import sys +import subprocess + +from urllib import request +from pathlib import Path +from typing import List +from packaging import version +from tqdm import tqdm + +# the tag number does not matter much since we update the sdk later +DEFAULT_SDK_TAG = 6514223 +ANDROID_NDK_VERSION = "26b" +ANDROID_NDK_VERSION_NUMBER_SUFFIX = "10909125" + + +def run_command(command: List[str], cwd: str = None, ignore_fail: bool = False, + dry_run: bool = False, accept_prompts: bool = False, show_stdout: bool = False, + capture_stdout: bool = False): + + if capture_stdout and not show_stdout: + raise RuntimeError("capture_stdout should always be used together with show_stdout") + + if dry_run: + print(" ".join(command)) + return + + input = None + if accept_prompts: + input = str.encode("y") + + if show_stdout: + stdout = None + else: + stdout = subprocess.DEVNULL + + result = subprocess.run(command, cwd=cwd, input=input, stdout=stdout, + capture_output=capture_stdout) + + if result.returncode != 0 and not ignore_fail: + sys.exit(result.returncode) + + if capture_stdout and not result.returncode: + return result.stdout.decode("utf-8") + + return None + + +class DownloadProgressBar(tqdm): + def update_to(self, b=1, bsize=1, tsize=None): + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) + + +class SdkManager: + def __init__(self, android_sdk_dir: Path, dry_run: bool = False): + self._sdk_manager = android_sdk_dir / "tools" / "bin" / "sdkmanager" + + if not self._sdk_manager.exists(): + raise RuntimeError(f"Unable to find SdkManager in {str(self._sdk_manager)}") + + if not os.access(self._sdk_manager, os.X_OK): + current_permissions = stat.S_IMODE(os.lstat(self._sdk_manager).st_mode) + os.chmod(self._sdk_manager, current_permissions | stat.S_IEXEC) + + self._android_sdk_dir = android_sdk_dir + self._dry_run = dry_run + + def list_packages(self): + command = [self._sdk_manager, f"--sdk_root={self._android_sdk_dir}", "--list"] + return run_command(command=command, dry_run=self._dry_run, show_stdout=True, + capture_stdout=True) + + def install(self, *args, accept_license: bool = False, show_stdout=False): + command = [str(self._sdk_manager), f"--sdk_root={self._android_sdk_dir}", *args] + run_command(command=command, dry_run=self._dry_run, + accept_prompts=accept_license, show_stdout=show_stdout) + + +def extract_zip(file: Path, destination: Path): + """ + Unpacks the zip file into destination preserving all permissions + + TODO: Try to use zipfile module. Currently we cannot use zipfile module here because + extractAll() does not preserve permissions. + + In case `unzip` is not available, the user is requested to install it manually + """ + unzip = shutil.which("unzip") + if not unzip: + raise RuntimeError("Unable to find program unzip. Use `sudo apt-get install unzip`" + "to install it") + + command = [unzip, file, "-d", destination] + run_command(command=command, show_stdout=True) + + +def extract_dmg(file: Path, destination: Path): + output = run_command(['hdiutil', 'attach', '-nobrowse', '-readonly', file], + show_stdout=True, capture_stdout=True) + + # find the mounted volume + mounted_vol_name = re.search(r'/Volumes/(.*)', output).group(1) + if not mounted_vol_name: + raise RuntimeError(f"Unable to find mounted volume for file {file}") + + # copy files + shutil.copytree(f'/Volumes/{mounted_vol_name}/', destination, dirs_exist_ok=True) + + # Detach mounted volume + run_command(['hdiutil', 'detach', f'/Volumes/{mounted_vol_name}']) + + +def _download(url: str, destination: Path): + """ + Download url to destination + """ + headers, download_path = None, None + # https://github.com/tqdm/tqdm#hooks-and-callbacks + with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: + download_path, headers = request.urlretrieve(url=url, filename=destination, + reporthook=t.update_to) + assert Path(download_path).resolve() == destination + + +def download_android_ndk(ndk_path: Path): + """ + Downloads the given ndk_version into ndk_path + """ + ndk_path = ndk_path / "android-ndk" + ndk_extension = "dmg" if sys.platform == "darwin" else "zip" + ndk_zip_path = ndk_path / f"android-ndk-r{ANDROID_NDK_VERSION}-{sys.platform}.{ndk_extension}" + ndk_version_path = "" + if sys.platform == "linux": + ndk_version_path = ndk_path / f"android-ndk-r{ANDROID_NDK_VERSION}" + elif sys.platform == "darwin": + ndk_version_path = (ndk_path + / f"AndroidNDK{ANDROID_NDK_VERSION_NUMBER_SUFFIX}.app/Contents/NDK") + else: + raise RuntimeError(f"Unsupported platform {sys.platform}") + + if ndk_version_path.exists(): + print(f"NDK path found in {str(ndk_version_path)}") + else: + ndk_path.mkdir(parents=True, exist_ok=True) + url = (f"https://dl.google.com/android/repository" + f"/android-ndk-r{ANDROID_NDK_VERSION}-{sys.platform}.{ndk_extension}") + + print(f"Downloading Android Ndk version r{ANDROID_NDK_VERSION}") + _download(url=url, destination=ndk_zip_path) + + print("Unpacking Android Ndk") + if sys.platform == "darwin": + extract_dmg(file=(ndk_path + / f"android-ndk-r{ANDROID_NDK_VERSION}-{sys.platform}.{ndk_extension}" + ), + destination=ndk_path) + ndk_version_path = (ndk_version_path + / f"AndroidNDK{ANDROID_NDK_VERSION_NUMBER_SUFFIX}.app/Contents/NDK") + else: + extract_zip(file=(ndk_path + / f"android-ndk-r{ANDROID_NDK_VERSION}-{sys.platform}.{ndk_extension}" + ), + destination=ndk_path) + + return ndk_version_path + + +def download_android_commandlinetools(android_sdk_dir: Path): + """ + Downloads Android commandline tools into cltools_path. + """ + sdk_platform = sys.platform if sys.platform != "darwin" else "mac" + android_sdk_dir = android_sdk_dir / "android-sdk" + url = ("https://dl.google.com/android/repository/" + f"commandlinetools-{sdk_platform}-{DEFAULT_SDK_TAG}_latest.zip") + cltools_zip_path = (android_sdk_dir + / f"commandlinetools-{sdk_platform}-{DEFAULT_SDK_TAG}_latest.zip") + cltools_path = android_sdk_dir / "tools" + + if cltools_path.exists(): + print(f"Command-line tools found in {str(cltools_path)}") + else: + android_sdk_dir.mkdir(parents=True, exist_ok=True) + + print("Download Android Command Line Tools: " + f"commandlinetools-{sys.platform}-{DEFAULT_SDK_TAG}_latest.zip") + _download(url=url, destination=cltools_zip_path) + + print("Unpacking Android Command Line Tools") + extract_zip(file=cltools_zip_path, destination=android_sdk_dir) + + return android_sdk_dir + + +def android_list_build_tools_versions(sdk_manager: SdkManager): + """ + List all the build-tools versions available for download + """ + available_packages = sdk_manager.list_packages() + build_tools_versions = [] + lines = available_packages.split('\n') + + for line in lines: + if not line.strip().startswith('build-tools;'): + continue + package_name = line.strip().split(' ')[0] + if package_name.count(';') != 1: + raise RuntimeError(f"Unable to parse build-tools version: {package_name}") + ver = package_name.split(';')[1] + + build_tools_versions.append(version.Version(ver)) + + return build_tools_versions + + +def find_installed_buildtools_version(build_tools_dir: Path): + """ + It is possible that the user has multiple build-tools installed. The newer version is generally + used. This function find the newest among the installed build-tools + """ + versions = [version.Version(bt_dir.name) for bt_dir in build_tools_dir.iterdir() + if bt_dir.is_dir()] + return max(versions) + + +def find_latest_buildtools_version(sdk_manager: SdkManager): + """ + Uses sdk manager to find the latest build-tools version + """ + available_build_tools_v = android_list_build_tools_versions(sdk_manager=sdk_manager) + + if not available_build_tools_v: + raise RuntimeError('Unable to find any build tools available for download') + + # find the latest build tools version that is not a release candidate + # release candidates end has rc in the version number + available_build_tools_v = [v for v in available_build_tools_v if "rc" not in str(v)] + + return max(available_build_tools_v) + + +def install_android_packages(android_sdk_dir: Path, android_api: str, dry_run: bool = False, + accept_license: bool = False, skip_update: bool = False): + """ + Use the sdk manager to install build-tools, platform-tools and platform API + """ + tools_dir = android_sdk_dir / "tools" + if not tools_dir.exists(): + raise RuntimeError("Unable to find Android command-line tools in " + f"{str(tools_dir)}") + + # incase of --verbose flag + show_output = (logging.getLogger().getEffectiveLevel() == logging.INFO) + + sdk_manager = SdkManager(android_sdk_dir=android_sdk_dir, dry_run=dry_run) + + # install/upgrade platform-tools + if not (android_sdk_dir / "platform-tools").exists(): + print("Installing/Updating Android platform-tools") + sdk_manager.install("platform-tools", accept_license=accept_license, + show_stdout=show_output) + # The --update command is only relevant for platform tools + if not skip_update: + sdk_manager.install("--update", show_stdout=show_output) + + # install/upgrade build-tools + buildtools_dir = android_sdk_dir / "build-tools" + + if not buildtools_dir.exists(): + latest_build_tools_v = find_latest_buildtools_version(sdk_manager=sdk_manager) + print(f"Installing Android build-tools version {latest_build_tools_v}") + sdk_manager.install(f"build-tools;{latest_build_tools_v}", show_stdout=show_output) + else: + if not skip_update: + latest_build_tools_v = find_latest_buildtools_version(sdk_manager=sdk_manager) + installed_build_tools_v = find_installed_buildtools_version(buildtools_dir) + if latest_build_tools_v > installed_build_tools_v: + print(f"Updating Android build-tools version to {latest_build_tools_v}") + sdk_manager.install(f"build-tools;{latest_build_tools_v}", show_stdout=show_output) + installed_build_tools_v = latest_build_tools_v + + # install the platform API + platform_api_dir = android_sdk_dir / "platforms" / f"android-{android_api}" + if not platform_api_dir.exists(): + print(f"Installing Android platform API {android_api}") + sdk_manager.install(f"platforms;android-{android_api}", show_stdout=show_output) + + print("Android packages installation done") diff --git a/tools/cross_compile_android/main.py b/tools/cross_compile_android/main.py new file mode 100644 index 000000000..200f494cf --- /dev/null +++ b/tools/cross_compile_android/main.py @@ -0,0 +1,309 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import sys +import logging +import argparse +import stat +import warnings +import shutil +from dataclasses import dataclass + +from pathlib import Path +from git import Repo, RemoteProgress +from tqdm import tqdm +from jinja2 import Environment, FileSystemLoader + +from android_utilities import (run_command, download_android_commandlinetools, + download_android_ndk, install_android_packages) + +# Note: Does not work with PyEnv. Your Host Python should contain openssl. +# also update the version in ShibokenHelpers.cmake if Python version changes. +PYTHON_VERSION = "3.11" + +SKIP_UPDATE_HELP = ("skip the updation of SDK packages build-tools, platform-tools to" + " latest version") + +ACCEPT_LICENSE_HELP = (''' +Accepts license automatically for Android SDK installation. Otherwise, +accept the license manually through command line. +''') + +CLEAN_CACHE_HELP = (''' +Cleans cache stored in $HOME/.pyside6_deploy_cache. +Options: + +1. all - all the cache including Android Ndk, Android Sdk and Cross-compiled Python are deleted. +2. ndk - Only the Android Ndk is deleted. +3. sdk - Only the Android Sdk is deleted. +4. python - The cross compiled Python for all platforms, the cloned CPython, the cross compilation + scripts for all platforms are deleted. +5. toolchain - The CMake toolchain file required for cross-compiling Qt for Python, for all + platforms are deleted. + +If --clean-cache is used and no explicit value is suppied, then `all` is used as default. +''') + + +@dataclass +class PlatformData: + plat_name: str + api_level: str + android_abi: str + qt_plat_name: str + gcc_march: str + plat_bits: str + + +def occp_exists(): + ''' + check if '--only-cross-compile-python' exists in command line arguments + ''' + return "-occp" in sys.argv or "--only-cross-compile-python" in sys.argv + + +class CloneProgress(RemoteProgress): + def __init__(self): + super().__init__() + self.pbar = tqdm() + + def update(self, op_code, cur_count, max_count=None, message=""): + self.pbar.total = max_count + self.pbar.n = cur_count + self.pbar.refresh() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="This tool cross builds CPython for Android and uses that Python to cross build" + "Android Qt for Python wheels", + formatter_class=argparse.RawTextHelpFormatter, + ) + + parser.add_argument("-p", "--plat-name", type=str, nargs="*", + choices=["aarch64", "armv7a", "i686", "x86_64"], + default=["aarch64", "armv7a", "i686", "x86_64"], dest="plat_names", + help="Android target platforms") + + parser.add_argument("-v", "--verbose", help="run in verbose mode", action="store_const", + dest="loglevel", const=logging.INFO) + parser.add_argument("--api-level", type=str, default="26", + help="Minimum Android API level to use") + parser.add_argument("--ndk-path", type=str, help="Path to Android NDK (Preferred r25c)") + # sdk path is needed to compile all the Qt Java Acitivity files into Qt6AndroidBindings.jar + parser.add_argument("--sdk-path", type=str, help="Path to Android SDK") + parser.add_argument("--qt-install-path", type=str, required=not occp_exists(), + help="Qt installation path eg: /home/Qt/6.5.0") + + parser.add_argument("-occp", "--only-cross-compile-python", action="store_true", + help="Only cross compiles Python for the specified Android platform") + + parser.add_argument("--dry-run", action="store_true", help="show the commands to be run") + + parser.add_argument("--skip-update", action="store_true", + help=SKIP_UPDATE_HELP) + + parser.add_argument("--auto-accept-license", action="store_true", + help=ACCEPT_LICENSE_HELP) + + parser.add_argument("--clean-cache", type=str, nargs="?", const="all", + choices=["all", "python", "ndk", "sdk", "toolchain"], + help=CLEAN_CACHE_HELP) + + args = parser.parse_args() + + logging.basicConfig(level=args.loglevel) + pyside_setup_dir = Path(__file__).parents[2].resolve() + qt_install_path = args.qt_install_path + ndk_path = args.ndk_path + sdk_path = args.sdk_path + only_py_cross_compile = args.only_cross_compile_python + android_abi = None + gcc_march = None + plat_bits = None + dry_run = args.dry_run + plat_names = args.plat_names + api_level = args.api_level + skip_update = args.skip_update + auto_accept_license = args.auto_accept_license + clean_cache = args.clean_cache + + # auto download Android NDK and SDK + pyside6_deploy_cache = Path.home() / ".pyside6_android_deploy" + logging.info(f"Cache created at {str(pyside6_deploy_cache.resolve())}") + pyside6_deploy_cache.mkdir(exist_ok=True) + + if pyside6_deploy_cache.exists() and clean_cache: + if clean_cache == "all": + shutil.rmtree(pyside6_deploy_cache) + elif clean_cache == "ndk": + cached_ndk_dir = pyside6_deploy_cache / "android-ndk" + if cached_ndk_dir.exists(): + shutil.rmtree(cached_ndk_dir) + elif clean_cache == "sdk": + cached_sdk_dir = pyside6_deploy_cache / "android-sdk" + if cached_sdk_dir.exists(): + shutil.rmtree(cached_sdk_dir) + elif clean_cache == "python": + cached_cpython_dir = pyside6_deploy_cache / "cpython" + if cached_cpython_dir.exists(): + shutil.rmtree(pyside6_deploy_cache / "cpython") + for cc_python_path in pyside6_deploy_cache.glob("Python-*"): + if cc_python_path.is_dir(): + shutil.rmtree(cc_python_path) + elif clean_cache == "toolchain": + for toolchain_path in pyside6_deploy_cache.glob("toolchain_*"): + if toolchain_path.is_file(): + toolchain_path.unlink() + + if not ndk_path: + # Download android ndk + ndk_path = download_android_ndk(pyside6_deploy_cache) + + if not sdk_path: + # download and unzip command-line tools + sdk_path = download_android_commandlinetools(pyside6_deploy_cache) + # install and update required android packages + install_android_packages(android_sdk_dir=sdk_path, android_api=api_level, dry_run=dry_run, + accept_license=auto_accept_license, skip_update=skip_update) + + templates_path = Path(__file__).parent / "templates" + + for plat_name in plat_names: + # for armv7a the API level dependent binaries like clang are named + # armv7a-linux-androideabi27-clang, as opposed to other platforms which + # are named like x86_64-linux-android27-clang + platform_data = None + if plat_name == "armv7a": + platform_data = PlatformData("armv7a", api_level, "armeabi-v7a", "armv7", + "armv7", "32") + elif plat_name == "aarch64": + platform_data = PlatformData("aarch64", api_level, "arm64-v8a", "arm64_v8a", "armv8-a", + "64") + elif plat_name == "i686": + platform_data = PlatformData("i686", api_level, "x86", "x86", "i686", "32") + else: # plat_name is x86_64 + platform_data = PlatformData("x86_64", api_level, "x86_64", "x86_64", "x86-64", "64") + + # python path is valid, if Python for android installation exists in python_path + python_path = (pyside6_deploy_cache + / f"Python-{platform_data.plat_name}-linux-android" / "_install") + valid_python_path = python_path.exists() + if Path(python_path).exists(): + expected_dirs = ["lib", "include"] + for expected_dir in expected_dirs: + if not (Path(python_path) / expected_dir).is_dir(): + valid_python_path = False + warnings.warn( + f"{str(python_path.resolve())} is corrupted. New Python for {plat_name} " + f"android will be cross-compiled into {str(pyside6_deploy_cache.resolve())}" + ) + break + + environment = Environment(loader=FileSystemLoader(templates_path)) + if not valid_python_path: + # clone cpython and checkout 3.10 + cpython_dir = pyside6_deploy_cache / "cpython" + python_ccompile_script = cpython_dir / f"cross_compile_{plat_name}.sh" + + if not cpython_dir.exists(): + logging.info(f"cloning cpython {PYTHON_VERSION}") + Repo.clone_from( + "https://github.com/python/cpython.git", + cpython_dir, + progress=CloneProgress(), + branch=PYTHON_VERSION, + ) + + if not python_ccompile_script.exists(): + host_system_config_name = run_command("./config.guess", cwd=cpython_dir, + dry_run=dry_run, show_stdout=True, + capture_stdout=True).strip() + + # use jinja2 to create cross_compile.sh script + template = environment.get_template("cross_compile.tmpl.sh") + content = template.render( + plat_name=platform_data.plat_name, + ndk_path=ndk_path, + api_level=platform_data.api_level, + android_py_install_path_prefix=pyside6_deploy_cache, + host_python_path=sys.executable, + python_version=PYTHON_VERSION, + host_system_name=host_system_config_name, + host_platform_name=sys.platform + ) + + logging.info(f"Writing Python cross compile script into {python_ccompile_script}") + with open(python_ccompile_script, mode="w", encoding="utf-8") as ccompile_script: + ccompile_script.write(content) + + # give run permission to cross compile script + python_ccompile_script.chmod(python_ccompile_script.stat().st_mode | stat.S_IEXEC) + + # clean built files + logging.info("Cleaning CPython built files") + run_command(["make", "distclean"], cwd=cpython_dir, dry_run=dry_run, ignore_fail=True) + + # run the cross compile script + logging.info(f"Running Python cross-compile for platform {platform_data.plat_name}") + run_command([f"./{python_ccompile_script.name}"], cwd=cpython_dir, dry_run=dry_run, + show_stdout=True) + + logging.info( + f"Cross compile Python for Android platform {platform_data.plat_name}. " + f"Final installation in {python_path}" + ) + + if only_py_cross_compile: + continue + + if only_py_cross_compile: + requested_platforms = ",".join(plat_names) + print(f"Python for Android platforms: {requested_platforms} cross compiled " + f"to {str(pyside6_deploy_cache)}") + sys.exit(0) + + qfp_toolchain = pyside6_deploy_cache / f"toolchain_{platform_data.plat_name}.cmake" + + if not qfp_toolchain.exists(): + template = environment.get_template("toolchain_default.tmpl.cmake") + content = template.render( + ndk_path=ndk_path, + sdk_path=sdk_path, + api_level=platform_data.api_level, + qt_install_path=qt_install_path, + plat_name=platform_data.plat_name, + android_abi=platform_data.android_abi, + qt_plat_name=platform_data.qt_plat_name, + gcc_march=platform_data.gcc_march, + plat_bits=platform_data.plat_bits, + python_version=PYTHON_VERSION, + target_python_path=python_path + ) + + logging.info(f"Writing Qt for Python toolchain file into {qfp_toolchain}") + with open(qfp_toolchain, mode="w", encoding="utf-8") as ccompile_script: + ccompile_script.write(content) + + # give run permission to cross compile script + qfp_toolchain.chmod(qfp_toolchain.stat().st_mode | stat.S_IEXEC) + + if sys.platform == "linux": + host_qt_install_suffix = "gcc_64" + elif sys.platform == "darwin": + host_qt_install_suffix = "macos" + else: + raise RuntimeError("Qt for Python cross compilation not supported on this platform") + + # run the cross compile script + logging.info(f"Running Qt for Python cross-compile for platform {platform_data.plat_name}") + qfp_ccompile_cmd = [sys.executable, "setup.py", "bdist_wheel", "--parallel=9", + "--standalone", + f"--cmake-toolchain-file={str(qfp_toolchain.resolve())}", + f"--qt-host-path={qt_install_path}/{host_qt_install_suffix}", + f"--plat-name=android_{platform_data.plat_name}", + f"--python-target-path={python_path}", + (f"--qt-target-path={qt_install_path}/" + f"android_{platform_data.qt_plat_name}"), + "--no-qt-tools"] + run_command(qfp_ccompile_cmd, cwd=pyside_setup_dir, dry_run=dry_run, show_stdout=True) diff --git a/tools/cross_compile_android/requirements.txt b/tools/cross_compile_android/requirements.txt new file mode 100644 index 000000000..62e8ee3b0 --- /dev/null +++ b/tools/cross_compile_android/requirements.txt @@ -0,0 +1,3 @@ +gitpython +Jinja2 +tqdm diff --git a/tools/cross_compile_android/templates/cross_compile.tmpl.sh b/tools/cross_compile_android/templates/cross_compile.tmpl.sh new file mode 100644 index 000000000..784e822ca --- /dev/null +++ b/tools/cross_compile_android/templates/cross_compile.tmpl.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +set -x -e +export HOST_ARCH={{ plat_name }}-linux-android +export TOOLCHAIN={{ ndk_path }}/toolchains/llvm/prebuilt/{{ host_platform_name }}-x86_64/bin +export TOOL_PREFIX=$TOOLCHAIN/$HOST_ARCH +export PLATFORM_API={{ api_level }} +{% if plat_name == "armv7a" -%} +export CXX=${TOOL_PREFIX}eabi${PLATFORM_API}-clang++ +export CPP="${TOOL_PREFIX}eabi${PLATFORM_API}-clang++ -E" +export CC=${TOOL_PREFIX}eabi${PLATFORM_API}-clang +{% else %} +export CXX=${TOOL_PREFIX}${PLATFORM_API}-clang++ +export CPP="${TOOL_PREFIX}${PLATFORM_API}-clang++ -E" +export CC=${TOOL_PREFIX}${PLATFORM_API}-clang +{% endif %} +export AR=$TOOLCHAIN/llvm-ar +export RANLIB=$TOOLCHAIN/llvm-ranlib +export LD=$TOOLCHAIN/ld +export READELF=$TOOLCHAIN/llvm-readelf +export CFLAGS='-fPIC -DANDROID' +./configure --host=$HOST_ARCH --target=$HOST_ARCH --build={{ host_system_name }} \ +--with-build-python={{ host_python_path }} --enable-shared \ +--enable-ipv6 ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no --without-ensurepip \ +ac_cv_little_endian_double=yes +make BLDSHARED="$CC -shared" CROSS-COMPILE=$TOOL_PREFIX- CROSS_COMPILE_TARGET=yes \ +INSTSONAME=libpython{{ python_version }}.so +make install prefix={{ android_py_install_path_prefix }}/Python-$HOST_ARCH/_install diff --git a/tools/cross_compile_android/templates/toolchain_default.tmpl.cmake b/tools/cross_compile_android/templates/toolchain_default.tmpl.cmake new file mode 100644 index 000000000..3c9752f43 --- /dev/null +++ b/tools/cross_compile_android/templates/toolchain_default.tmpl.cmake @@ -0,0 +1,73 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +# toolchain file to cross compile Qt for Python wheels for Android +cmake_minimum_required(VERSION 3.23) +include_guard(GLOBAL) +set(CMAKE_SYSTEM_NAME Android) +{% if plat_name == "armv7a" -%} +set(CMAKE_SYSTEM_PROCESSOR armv7-a) +{% else %} +set(CMAKE_SYSTEM_PROCESSOR {{ plat_name }}) +{% endif %} +set(CMAKE_ANDROID_API {{ api_level }}) +set(CMAKE_ANDROID_NDK {{ ndk_path }}) +set(CMAKE_ANDROID_ARCH_ABI {{ android_abi }}) +set(CMAKE_ANDROID_NDK_TOOLCHAIN_VERSION clang) +set(CMAKE_ANDROID_STL_TYPE c++_shared) +if(NOT DEFINED ANDROID_PLATFORM AND NOT DEFINED ANDROID_NATIVE_API_LEVEL) + set(ANDROID_PLATFORM "android-{{ api_level }}" CACHE STRING "") +endif() +set(ANDROID_SDK_ROOT {{ sdk_path }}) +{% if plat_name == "armv7a" -%} +set(_TARGET_NAME_ENDING "eabi{{ api_level }}") +{% else %} +set(_TARGET_NAME_ENDING "{{ api_level }}") +{% endif %} +set(QT_COMPILER_FLAGS "--target={{ plat_name }}-linux-android${_TARGET_NAME_ENDING} \ + -fomit-frame-pointer \ + -march={{ gcc_march }} \ + -msse4.2 \ + -mpopcnt \ + -m{{ plat_bits }} \ + -fPIC \ + -I{{ target_python_path }}/include/python{{ python_version }} \ + -Wno-unused-command-line-argument") +set(QT_COMPILER_FLAGS_RELEASE "-O2 -pipe") + +# FIXME +# https://gitlab.kitware.com/cmake/cmake/-/issues/23670 +# The CMake Android toolchain does not allow RPATHS. Hence CMAKE_INSTALL_RPATH does not work. +# Currently the linker flags are set directly as -Wl,-rpath='$ORIGIN' -Wl,-rpath='$ORIGIN/Qt/lib' +# set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) +# set(CMAKE_INSTALL_RPATH "$ORIGIN") + +set(QT_LINKER_FLAGS "-Wl,-O1 -Wl,--hash-style=gnu -Wl,-rpath='$ORIGIN' -Wl,-rpath='$ORIGIN/Qt/lib' \ + -Wl,--as-needed -L{{ qt_install_path }}/android_{{ qt_plat_name }}/lib \ + -L{{ qt_install_path }}/android_{{ qt_plat_name }}/plugins/platforms \ + -L{{ target_python_path }}/lib \ + -lpython{{ python_version }}") +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +add_compile_definitions(ANDROID) + +include(CMakeInitializeConfigs) +function(cmake_initialize_per_config_variable _PREFIX _DOCSTRING) + if (_PREFIX MATCHES "CMAKE_(C|CXX|ASM)_FLAGS") + set(CMAKE_${CMAKE_MATCH_1}_FLAGS_INIT "${QT_COMPILER_FLAGS}") + foreach (config DEBUG RELEASE MINSIZEREL RELWITHDEBINFO) + if (DEFINED QT_COMPILER_FLAGS_${config}) + set(CMAKE_${CMAKE_MATCH_1}_FLAGS_${config}_INIT "${QT_COMPILER_FLAGS_${config}}") + endif() + endforeach() + endif() + if (_PREFIX MATCHES "CMAKE_(SHARED|MODULE|EXE)_LINKER_FLAGS") + foreach (config SHARED MODULE EXE) + set(CMAKE_${config}_LINKER_FLAGS_INIT "${QT_LINKER_FLAGS}") + endforeach() + endif() + _cmake_initialize_per_config_variable(${ARGV}) +endfunction() diff --git a/tools/debug_renamer.py b/tools/debug_renamer.py new file mode 100644 index 000000000..ec777388b --- /dev/null +++ b/tools/debug_renamer.py @@ -0,0 +1,113 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +import re +import sys +from argparse import ArgumentParser, FileType, RawTextHelpFormatter +from collections import OrderedDict + +DESC = """ +debug_renamer.py +================ + +This script renames object addresses in debug protocols to useful names. +Comparing output will produce minimal deltas. + + +Problem: +-------- + +In the debugging output of PYSIDE-79, we want to study different output +before and after applying some change to the implementation. + +We have support from the modified Python interpreter that creates full +traces of every object creation and increment/decrement of refcounts. + +The comparison between "before" and "after" gets complicated because +the addresses of objects do not compare well. + + +Input format: +------------- +The Python output lines can be freely formatted. + +Any line which contains "0x.." followed by some name will be changed, +all others are left alone. + + +To Do List +---------- + +Names of objects which are already deleted should be monitored and +not by chance be re-used. We need to think of a way to specify deletion. +""" + + +def make_name(typename, name_pos): + """ + Build a name by using uppercase letters and numbers + """ + if name_pos < 26: + name = chr(ord("A") + name_pos) + return f"{typename}_{name}" + return f"{typename}_{str(name_pos)}" + + +known_types = {} +pattern = r"0x\w+\s+\S+" # hex word followed by non-WS +rex = re.compile(pattern, re.IGNORECASE) + + +def rename_hexval(line): + if not (res := rex.search(line)): + return line + start_pos, end_pos = res.start(), res.end() + beg, mid, end = line[:start_pos], line[start_pos:end_pos], line[end_pos:] + object_id, typename = mid.split() + if int(object_id, 16) == 0: + return(f"{beg}{typename}_NULL{end}") + if typename not in known_types: + known_types[typename] = OrderedDict() + obj_store = known_types[typename] + if object_id not in obj_store: + obj_store[object_id] = make_name(typename, len(obj_store)) + return(f"{beg}{obj_store[object_id]}{end}") + + +def hide_floatval(line): + return re.sub(r"\d+\.\d+", "<float>", line) + + +def process_all_lines(options): + """ + Process all lines from fin to fout. + The caller is responsible of opening and closing files if at all. + """ + fi, fo = options.input, options.output + rename = options.rename + float_ = options.float + while line := fi.readline(): + if rename: + line = rename_hexval(line) + if float_: + line = hide_floatval(line) + fo.write(line) + + +def create_argument_parser(desc): + parser = ArgumentParser(description=desc, formatter_class=RawTextHelpFormatter) + parser.add_argument("--rename", "-r", action="store_true", + help="Rename hex value and following word to a readable f'{word}_{anum}'") + parser.add_argument("--float", "-f", action="store_true", + help="Replace timing numbers by '<float>' (for comparing ctest output)") + parser.add_argument("--input", "-i", nargs="?", type=FileType("r"), default=sys.stdin, + help="Use the specified file instead of sys.stdin") + parser.add_argument("--output", "-o", nargs="?", type=FileType("w"), default=sys.stdout, + help="Use the specified file instead of sys.stdout") + return parser + + +if __name__ == "__main__": + argument_parser = create_argument_parser(DESC) + options = argument_parser.parse_args() + process_all_lines(options) diff --git a/tools/debug_windows.py b/tools/debug_windows.py new file mode 100644 index 000000000..832282895 --- /dev/null +++ b/tools/debug_windows.py @@ -0,0 +1,329 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import argparse +import ctypes +import logging +import re +import subprocess +import sys +from os import path +from textwrap import dedent + +is_win = sys.platform == "win32" +if is_win: + import winreg + + +EPILOG = """ +This is a troubleshooting script that assists finding out which DLLs or +which symbols in a DLL are missing when executing a PySide6 python +script. +It can also be used with any other non Python executable. + +Usage: python debug_windows.py + When no arguments are given the script will try to import + PySide6.QtCore. + +Usage: python debug_windows.py python -c "import PySide6.QtWebEngine" + python debug_windows.py my_executable.exe arg1 arg2 --arg3=4 + Any arguments given after the script name will be considered + as the target executable and the arguments passed to that + executable. + +The script requires administrator privileges. + +The script uses cdb.exe and gflags.exe under the hood, which are +installed together with the Windows Kit found at: +https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk + +""" + + +def get_parser_args(): + desc_msg = "Run an executable under cdb with loader snaps set." + help_msg = "Pass the executable and the arguments passed to it as a list." + parser = argparse.ArgumentParser(description=desc_msg, + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=EPILOG) + parser.add_argument('args', nargs='*', help=help_msg) + # Prepend -- so that python options like '-c' are ignored by + # argparse. + help_requested = '-h' in sys.argv or '--help' in sys.argv + massaged_args = ['--'] + sys.argv[1:] if not help_requested else sys.argv + return parser, parser.parse_args(massaged_args) + + +verbose_log_file_name = path.join(path.dirname(path.abspath(__file__)), + 'log_debug_windows.txt') + + +def is_admin(): + try: + return ctypes.windll.shell32.IsUserAnAdmin() + except Exception as e: + log.error("is_admin: Exception error: {}".format(e)) + return False + + +def get_verbose_logger(): + handler = logging.FileHandler(verbose_log_file_name, mode='w') + main_logger = logging.getLogger('main') + main_logger.setLevel(logging.INFO) + main_logger.addHandler(handler) + return main_logger + + +def get_non_verbose_logger(): + handler = logging.StreamHandler() + main_logger = logging.getLogger('main.non_verbose') + main_logger.setLevel(logging.INFO) + main_logger.addHandler(handler) + return main_logger + + +big_log = get_verbose_logger() +log = get_non_verbose_logger() + + +def sub_keys(key): + i = 0 + while True: + try: + sub_key = winreg.EnumKey(key, i) + yield sub_key + i += 1 + except WindowsError as e: + log.error(e) + break + + +def sub_values(key): + i = 0 + while True: + try: + v = winreg.EnumValue(key, i) + yield v + i += 1 + except WindowsError as e: + log.error(e) + break + + +def get_installed_windows_kits(): + roots_key = r"SOFTWARE\Microsoft\Windows Kits\Installed Roots" + log.info("Searching for Windows kits in registry path: " + "{}".format(roots_key)) + + kits = [] + pattern = re.compile(r'KitsRoot(\d+)') + try: + roots = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, roots_key, 0, + winreg.KEY_READ) + + for (name, value, value_type) in sub_values(roots): + if value_type == winreg.REG_SZ and name.startswith('KitsRoot'): + match = pattern.search(name) + if match: + version = match.group(1) + kits.append({'version': version, 'value': value}) + + except WindowsError as e: + log.exception(e) + + if not kits: + log.error(dedent(""" + No windows kits found in the registry. + Consider downloading and installing the latest kit, either from + https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/debugger-download-tools + or from + https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk + """)) + exit(1) + return kits + + +def get_appropriate_kit(kits): + # Fixme, figure out if there is a more special way to choose a kit + # and not just latest version. + log.info("Found Windows kits are: {}".format(kits)) + chosen_kit = {'version': "0", 'value': None} + for kit in kits: + if (kit['version'] > chosen_kit['version'] + # version 8.1 is actually '81', so consider everything + # above version 20, as '2.0', etc. + and kit['version'] < "20"): + chosen_kit = kit + first_kit = kits[0] + return first_kit + + +def get_cdb_and_gflags_path(kits): + first_kit = get_appropriate_kit(kits) + first_path_path = first_kit['value'] + log.info('Using kit found at {}'.format(first_path_path)) + bits = 'x64' if (sys.maxsize > 2 ** 32) else 'x32' + debuggers_path = path.join(first_path_path, 'Debuggers', bits) + cdb_path = path.join(debuggers_path, 'cdb.exe') + # Try for older "Debugging Tools" packages + if not path.exists(cdb_path): + debuggers_path = "C:\\Program Files\\Debugging Tools for Windows (x64)" + cdb_path = path.join(debuggers_path, 'cdb.exe') + + if not path.exists(cdb_path): + log.error("Couldn't find cdb.exe at: {}.".format(cdb_path)) + exit(1) + else: + log.info("Found cdb.exe at: {}.".format(cdb_path)) + + gflags_path = path.join(debuggers_path, 'gflags.exe') + + if not path.exists(gflags_path): + log.error('Couldn\'t find gflags.exe at: {}.'.format(gflags_path)) + exit(1) + else: + log.info('Found gflags.exe at: {}.'.format(cdb_path)) + + return cdb_path, gflags_path + + +def toggle_loader_snaps(executable_name, gflags_path, enable=True): + arg = '+sls' if enable else '-sls' + gflags_args = [gflags_path, '-i', executable_name, arg] + try: + log.info('Invoking gflags: {}'.format(gflags_args)) + output = subprocess.check_output(gflags_args, stderr=subprocess.STDOUT, + universal_newlines=True) + log.info(output) + except WindowsError as e: + log.error("\nRunning {} exited with exception: " + "\n{}".format(gflags_args, e)) + exit(1) + except subprocess.CalledProcessError as e: + log.error("\nRunning {} exited with: {} and stdout was: " + "{}".format(gflags_args, e.returncode, e.output)) + exit(1) + + +def find_error_like_snippets(content): + snippets = [] + lines = content.splitlines() + context_lines = 4 + + def error_predicate(line): + # A list of mostly false positives are filtered out. + # For deeper inspection, the full log exists. + errors = {'errorhandling', + 'windowserrorreporting', + 'core-winrt-error', + 'RtlSetLastWin32Error', + 'RaiseInvalid16BitExeError', + 'BaseWriteErrorElevationRequiredEvent', + 'for DLL "Unknown"', + 'LdrpGetProcedureAddress', + 'X509_STORE_CTX_get_error', + 'ERR_clear_error', + 'ERR_peek_last_error', + 'ERR_error_string', + 'ERR_get_error', + ('ERROR: Module load completed but symbols could ' + 'not be loaded')} + return (re.search('error', line, re.IGNORECASE) + and all(e not in line for e in errors)) + + for i in range(1, len(lines)): + line = lines[i] + if error_predicate(line): + snippets.append(lines[i - context_lines:i + context_lines + 1]) + + return snippets + + +def print_error_snippets(snippets): + if len(snippets) > 0: + log.info("\nThe following possible errors were found:\n") + + for i in range(1, len(snippets)): + log.info("Snippet {}:".format(i)) + for line in snippets[i]: + log.info(line) + log.info("") + + +def call_command_under_cdb_with_gflags(executable_path, args): + executable_name = path.basename(executable_path) + invocation = [executable_path] + args + + kits = get_installed_windows_kits() + cdb_path, gflags_path = get_cdb_and_gflags_path(kits) + + toggle_loader_snaps(executable_name, gflags_path, enable=True) + + log.info("Debugging the following command invocation: " + "{}".format(invocation)) + + cdb_args = [cdb_path] + invocation + + log.info('Invoking cdb: {}'.format(cdb_args)) + + p = subprocess.Popen(cdb_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + shell=False) + + # Symbol fix, start process, print all thread stack traces, exit. + cdb_commands = ['.symfix', 'g', '!uniqstack', 'q'] + cdb_commands_text = '\n'.join(cdb_commands) + out, err = p.communicate(input=cdb_commands_text.encode('utf-8')) + + out_decoded = out.decode('utf-8') + big_log.info('stdout: {}'.format(out_decoded)) + if err: + big_log.info('stderr: {}'.format(err.decode('utf-8'))) + + log.info('Finished execution of process under cdb.') + + toggle_loader_snaps(executable_name, gflags_path, enable=False) + + snippets = find_error_like_snippets(out_decoded) + print_error_snippets(snippets) + + log.info("Finished processing.\n !!! Full log can be found at:\n" + "{}".format(verbose_log_file_name)) + + +def test_run_import_qt_core_under_cdb_with_gflags(): + # The weird characters are there for faster grepping of the output + # because there is a lot of content in the full log. + # The 2+2 is just ensure that Python itself works. + python_code = """ +print(">>>>>>>>>>>>>>>>>>>>>>> Test computation of 2+2 is: {}".format(2+2)) +import PySide6.QtCore +print(">>>>>>>>>>>>>>>>>>>>>>> QtCore object instance: {}".format(PySide6.QtCore)) +""" + call_command_under_cdb_with_gflags(sys.executable, ["-c", python_code]) + + +def handle_args(parser_args): + if not parser_args.args: + test_run_import_qt_core_under_cdb_with_gflags() + else: + call_command_under_cdb_with_gflags(parser_args.args[0], + parser_args.args[1:]) + + +if __name__ == '__main__': + if not is_win: + log.error("This script only works on Windows.") + exit(1) + + parser, parser_args = get_parser_args() + + if is_admin(): + handle_args(parser_args) + else: + log.error("Please rerun the script with administrator privileges. " + "It is required for gflags.exe to work. ") + parser.print_help() + exit(1) diff --git a/tools/doc_modules.py b/tools/doc_modules.py new file mode 100644 index 000000000..d46f4db02 --- /dev/null +++ b/tools/doc_modules.py @@ -0,0 +1,209 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import os +import subprocess +import sys +from argparse import ArgumentParser, RawTextHelpFormatter +from pathlib import Path +import xml.sax +from xml.sax.handler import ContentHandler + +DESC = """Print a list of module short names ordered by typesystem dependencies +for which documentation can be built by intersecting the PySide6 modules with +the modules built in Qt.""" + + +ROOT_DIR = Path(__file__).parents[1].resolve() +SOURCE_DIR = ROOT_DIR / "sources" / "pyside6" / "PySide6" + + +qt_version = None +qt_include_dir = None + + +class TypeSystemContentHandler(ContentHandler): + """XML SAX content handler that extracts required modules from the + "load-typesystem" elements of the typesystem_file. Nodes that start + with Qt and are marked as generate == "no" are considered required.""" + + def __init__(self): + self.required_modules = [] + + def startElement(self, name, attrs): + if name == "load-typesystem": + generate = attrs.get("generate", "").lower() + if generate == "no" or generate == "false": + load_file_name = attrs.get("name") # "QtGui/typesystem_gui.xml" + if load_file_name.startswith("Qt"): + slash = load_file_name.find("/") + if slash > 0: + self.required_modules.append(load_file_name[:slash]) + + +def required_typesystems(module): + """Determine the required Qt modules by looking at the "load-typesystem" + elements of the typesystem_file.""" + name = module[2:].lower() + typesystem_file = SOURCE_DIR / module / f"typesystem_{name}.xml" + # Use a SAX parser since that works despite undefined entity + # errors for typesystem entities. + handler = TypeSystemContentHandler() + try: + parser = xml.sax.make_parser() + parser.setContentHandler(handler) + parser.parse(typesystem_file) + except Exception as e: + print(f"Warning: XML error parsing {typesystem_file}: {e}", file=sys.stderr) + return handler.required_modules + + +def query_qtpaths(keyword): + query_cmd = ["qtpaths", "-query", keyword] + output = subprocess.check_output(query_cmd, stderr=subprocess.STDOUT, + universal_newlines=True) + return output.strip() + + +def sort_modules(dependency_dict): + """Sort the modules by dependencies using brute force: Keep adding + modules all of whose requirements are present to the result list + until done.""" + result = [] + while True: + found = False + for module, dependencies in dependency_dict.items(): + if module not in result: + if all(dependency in result for dependency in dependencies): + result.append(module) + found = True + if not found: + break + + if len(result) < len(dependency_dict) and verbose: + for desired_module in dependency_dict.keys(): + if desired_module not in result: + print(f"Not documenting {desired_module} (missing dependency)", + file=sys.stderr) + return result + + +def _write_type_system(modules, file): + """Helper to write the type system for shiboken. It needs to be in + dependency order to prevent shiboken from loading the included + typesystems with generate="no", which causes those modules to be + missing.""" + for module in modules: + name = module[2:].lower() + filename = f"{module}/typesystem_{name}.xml" + print(f' <load-typesystem name="{filename}" generate="yes"/>', + file=file) + print("</typesystem>", file=file) + + +def write_type_system(modules, filename): + """Write the type system for shiboken in dependency order.""" + if filename == "-": + _write_type_system(modules, sys.stdout) + else: + path = Path(filename) + exists = path.exists() + with path.open(mode="a") as f: + if not exists: + print('<typesystem package="PySide">', file=f) + _write_type_system(modules, f) + + +def _write_global_header(modules, file): + """Helper to write the global header for shiboken.""" + for module in modules: + print(f"#include <{module}/{module}>", file=file) + + +def write_global_header(modules, filename): + """Write the global header for shiboken.""" + if filename == "-": + _write_global_header(modules, sys.stdout) + else: + with Path(filename).open(mode="a") as f: + _write_global_header(modules, f) + + +def _write_docconf(modules, file): + """Helper to write the include paths for the .qdocconf file.""" + # @TODO fix this for macOS frameworks. + for module in modules: + root = f" -I/{qt_include_dir}/{module}" + print(f"{root} \\", file=file) + print(f"{root}/{qt_version} \\", file=file) + print(f"{root}/{qt_version}/{module} \\", file=file) + + +def write_docconf(modules, filename): + """Write the include paths for the .qdocconf file.""" + if filename == "-": + _write_docconf(modules, sys.stdout) + else: + with Path(filename).open(mode="a") as f: + _write_docconf(modules, f) + + +if __name__ == "__main__": + argument_parser = ArgumentParser(description=DESC, + formatter_class=RawTextHelpFormatter) + argument_parser.add_argument("--verbose", "-v", action="store_true", + help="Verbose") + argument_parser.add_argument("qt_include_dir", help="Qt Include dir", + nargs='?', type=str) + argument_parser.add_argument("qt_version", help="Qt version string", + nargs='?', type=str) + argument_parser.add_argument("--typesystem", "-t", help="Typesystem file to write", + action="store", type=str) + argument_parser.add_argument("--global-header", "-g", help="Global header to write", + action="store", type=str) + argument_parser.add_argument("--docconf", "-d", help="docconf file to write", + action="store", type=str) + + options = argument_parser.parse_args() + verbose = options.verbose + if options.qt_include_dir: + qt_include_dir = Path(options.qt_include_dir) + if not qt_include_dir.is_dir(): + print(f"Invalid include directory passed: {options.qt_include_dir}", + file=sys.stderr) + sys.exit(-1) + else: + verbose = True # Called by hand to find out about available modules + query_cmd = ["qtpaths", "-query", "QT_INSTALL_HEADERS"] + qt_include_dir = Path(query_qtpaths("QT_INSTALL_HEADERS")) + if not qt_include_dir.is_dir(): + print("Cannot determine include directory", file=sys.stderr) + sys.exit(-1) + + qt_version = options.qt_version if options.qt_version else query_qtpaths("QT_VERSION") + + # Build a typesystem dependency dict of the available modules in order + # to be able to sort_modules by dependencies. This is required as + # otherwise shiboken will read the required typesystems with + # generate == "no" and thus omit modules. + module_dependency_dict = {} + for m in SOURCE_DIR.glob("Qt*"): + module = m.name + # QtGraphs duplicates symbols from QtDataVisualization causing shiboken errors + if module == "QtDataVisualization": + continue + qt_include_path = qt_include_dir / module + if qt_include_path.is_dir(): + module_dependency_dict[module] = required_typesystems(module) + elif verbose: + print(f"Not documenting {module} (not built)", file=sys.stderr) + + modules = sort_modules(module_dependency_dict) + print(" ".join([m[2:] for m in modules])) + + if options.typesystem: + write_type_system(modules, options.typesystem) + if options.global_header: + write_global_header(modules, options.global_header) + if options.docconf: + write_docconf(modules, options.docconf) diff --git a/tools/dump_metaobject.py b/tools/dump_metaobject.py new file mode 100644 index 000000000..6898e9317 --- /dev/null +++ b/tools/dump_metaobject.py @@ -0,0 +1,129 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +"""Helper functions for formatting information on QMetaObject""" + +from PySide6.QtCore import QMetaMethod + + +def _qbytearray_to_string(b): + return bytes(b.data()).decode('utf-8') + + +def _format_metatype(meta_type): + return meta_type.id() if meta_type.isValid() else '<invalid>' + + +def _dump_metaobject_helper(meta_obj, indent): + meta_id = 0 + # FIXME: Otherwise crashes in Qt + if meta_obj.propertyOffset() < meta_obj.propertyCount(): + meta_id = _format_metatype(meta_obj.metaType()) + print(f'{indent}class {meta_obj.className()}/{meta_id}:') + indent += ' ' + + info_offset = meta_obj.classInfoOffset() + info_count = meta_obj.classInfoCount() + if info_offset < info_count: + print(f'{indent}Info:') + for i in range(info_offset, info_count): + name = meta_obj.classInfo(i).name() + value = meta_obj.classInfo(i).value() + print(f'{indent}{i:4d} {name}+{value}') + + enumerator_offset = meta_obj.enumeratorOffset() + enumerator_count = meta_obj.enumeratorCount() + if enumerator_offset < enumerator_count: + print(f'{indent}Enumerators:') + for e in range(enumerator_offset, enumerator_count): + meta_enum = meta_obj.enumerator(e) + name = meta_enum.name() + value_str = '' + descr = '' + if meta_enum.isFlag(): + descr += ' flag' + if meta_enum.isScoped(): + descr += ' scoped' + for k in range(meta_enum.keyCount()): + if k > 0: + value_str += ', ' + key = meta_enum.key(k) + value = meta_enum.value(k) + value_str += f'{key} = {value}' + print(f'{indent}{e:4d} {name}{descr} ({value_str})') + + property_offset = meta_obj.propertyOffset() + property_count = meta_obj.propertyCount() + if property_offset < property_count: + print(f'{indent}Properties:') + for p in range(property_offset, property_count): + meta_property = meta_obj.property(p) + name = meta_property.name() + desc = '' + if meta_property.isConstant(): + desc += ', constant' + if meta_property.isDesignable(): + desc += ', designable' + if meta_property.isFlagType(): + desc += ', flag' + if meta_property.isEnumType(): + desc += ', enum' + if meta_property.isStored(): + desc += ', stored' + if meta_property.isWritable(): + desc += ', writable' + if meta_property.isResettable(): + desc += ', resettable' + if meta_property.hasNotifySignal(): + notify_name_b = meta_property.notifySignal().name() + notify_name = _qbytearray_to_string(notify_name_b) + desc += f', notify="{notify_name}"' + meta_id = _format_metatype(meta_property.metaType()) + type_name = meta_property.typeName() + print(f'{indent}{p:4d} {type_name}/{meta_id} "{name}"{desc}') + + method_offset = meta_obj.methodOffset() + method_count = meta_obj.methodCount() + if method_offset < method_count: + print('{}Methods:'.format(indent)) + for m in range(method_offset, method_count): + method = meta_obj.method(m) + signature = _qbytearray_to_string(method.methodSignature()) + access = '' + if method.access() == QMetaMethod.Protected: + access += 'protected ' + elif method.access() == QMetaMethod.Private: + access += 'private ' + type = method.methodType() + typeString = '' + if type == QMetaMethod.Signal: + typeString = ' (Signal)' + elif type == QMetaMethod.Slot: + typeString = ' (Slot)' + elif type == QMetaMethod.Constructor: + typeString = ' (Ct)' + type_name = method.typeName() + desc = f'{indent}{m:4d} {access}{type_name} "{signature}"{typeString}' + parameter_names = method.parameterNames() + if parameter_names: + parameter_types = method.parameterTypes() + desc += ' Parameters:' + for p, bname in enumerate(parameter_names): + name = _qbytearray_to_string(bname) + type_name = _qbytearray_to_string(parameter_types[p]) + if not name: + name = '<unnamed>' + desc += f' "{name}": {type_name}' + print(desc) + + +def dump_metaobject(meta_obj): + super_classes = [meta_obj] + super_class = meta_obj.superClass() + while super_class: + super_classes.append(super_class) + super_class = super_class.superClass() + indent = '' + for c in reversed(super_classes): + _dump_metaobject_helper(c, indent) + indent += ' ' diff --git a/tools/example_gallery/main.py b/tools/example_gallery/main.py new file mode 100644 index 000000000..b5aa632c0 --- /dev/null +++ b/tools/example_gallery/main.py @@ -0,0 +1,687 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + + +""" +This tool reads all the examples from the main repository that have a +'.pyproject' file, and generates a special table/gallery in the documentation +page. + +For the usage, simply run: + python tools/example_gallery/main.py +since there is no special requirements. +""" + +import json +import math +import os +import shutil +import zipfile +import sys +from argparse import ArgumentParser, RawTextHelpFormatter +from dataclasses import dataclass +from enum import IntEnum, Enum +from pathlib import Path +from textwrap import dedent + + +class Format(Enum): + RST = 0 + MD = 1 + + +class ModuleType(IntEnum): + ESSENTIALS = 0 + ADDONS = 1 + M2M = 2 + + +SUFFIXES = {Format.RST: "rst", Format.MD: "md"} + + +opt_quiet = False + + +LITERAL_INCLUDE = ".. literalinclude::" + + +IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".svgz", ".webp") + + +IGNORED_SUFFIXES = IMAGE_SUFFIXES + (".pdf", ".pyc", ".obj", ".mesh") + + +suffixes = { + ".h": "cpp", + ".cpp": "cpp", + ".md": "markdown", + ".py": "py", + ".qml": "js", + ".conf": "ini", + ".qrc": "xml", + ".ui": "xml", + ".xbel": "xml", + ".xml": "xml", +} + + +BASE_CONTENT = """\ +Examples +======== + + A collection of examples are provided with |project| to help new users + to understand different use cases of the module. + + You can find all these examples inside the + `pyside-setup <https://code.qt.io/cgit/pyside/pyside-setup.git/>`_ repository + on the `examples <https://code.qt.io/cgit/pyside/pyside-setup.git/tree/examples>`_ + directory. + +""" + + +def ind(x): + return " " * 4 * x + + +def get_lexer(path): + if path.name == "CMakeLists.txt": + return "cmake" + lexer = suffixes.get(path.suffix) + return lexer if lexer else "text" + + +def add_indent(s, level): + new_s = "" + for line in s.splitlines(): + if line.strip(): + new_s += f"{ind(level)}{line}\n" + else: + new_s += "\n" + return new_s + + +def check_img_ext(i): + """Check whether path is an image.""" + return i.suffix in IMAGE_SUFFIXES + + +@dataclass +class ModuleDescription: + """Specifies a sort key and type for a Qt module.""" + sort_key: int = 0 + module_type: ModuleType = ModuleType.ESSENTIALS + description: str = '' + + +MODULE_DESCRIPTIONS = { + "async": ModuleDescription(16, ModuleType.ESSENTIALS, ''), + "corelib": ModuleDescription(15, ModuleType.ESSENTIALS, ''), + "dbus": ModuleDescription(22, ModuleType.ESSENTIALS, ''), + "designer": ModuleDescription(11, ModuleType.ESSENTIALS, ''), + "gui": ModuleDescription(25, ModuleType.ESSENTIALS, ''), + "network": ModuleDescription(20, ModuleType.ESSENTIALS, ''), + "opengl": ModuleDescription(26, ModuleType.ESSENTIALS, ''), + "qml": ModuleDescription(0, ModuleType.ESSENTIALS, ''), + "quick": ModuleDescription(1, ModuleType.ESSENTIALS, ''), + "quickcontrols": ModuleDescription(2, ModuleType.ESSENTIALS, ''), + "samplebinding": ModuleDescription(30, ModuleType.ESSENTIALS, ''), + "scriptableapplication": ModuleDescription(30, ModuleType.ESSENTIALS, ''), + "sql": ModuleDescription(21, ModuleType.ESSENTIALS, ''), + "uitools": ModuleDescription(12, ModuleType.ESSENTIALS, ''), + "widgetbinding": ModuleDescription(30, ModuleType.ESSENTIALS, ''), + "widgets": ModuleDescription(10, ModuleType.ESSENTIALS, ''), + "xml": ModuleDescription(24, ModuleType.ESSENTIALS, ''), + "Qt Demos": ModuleDescription(0, ModuleType.ADDONS, ''), # from Qt repos + "3d": ModuleDescription(30, ModuleType.ADDONS, ''), + "axcontainer": ModuleDescription(20, ModuleType.ADDONS, ''), + "bluetooth": ModuleDescription(20, ModuleType.ADDONS, ''), + "charts": ModuleDescription(12, ModuleType.ADDONS, ''), + "datavisualization": ModuleDescription(11, ModuleType.ADDONS, ''), + "demos": ModuleDescription(0, ModuleType.ADDONS, ''), + "external": ModuleDescription(20, ModuleType.ADDONS, ''), + "graphs": ModuleDescription(10, ModuleType.ADDONS, ''), + "httpserver": ModuleDescription(0, ModuleType.ADDONS, ''), + "location": ModuleDescription(20, ModuleType.ADDONS, ''), + "multimedia": ModuleDescription(12, ModuleType.ADDONS, ''), + "networkauth": ModuleDescription(20, ModuleType.ADDONS, ''), + "pdf": ModuleDescription(20, ModuleType.ADDONS, ''), + "pdfwidgets": ModuleDescription(20, ModuleType.ADDONS, ''), + "quick3d": ModuleDescription(20, ModuleType.ADDONS, ''), + "remoteobjects": ModuleDescription(20, ModuleType.ADDONS, ''), + "serialbus": ModuleDescription(30, ModuleType.ADDONS, ''), + "serialport": ModuleDescription(30, ModuleType.ADDONS, ''), + "spatialaudio": ModuleDescription(20, ModuleType.ADDONS, ''), + "speech": ModuleDescription(20, ModuleType.ADDONS, ''), + "statemachine": ModuleDescription(30, ModuleType.ADDONS, ''), + "webchannel": ModuleDescription(30, ModuleType.ADDONS, ''), + "webenginequick": ModuleDescription(15, ModuleType.ADDONS, ''), + "webenginewidgets": ModuleDescription(16, ModuleType.ADDONS, ''), + "coap": ModuleDescription(0, ModuleType.M2M, ''), + "mqtt": ModuleDescription(0, ModuleType.M2M, ''), + "opcua": ModuleDescription(0, ModuleType.M2M, '') +} + + +def module_sort_key(name): + """Return key for sorting modules.""" + description = MODULE_DESCRIPTIONS.get(name) + module_type = int(description.module_type) if description else 5 + sort_key = description.sort_key if description else 100 + return f"{module_type}:{sort_key:04}:{name}" + + +def module_title(name): + """Return title for a module.""" + result = name.title() + description = MODULE_DESCRIPTIONS.get(name) + if description: + if description.description: + result += " - " + description.description + if description.module_type == ModuleType.M2M: + result += " (M2M)" + elif description.module_type == ModuleType.ADDONS: + result += " (Add-ons)" + else: + result += " (Essentials)" + return result + + +@dataclass +class ExampleData: + """Example data for formatting the gallery.""" + + def __init__(self): + self.headline = "" + + example: str + module: str + extra: str + doc_file: str + file_format: Format + abs_path: str + has_doc: bool + img_doc: Path + headline: str + + +def get_module_gallery(examples): + """ + This function takes a list of dictionaries, that contain examples + information, from one specific module. + """ + + gallery = ( + ".. grid:: 1 4 4 4\n" + f"{ind(1)}:gutter: 2\n\n" + ) + + # Iteration per rows + for i in range(math.ceil(len(examples))): + e = examples[i] + suffix = SUFFIXES[e.file_format] + url = e.doc_file.replace(f".{suffix}", ".html") + name = e.example + underline = e.module + + if e.extra: + underline += f"/{e.extra}" + + if i > 0: + gallery += "\n" + img_name = e.img_doc.name if e.img_doc else "../example_no_image.png" + + # Fix long names + if name.startswith("chapter"): + name = name.replace("chapter", "c") + elif name.startswith("advanced"): + name = name.replace("advanced", "a") + + desc = e.headline + if not desc: + desc = f"found in the ``{underline}`` directory." + + gallery += f"{ind(1)}.. grid-item-card:: {name}\n" + gallery += f"{ind(2)}:class-item: cover-img\n" + gallery += f"{ind(2)}:link: {url}\n" + gallery += f"{ind(2)}:img-top: {img_name}\n\n" + gallery += f"{ind(2)}{desc}\n" + + return f"{gallery}\n" + + +def remove_licenses(s): + new_s = [] + for line in s.splitlines(): + if line.strip().startswith(("/*", "**", "##")): + continue + new_s.append(line) + return "\n".join(new_s) + + +def make_zip_archive(zip_name, src, skip_dirs=None): + src_path = Path(src).expanduser().resolve(strict=True) + if skip_dirs is None: + skip_dirs = [] + if not isinstance(skip_dirs, list): + print("Error: A list needs to be passed for 'skip_dirs'") + return + with zipfile.ZipFile(src_path.parents[0] / Path(zip_name), 'w', zipfile.ZIP_DEFLATED) as zf: + for file in src_path.rglob('*'): + skip = False + _parts = file.relative_to(src_path).parts + for sd in skip_dirs: + if sd in _parts: + skip = True + break + if not skip: + zf.write(file, file.relative_to(src_path.parent)) + + +def doc_file(project_dir, project_file_entry): + """Return the (optional) .rstinc file describing a source file.""" + rst_file = project_dir + if rst_file.name != "doc": # Special case: Dummy .pyproject file in doc dir + rst_file /= "doc" + rst_file /= Path(project_file_entry).name + ".rstinc" + return rst_file if rst_file.is_file() else None + + +def get_code_tabs(files, project_dir, file_format): + content = "\n" + + # Prepare ZIP file, and copy to final destination + zip_name = f"{project_dir.name}.zip" + make_zip_archive(zip_name, project_dir, skip_dirs=["doc"]) + zip_src = f"{project_dir}.zip" + zip_dst = EXAMPLES_DOC / zip_name + shutil.move(zip_src, zip_dst) + + if file_format == Format.RST: + content += f":download:`Download this example <{zip_name}>`\n\n" + else: + content += f"{{download}}`Download this example <{zip_name}>`\n\n" + content += "```{eval-rst}\n" + + for i, project_file in enumerate(files): + if i == 0: + content += ".. tab-set::\n\n" + + pfile = Path(project_file) + if pfile.suffix in IGNORED_SUFFIXES: + continue + + content += f"{ind(1)}.. tab-item:: {project_file}\n\n" + + doc_rstinc_file = doc_file(project_dir, project_file) + if doc_rstinc_file: + indent = ind(2) + for line in doc_rstinc_file.read_text("utf-8").split("\n"): + content += indent + line + "\n" + content += "\n" + + lexer = get_lexer(pfile) + content += add_indent(f"{ind(1)}.. code-block:: {lexer}", 1) + content += "\n" + + _path = project_dir / project_file + _file_content = "" + try: + with open(_path, "r", encoding="utf-8") as _f: + _file_content = remove_licenses(_f.read()) + except UnicodeDecodeError as e: + print(f"example_gallery: error decoding {project_dir}/{_path}:{e}", + file=sys.stderr) + raise + except FileNotFoundError as e: + print(f"example_gallery: error opening {project_dir}/{_path}:{e}", + file=sys.stderr) + raise + + content += add_indent(_file_content, 3) + content += "\n\n" + + if file_format == Format.MD: + content += "```" + + return content + + +def get_header_title(example_dir): + _index = example_dir.parts.index("examples") + rel_path = "/".join(example_dir.parts[_index:]) + _title = rel_path + url = f"{BASE_URL}/{rel_path}" + return ( + "..\n This file was auto-generated by the 'examples_gallery' " + "script.\n Any change will be lost!\n\n" + f"{_title}\n" + f"{'=' * len(_title)}\n\n" + f"(You can also check this code `in the repository <{url}>`_)\n\n" + ) + + +def rel_path(from_path, to_path): + """Determine relative paths for paths that are not subpaths (where + relative_to() fails) via a common root.""" + common = Path(*os.path.commonprefix([from_path.parts, to_path.parts])) + up_dirs = len(from_path.parts) - len(common.parts) + prefix = up_dirs * "../" + rel_to_common = os.fspath(to_path.relative_to(common)) + return f"{prefix}{rel_to_common}" + + +def read_rst_file(project_dir, project_files, doc_rst): + """Read the example .rst file and expand literal includes to project files + by relative paths to the example directory. Note: sphinx does not + handle absolute paths as expected, they need to be relative.""" + content = "" + with open(doc_rst, encoding="utf-8") as doc_f: + content = doc_f.read() + if LITERAL_INCLUDE not in content: + return content + + result = [] + path_to_example = rel_path(EXAMPLES_DOC, project_dir) + for line in content.split("\n"): + if line.startswith(LITERAL_INCLUDE): + file = line[len(LITERAL_INCLUDE) + 1:].strip() + if file in project_files: + line = f"{LITERAL_INCLUDE} {path_to_example}/{file}" + result.append(line) + return "\n".join(result) + + +def get_headline(text, file_format): + """Find the headline in the .rst file.""" + if file_format == Format.RST: + underline = text.find("\n====") + if underline != -1: + start = text.rfind("\n", 0, underline - 1) + return text[start + 1:underline] + elif file_format == Format.MD: + headline = text.find("# ") + if headline != -1: + new_line = text.find("\n", headline + 1) + if new_line != -1: + return text[headline + 2:new_line].strip() + return "" + + +def get_doc_source_file(original_doc_dir, example_name): + """Find the doc source file, return (Path, Format).""" + if original_doc_dir.is_dir(): + for file_format in (Format.RST, Format.MD): + suffix = SUFFIXES[file_format] + result = original_doc_dir / f"{example_name}.{suffix}" + if result.is_file(): + return result, file_format + return None, Format.RST + + +def get_screenshot(image_dir, example_name): + """Find screen shot: We look for an image with the same + example_name first, if not, we select the first.""" + if not image_dir.is_dir(): + return None + images = [i for i in image_dir.glob("*") if i.is_file() and check_img_ext(i)] + example_images = [i for i in images if i.name.startswith(example_name)] + if example_images: + return example_images[0] + if images: + return images[0] + return None + + +def write_resources(src_list, dst): + """Write a list of example resource paths to the dst path.""" + for src in src_list: + resource_written = shutil.copy(src, dst / src.name) + if not opt_quiet: + print("Written resource:", resource_written) + + +@dataclass +class ExampleParameters: + """Parameters obtained from scanning the examples directory.""" + + def __init__(self): + self.file_format = Format.RST + self.src_doc_dir = self.src_doc_file_path = self.src_screenshot = None + self.extra_names = "" + + example_dir: Path + module_name: str + example_name: str + extra_names: str + file_format: Format + target_doc_file: str + src_doc_dir: Path + src_doc_file_path: Path + src_screenshot: Path + + +def detect_pyside_example(example_root, pyproject_file): + """Detemine parameters of a PySide example.""" + p = ExampleParameters() + + p.example_dir = pyproject_file.parent + if p.example_dir.name == "doc": # Dummy pyproject in doc dir (scriptableapplication) + p.example_dir = p.example_dir.parent + + parts = p.example_dir.parts[len(example_root.parts):] + p.module_name = parts[0] + p.example_name = parts[-1] + # handling subdirectories besides the module level and the example + p.extra_names = "" if len(parts) == 2 else "_".join(parts[1:-1]) + + # Check for a 'doc' directory inside the example + src_doc_dir = p.example_dir / "doc" + + if src_doc_dir.is_dir(): + src_doc_file_path, fmt = get_doc_source_file(src_doc_dir, p.example_name) + if src_doc_file_path: + p.src_doc_file_path = src_doc_file_path + p.file_format = fmt + p.src_doc_dir = src_doc_dir + p.src_screenshot = get_screenshot(src_doc_dir, p.example_name) + + target_suffix = SUFFIXES[p.file_format] + doc_file = f"example_{p.module_name}_{p.extra_names}_{p.example_name}.{target_suffix}" + p.target_doc_file = doc_file.replace("__", "_") + return p + + +def detect_qt_example(example_root, pyproject_file): + """Detemine parameters of an example from a Qt repository.""" + p = ExampleParameters() + + p.example_dir = pyproject_file.parent + p.module_name = "Qt Demos" + p.example_name = p.example_dir.name + # Check for a 'doc' directory inside the example (qdoc) + doc_root = p.example_dir / "doc" + if doc_root.is_dir(): + src_doc_file_path, fmt = get_doc_source_file(doc_root / "src", p.example_name) + if src_doc_file_path: + p.src_doc_file_path = src_doc_file_path + p.file_format = fmt + p.src_doc_dir = doc_root + p.src_screenshot = get_screenshot(doc_root / "images", p.example_name) + + target_suffix = SUFFIXES[p.file_format] + p.target_doc_file = f"example_qtdemos_{p.example_name}.{target_suffix}" + return p + + +def write_example(example_root, pyproject_file, pyside_example=True): + """Read the project file and documentation, create the .rst file and + copy the data. Return a tuple of module name and a dict of example data.""" + p = (detect_pyside_example(example_root, pyproject_file) if pyside_example + else detect_qt_example(example_root, pyproject_file)) + + result = ExampleData() + result.example = p.example_name + result.module = p.module_name + result.extra = p.extra_names + result.doc_file = p.target_doc_file + result.file_format = p.file_format + result.abs_path = str(p.example_dir) + result.has_doc = bool(p.src_doc_file_path) + result.img_doc = p.src_screenshot + + files = [] + try: + with pyproject_file.open("r", encoding="utf-8") as pyf: + pyproject = json.load(pyf) + # iterate through the list of files in .pyproject and + # check if they exist, before appending to the list. + for f in pyproject["files"]: + if not Path(f).exists: + print(f"example_gallery: {f} listed in {pyproject_file} does not exist") + raise FileNotFoundError + else: + files.append(f) + except (json.JSONDecodeError, KeyError, FileNotFoundError) as e: + print(f"example_gallery: error reading {pyproject_file}: {e}") + raise + + headline = "" + if files: + doc_file = EXAMPLES_DOC / p.target_doc_file + with open(doc_file, "w", encoding="utf-8") as out_f: + if p.src_doc_file_path: + content_f = read_rst_file(p.example_dir, files, p.src_doc_file_path) + headline = get_headline(content_f, p.file_format) + if not headline: + print(f"example_gallery: No headline found in {doc_file}", + file=sys.stderr) + + # Copy other files in the 'doc' directory, but + # excluding the main '.rst' file and all the + # directories. + resources = [] + if pyside_example: + for _f in p.src_doc_dir.glob("*"): + if _f != p.src_doc_file_path and not _f.is_dir(): + resources.append(_f) + else: # Qt example: only use image. + if p.src_screenshot: + resources.append(p.src_screenshot) + write_resources(resources, EXAMPLES_DOC) + else: + content_f = get_header_title(p.example_dir) + content_f += get_code_tabs(files, pyproject_file.parent, p.file_format) + out_f.write(content_f) + + if not opt_quiet: + print(f"Written: {doc_file}") + else: + if not opt_quiet: + print("Empty '.pyproject' file, skipping") + + result.headline = headline + + return (p.module_name, result) + + +def example_sort_key(example: ExampleData): + name = example.example + return "AAA" + name if "gallery" in name else name + + +def sort_examples(example): + result = {} + for module in example.keys(): + result[module] = sorted(example.get(module), key=example_sort_key) + return result + + +def scan_examples_dir(examples_dir, pyside_example=True): + """Scan a directory of examples.""" + for pyproject_file in examples_dir.glob("**/*.pyproject"): + if pyproject_file.name != "examples.pyproject": + module_name, data = write_example(examples_dir, pyproject_file, + pyside_example) + if module_name not in examples: + examples[module_name] = [] + examples[module_name].append(data) + + +if __name__ == "__main__": + # Only examples with a '.pyproject' file will be listed. + DIR = Path(__file__).parent + EXAMPLES_DOC = Path(f"{DIR}/../../sources/pyside6/doc/examples").resolve() + EXAMPLES_DIR = Path(f"{DIR}/../../examples/").resolve() + BASE_URL = "https://code.qt.io/cgit/pyside/pyside-setup.git/tree" + columns = 5 + gallery = "" + + parser = ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter) + TARGET_HELP = f"Directory into which to generate Doc files (default: {str(EXAMPLES_DOC)})" + parser.add_argument("--target", "-t", action="store", dest="target_dir", help=TARGET_HELP) + parser.add_argument("--qt-src-dir", "-s", action="store", help="Qt source directory") + parser.add_argument("--quiet", "-q", action="store_true", help="Quiet") + options = parser.parse_args() + opt_quiet = options.quiet + if options.target_dir: + EXAMPLES_DOC = Path(options.target_dir).resolve() + + # This main loop will be in charge of: + # * Getting all the .pyproject files, + # * Gather the information of the examples and store them in 'examples' + # * Read the .pyproject file to output the content of each file + # on the final .rst file for that specific example. + examples = {} + + # Create the 'examples' directory if it doesn't exist + # If it does exist, remove it and create a new one to start fresh + if EXAMPLES_DOC.is_dir(): + shutil.rmtree(EXAMPLES_DOC, ignore_errors=True) + if not opt_quiet: + print("WARNING: Deleted old html directory") + EXAMPLES_DOC.mkdir(exist_ok=True) + + scan_examples_dir(EXAMPLES_DIR) + if options.qt_src_dir: + qt_src = Path(options.qt_src_dir) + if not qt_src.is_dir(): + print("Invalid Qt source directory: {}", file=sys.stderr) + sys.exit(-1) + scan_examples_dir(qt_src.parent / "qtdoc", pyside_example=False) + + examples = sort_examples(examples) + + # We generate a 'toctree' at the end of the file, to include the new + # 'example' rst files, so we get no warnings, and also that users looking + # for them will be able to, since they are indexed. + # Notice that :hidden: will not add the list of files by the end of the + # main examples HTML page. + footer_index = dedent( + """\ + .. toctree:: + :hidden: + :maxdepth: 1 + + """ + ) + + # Writing the main example rst file. + index_files = [] + with open(f"{EXAMPLES_DOC}/index.rst", "w") as f: + f.write(BASE_CONTENT) + for module_name in sorted(examples.keys(), key=module_sort_key): + e = examples.get(module_name) + for i in e: + index_files.append(i.doc_file) + title = module_title(module_name) + f.write(f"{title}\n") + f.write(f"{'*' * len(title)}\n") + f.write(get_module_gallery(e)) + f.write("\n\n") + f.write(footer_index) + for i in index_files: + f.write(f" {i}\n") + + if not opt_quiet: + print(f"Written index: {EXAMPLES_DOC}/index.rst") diff --git a/tools/leak_finder.py b/tools/leak_finder.py new file mode 100644 index 000000000..8a21c2337 --- /dev/null +++ b/tools/leak_finder.py @@ -0,0 +1,170 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +""" +leak_finder.py +============== + +This script finds memory leaks in Python. + +Usage: +------ + +Place one or more lines which should be tested for leaks in a loop: + + from leak_finder import LeakFinder + ... + lf = LeakFinder() + for i in range(1000): + leaking_statement() + lf.find_leak() + + +Theory +------ + +How to find a leak? + +We repeatedly perform an action and observe if that has an unexpected +side effect. There are typically two observations: + +* one object is growing its refcount (a pseudo-leak) +* we get many new objects of one type (a true leak) + +A difficulty in trying to get leak info is avoiding side effects +of the measurement. Early attempts with lists of refcounts were +unsuccessful. Using array.array for counting refcounts avoids that. + + +Algorithm +--------- +We record a snapshot of all objects in a list and a parallel array +of refcounts. + +Then we do some computation and do the same snapshot again. + +The structure of a list of all objects is extending at the front for +some reason. That makes the captured structures easy to compare. +We reverse that list and array and have for the objects: + + len(all2) >= len(all1) + + all1[idx] == all2[idx] for idx in range(len(all1)) + +When taking the second snapshot, the objects still have references from +the first snapshot. +For objects with no effect, the following relation is true: + + refs1[idx] == refs2[idx] - 1 for idx in range(len(all1)) + +All other objects are potential pseudo-leaks, because they waste +references but no objects in the first place. + +Then we look at the newly created objects: +These objects are real leaks if their number is growing with the probe +size. For analysis, the number of new objects per type is counted. +""" + +import array +import gc +import sys +import unittest +# this comes from Python, too +from test import support + +try: + sys.getobjects + have_debug = True +except AttributeError: + have_debug = False + + +class LeakFinder(object): + def __init__(self): + self.all, self.refs = self._make_snapshot() + + @staticmethod + def _make_snapshot(): + gc.collect() + # get all objects + all = sys.getobjects(0) + # get an array with the refcounts + g = sys.getrefcount + refs = array.array("l", (g(obj) for obj in all)) + # the lists have the same endind. Make comparison easier. + all.reverse() + refs.reverse() + return all, refs + + @staticmethod + def _short_repr(x, limit=76): + s = repr(x) + if len(s) > limit: + s = s[:limit] + "..." + return s + + def find_leak(self): + all1 = self.all + refs1 = self.refs + del self.all, self.refs + all2, refs2 = self._make_snapshot() + common = len(all1) + del all1 + + srepr = self._short_repr + # look into existing objects for increased refcounts + first = True + for idx in range(common): + ref = refs2[idx] - refs1[idx] - 1 + if abs(ref) <= 10: + continue + obj = all2[idx] + if first: + print() + first = False + print(f"Fake Leak ref={ref} obj={srepr(obj)}") + + # look at the extra objects by type size + types = {} + for idx in range(common, len(all2)): + obj = all2[idx] + typ = type(obj) + if typ not in types: + types[typ] = [] + types[typ].append(obj) + first = True + for typ in types: + oblis = types[typ] + ref = len(oblis) + if ref <= 10: + continue + try: + oblis.sort() + except TypeError: + pass + if first: + print() + first = False + left, mid, right = oblis[0], oblis[ref // 2], oblis[-1] + print(f"True Leak ref={ref} typ={typ} left={left} mid={mid} right={right}") + + +class TestDemo(unittest.TestCase): + + @unittest.skipUnless(have_debug, 'You need a debug build with "--with-trace-refs"') + def test_demo(self): + # create a pseudo leak and a true leak + fake_leak_obj = [] + true_leak_obj = [] + lf = LeakFinder() + refs_before = sys.gettotalrefcount() + for idx in range(100): + fake_leak_obj.append("same string") + true_leak_obj.append(idx + 1000) # avoiding cached low numbers + refs_after = sys.gettotalrefcount() + lf.find_leak() + self.assertNotAlmostEqual(refs_after - refs_before, 0, delta=10) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/license_check.py b/tools/license_check.py new file mode 100644 index 000000000..4b12a05fd --- /dev/null +++ b/tools/license_check.py @@ -0,0 +1,33 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import os +import subprocess +import sys +from pathlib import Path + +"""Tool to run a license check + +Requires the qtqa repo to be checked out as sibling. +""" + + +REPO_DIR = Path(__file__).resolve().parents[1] + + +if __name__ == '__main__': + license_check = (REPO_DIR.parent / 'qtqa' / 'tests' / 'prebuild' + / 'license' / 'tst_licenses.pl') + print('Checking ', license_check) + if not license_check.is_file(): + print('Not found, please clone the qtqa repo') + sys.exit(1) + + os.environ['QT_MODULE_TO_TEST'] = str(REPO_DIR) + cmd = [str(license_check), '-m', 'pyside-setup'] + cmds = ' '.join(cmd) + print('Running: ', cmds) + ex = subprocess.call(cmd) + if ex != 0: + print('FAIL! ', cmds) + sys.exit(1) diff --git a/tools/metaobject_dump.py b/tools/metaobject_dump.py new file mode 100644 index 000000000..b6cde13ef --- /dev/null +++ b/tools/metaobject_dump.py @@ -0,0 +1,30 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import sys + +from dump_metaobject import dump_metaobject +# Import all widget classes to enable instantiating them by type name +from PySide6.QtWidgets import * + +DESC = """ +metaobject_dump.py <class_name> + +Dumps the QMetaObject of a class + +Example: metaobject_dump QLabel +""" + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print(DESC) + sys.exit(0) + app = QApplication(sys.argv) + + type_name = sys.argv[1] + type_instance = eval(type_name) + if not type_instance: + print(f'Invalid type {type_name}') + sys.exit(1) + dump_metaobject(type_instance.staticMetaObject) diff --git a/tools/metaobject_dump.pyproject b/tools/metaobject_dump.pyproject new file mode 100644 index 000000000..f6d85b571 --- /dev/null +++ b/tools/metaobject_dump.pyproject @@ -0,0 +1,3 @@ +{ + "files": ["metaobject_dump.py", "dump_metaobject.py"] +} diff --git a/tools/missing_bindings/config.py b/tools/missing_bindings/config.py new file mode 100644 index 000000000..ddaf20685 --- /dev/null +++ b/tools/missing_bindings/config.py @@ -0,0 +1,144 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +modules_to_test = { + # 6.0 + 'QtCore': 'qtcore-module.html', + 'QtGui': 'qtgui-module.html', + 'QtNetwork': 'qtnetwork-module.html', + 'QtQml': 'qtqml-module.html', + 'QtQuick': 'qtquick-module.html', + 'QtQuickWidgets': 'qtquickwidgets-module.html', + # Broken in 6.5.0 + #'QtQuickControls2': 'qtquickcontrols-module.html', + 'QtSql': 'qtsql-module.html', + 'QtWidgets': 'qtwidgets-module.html', + 'QtConcurrent': 'qtconcurrent-module.html', + 'QtDBus': 'qtdbus-module.html', + 'QtHelp': 'qthelp-module.html', + 'QtOpenGL': 'qtopengl-module.html', + 'QtPrintSupport': 'qtprintsupport-module.html', + 'QtSvg': 'qtsvg-module.html', + 'QtSvgWidgets': 'qtsvgwidgets-module.html', + 'QtUiTools': 'qtuitools-module.html', + 'QtXml': 'qtxml-module.html', + 'QtTest': 'qttest-module.html', + 'Qt3DCore': 'qt3dcore-module.html', + 'Qt3DInput': 'qt3dinput-module.html', + 'Qt3DLogic': 'qt3dlogic-module.html', + 'Qt3DRender': 'qt3drender-module.html', + 'Qt3DAnimation': 'qt3danimation-module.html', + 'Qt3DExtras': 'qt3dextras-module.html', + 'QtNetworkAuth': 'qtnetworkauth-module.html', + 'QtStateMachine': 'qtstatemachine-module.html', + # 'QtCoAp' -- TODO + # 'QtMqtt' -- TODO + # 'QtOpcUA' -- TODO + + # 6.1 + 'QtScxml': 'qtscxml-module.html', + 'QtCharts': 'qtcharts-module.html', + 'QtDataVisualization': 'qtdatavisualization-module.html', + + # 6.2 + 'QtBluetooth': 'qtbluetooth-module.html', + 'QtPositioning': 'qtpositioning-module.html', + 'QtMultimedia': 'qtmultimedia-module.html', + 'QtRemoteObjects': 'qtremoteobjects-module.html', + 'QtSensors': 'qtsensors-module.html', + 'QtSerialPort': 'qtserialport-module.html', + 'QtWebChannel': 'qtwebchannel-module.html', + 'QtWebEngineCore': 'qtwebenginecore-module.html', + 'QtWebEngineQuick': 'qtwebenginequick-module.html', + 'QtWebEngineWidgets': 'qtwebenginewidgets-module.html', + 'QtWebSockets': 'qtwebsockets-module.html', + 'QtHttpServer': 'qthttpserver-module.html', + + # 6.3 + #'QtSpeech': 'qtspeech-module.html', + 'QtMultimediaWidgets': 'qtmultimediawidgets-module.html', + 'QtNfc': 'qtnfc-module.html', + 'QtQuick3D': 'qtquick3d-module.html', + + # 6.4 + 'QtPdf': 'qtpdf-module.html', # this include qtpdfwidgets + 'QtSpatialAudio': 'qtspatialaudio-module.html', + + # 6.5 + 'QtSerialBus': 'qtserialbus-module.html', + 'QtTextToSpeech': 'qttexttospeech-module.html', + 'QtLocation': 'qtlocation-module.html', + +} + +types_to_ignore = { + # QtCore + 'QFlag', + 'QFlags', + 'QGlobalStatic', + 'QDebug', + 'QDebugStateSaver', + 'QMetaObject.Connection', + 'QPointer', + 'QAssociativeIterable', + 'QSequentialIterable', + 'QStaticPlugin', + 'QChar', + 'QLatin1Char', + 'QHash', + 'QMultiHash', + 'QLinkedList', + 'QList', + 'QMap', + 'QMultiMap', + 'QMap.key_iterator', + 'QPair', + 'QQueue', + 'QScopedArrayPointer', + 'QScopedPointer', + 'QScopedValueRollback', + 'QMutableSetIterator', + 'QSet', + 'QSet.const_iterator', + 'QSet.iterator', + 'QExplicitlySharedDataPointer', + 'QSharedData', + 'QSharedDataPointer', + 'QEnableSharedFromThis', + 'QSharedPointer', + 'QWeakPointer', + 'QStack', + 'QLatin1String', + 'QString', + 'QStringRef', + 'QStringList', + 'QStringMatcher', + 'QVarLengthArray', + 'QVector', + 'QFutureIterator', + 'QHashIterator', + 'QMutableHashIterator', + 'QLinkedListIterator', + 'QMutableLinkedListIterator', + 'QListIterator', + 'QMutableListIterator', + 'QMapIterator', + 'QMutableMapIterator', + 'QSetIterator', + 'QMutableVectorIterator', + 'QVectorIterator', + # QtGui + 'QIconEnginePlugin', + 'QImageIOPlugin', + 'QGenericPlugin', + 'QGenericPluginFactory', + 'QGenericMatrix', + 'QOpenGLExtraFunctions', + # QtWidgets + 'QItemEditorCreator', + 'QStandardItemEditorCreator', + 'QStylePlugin', + # QtSql + 'QSqlDriverCreator', + 'QSqlDriverPlugin', +} diff --git a/tools/missing_bindings/main.py b/tools/missing_bindings/main.py new file mode 100644 index 000000000..4c223050d --- /dev/null +++ b/tools/missing_bindings/main.py @@ -0,0 +1,349 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +# This script is used to generate a summary of missing types / classes +# which are present in C++ Qt6, but are missing in PySide6. +# +# Required packages: bs4 +# Installed via: pip install bs4 +# +# The script uses beautiful soup 4 to parse out the class names from +# the online Qt documentation. It then tries to import the types from +# PySide6. +# +# Example invocation of script: +# python missing_bindings.py --qt-version 6.3 -w all +# --qt-version - specify which version of qt documentation to load. +# -w - if PyQt6 is an installed package, check if the tested +# class also exists there. + +import argparse +import sys +from textwrap import dedent +from time import gmtime, strftime +from urllib import request +from pathlib import Path + +from bs4 import BeautifulSoup +from config import modules_to_test, types_to_ignore +import pandas as pd +import matplotlib.pyplot as plt + +qt_documentation_website_prefixes = { + "6.5": "https://doc.qt.io/qt-6/", + "dev": "https://doc-snapshots.qt.io/qt6-dev/", +} + + +def qt_version_to_doc_prefix(version): + if version in qt_documentation_website_prefixes: + return qt_documentation_website_prefixes[version] + else: + raise RuntimeError("The specified qt version is not supported") + + +def create_doc_url(module_doc_page_url, version): + return qt_version_to_doc_prefix(version) + module_doc_page_url + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + "module", + default="all", + choices=list(modules_to_test.keys()).append("all"), + nargs="?", + type=str, + help="the Qt module for which to get the missing types", + ) + parser.add_argument( + "--qt-version", + "-v", + default="6.5", + choices=["6.5", "dev"], + type=str, + dest="version", + help="the Qt version to use to check for types", + ) + parser.add_argument( + "--which-missing", + "-w", + default="all", + choices=["all", "in-pyqt", "not-in-pyqt", "in-pyside-not-in-pyqt"], + type=str, + dest="which_missing", + help="Which missing types to show (all, or just those that are not present in PyQt)", + ) + parser.add_argument( + "--plot", + action="store_true", + help="Create module-wise bar plot comparisons for the missing bindings comparisons" + " between Qt, PySide6 and PyQt6", + ) + return parser + + +def wikilog(*pargs, **kw): + print(*pargs) + + computed_str = "".join(str(arg) for arg in pargs) + + style = "text" + if "style" in kw: + style = kw["style"] + + if style == "heading1": + computed_str = f"= {computed_str} =" + elif style == "heading5": + computed_str = f"===== {computed_str} =====" + elif style == "with_newline": + computed_str = f"{computed_str}\n" + elif style == "bold_colon": + computed_str = computed_str.replace(":", ":'''") + computed_str = f"{computed_str}'''\n" + elif style == "error": + computed_str = computed_str.strip("\n") + computed_str = f"''{computed_str}''\n" + elif style == "text_with_link": + computed_str = computed_str + elif style == "code": + computed_str = f" {computed_str}" + elif style == "end": + return + + print(computed_str, file=wiki_file) + + +if __name__ == "__main__": + parser = get_parser() + args = parser.parse_args() + + if hasattr(args, "module") and args.module != "all": + saved_value = modules_to_test[args.module] + modules_to_test.clear() + modules_to_test[args.module] = saved_value + + pyside_package_name = "PySide6" + pyqt_package_name = "PyQt6" + + data = {"module": [], "qt": [], "pyside": [], "pyqt": []} + total_missing_types_count = 0 + total_missing_types_count_compared_to_pyqt = 0 + total_missing_modules_count = 0 + total_missing_pyqt_types_count = 0 + total_missing_pyqt_modules_count = 0 + + wiki_file = open("missing_bindings_for_wiki_qt_io.txt", "w") + wiki_file.truncate() + + wikilog(f"PySide6 bindings for Qt {args.version}", style="heading1") + + wikilog( + f"Using Qt version {args.version} documentation to find public " + "API Qt types and test if the types are present in the PySide6 " + "package." + ) + + wikilog( + dedent( + """\ + Results are usually stored at + https://wiki.qt.io/PySide6_Missing_Bindings + so consider taking the contents of the generated + missing_bindings_for_wiki_qt_io.txt + file and updating the linked wiki page.""" + ), + style="end", + ) + + wikilog( + "Similar report:\n https://gist.github.com/ethanhs/6c626ca4e291f3682589699296377d3a", + style="text_with_link", + ) + + python_executable = Path(sys.executable).name or "" + command_line_arguments = " ".join(sys.argv) + report_date = strftime("%Y-%m-%d %H:%M:%S %Z", gmtime()) + + wikilog( + dedent( + f""" + This report was generated by running the following command: + {python_executable} {command_line_arguments} + on the following date: + {report_date} + """ + ) + ) + + for module_name in modules_to_test.keys(): + wikilog(module_name, style="heading5") + + url = create_doc_url(modules_to_test[module_name], args.version) + wikilog(f"Documentation link: {url}\n", style="text_with_link") + + # Import the tested module + try: + pyside_tested_module = getattr( + __import__(pyside_package_name, fromlist=[module_name]), module_name + ) + except Exception as e: + e_str = str(e).replace('"', "") + wikilog( + f"\nCould not load {pyside_package_name}.{module_name}. " + f"Received error: {e_str}. Skipping.\n", + style="error", + ) + total_missing_modules_count += 1 + continue + + try: + pyqt_module_name = module_name + + pyqt_tested_module = getattr( + __import__(pyqt_package_name, fromlist=[pyqt_module_name]), pyqt_module_name + ) + except Exception as e: + e_str = str(e).replace("'", "") + wikilog( + f"\nCould not load {pyqt_package_name}.{module_name} for comparison. " + f"Received error: {e_str}.\n", + style="error", + ) + total_missing_pyqt_modules_count += 1 + + # Get C++ class list from documentation page. + page = request.urlopen(url) + soup = BeautifulSoup(page, "html.parser") + + # Extract the Qt type names from the documentation classes table + links = soup.body.select(".annotated a") + types_on_html_page = [] + + for link in links: + link_text = link.text.replace("::", ".") + if link_text not in types_to_ignore: + types_on_html_page.append(link_text) + + total_qt_types = len(types_on_html_page) + wikilog(f"Number of types in {module_name}: {total_qt_types}", style="bold_colon") + + missing_pyside_types_count = 0 + missing_pyqt_types_count = 0 + missing_types_compared_to_pyqt = 0 + missing_types = [] + for qt_type in types_on_html_page: + is_present_in_pyqt = False + is_present_in_pyside = False + missing_type = None + + try: + pyqt_qualified_type = f"pyqt_tested_module.{qt_type}" + eval(pyqt_qualified_type) + is_present_in_pyqt = True + except Exception as e: + print(f"{type(e).__name__}: {e}") + missing_pyqt_types_count += 1 + total_missing_pyqt_types_count += 1 + + try: + pyside_qualified_type = f"pyside_tested_module.{qt_type}" + eval(pyside_qualified_type) + is_present_in_pyside = True + except Exception as e: + print("Failed eval-in pyside qualified types") + print(f"{type(e).__name__}: {e}") + missing_type = qt_type + missing_pyside_types_count += 1 + total_missing_types_count += 1 + + if is_present_in_pyqt: + missing_type = f"{missing_type} (is present in PyQt6)" + missing_types_compared_to_pyqt += 1 + total_missing_types_count_compared_to_pyqt += 1 + + # missing in PySide + if not is_present_in_pyside: + if args.which_missing == "all": + missing_types.append(missing_type) + message = f"Missing types in PySide (all) {module_name}:" + # missing in PySide and present in pyqt + elif args.which_missing == "in-pyqt" and is_present_in_pyqt: + missing_types.append(missing_type) + message = f"Missing types in PySide6 (but present in PyQt6) {module_name}:" + # missing in both PyQt and PySide + elif args.which_missing == "not-in-pyqt" and not is_present_in_pyqt: + missing_types.append(missing_type) + message = f"Missing types in PySide6 (also missing in PyQt6) {module_name}:" + elif ( + args.which_missing == "in-pyside-not-in-pyqt" + and not is_present_in_pyqt + ): + missing_types.append(qt_type) + message = f"Missing types in PyQt6 (but present in PySide6) {module_name}:" + + if len(missing_types) > 0: + wikilog(message, style="with_newline") + missing_types.sort() + for missing_type in missing_types: + wikilog(missing_type, style="code") + wikilog("") + + if args.which_missing != "in-pyside-not-in-pyqt": + missing_types_count = missing_pyside_types_count + else: + missing_types_count = missing_pyqt_types_count + + if args.plot: + total_pyside_types = total_qt_types - missing_pyside_types_count + total_pyqt_types = total_qt_types - missing_pyqt_types_count + data["module"].append(module_name) + data["qt"].append(total_qt_types) + data["pyside"].append(total_pyside_types) + data["pyqt"].append(total_pyqt_types) + + wikilog(f"Number of missing types: {missing_types_count}", style="bold_colon") + if len(missing_types) > 0 and args.which_missing != "in-pyside-not-in-pyqt": + wikilog( + "Number of missing types that are present in PyQt6: " + f"{missing_types_compared_to_pyqt}", + style="bold_colon", + ) + wikilog(f"End of missing types for {module_name}\n", style="end") + else: + wikilog("", style="end") + + if args.plot: + df = pd.DataFrame(data=data, columns=["module", "qt", "pyside", "pyqt"]) + df.set_index("module", inplace=True) + df.plot(kind="bar", title="Qt API Coverage plot") + plt.legend() + plt.xticks(rotation=45) + plt.ylabel("Types Count") + figure = plt.gcf() + figure.set_size_inches(32, 18) # set to full_screen + plt.savefig("missing_bindings_comparison_plot.png", bbox_inches='tight') + print(f"Plot saved in {Path.cwd() / 'missing_bindings_comparison_plot.png'}\n") + + wikilog("Summary", style="heading5") + + if args.which_missing != "in-pyside-not-in-pyqt": + wikilog(f"Total number of missing types: {total_missing_types_count}", style="bold_colon") + wikilog( + "Total number of missing types that are present in PyQt6: " + f"{total_missing_types_count_compared_to_pyqt}", + style="bold_colon", + ) + wikilog( + f"Total number of missing modules: {total_missing_modules_count}", style="bold_colon" + ) + else: + wikilog( + f"Total number of missing types in PyQt6: {total_missing_pyqt_types_count}", + style="bold_colon", + ) + wikilog( + f"Total number of missing modules in PyQt6: {total_missing_pyqt_modules_count}", + style="bold_colon", + ) + wiki_file.close() diff --git a/tools/missing_bindings/requirements.txt b/tools/missing_bindings/requirements.txt new file mode 100644 index 000000000..08aa0a024 --- /dev/null +++ b/tools/missing_bindings/requirements.txt @@ -0,0 +1,14 @@ +beautifulsoup4 +pandas +matplotlib + +# PySide +PySide6 + +# PyQt +PyQt6 +PyQt6-3D +PyQt6-Charts +PyQt6-DataVisualization +PyQt6-NetworkAuth +PyQt6-WebEngine diff --git a/tools/qtcpp2py.py b/tools/qtcpp2py.py new file mode 100644 index 000000000..e4e381675 --- /dev/null +++ b/tools/qtcpp2py.py @@ -0,0 +1,63 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import logging +import os +import sys +from argparse import ArgumentParser, RawTextHelpFormatter +from pathlib import Path + +sys.path.append(os.fspath(Path(__file__).parent / "snippets_translate")) + +from converter import snippet_translate + +DESCRIPTION = "Tool to convert C++ to Python based on snippets_translate" + + +def create_arg_parser(desc): + parser = ArgumentParser(description=desc, + formatter_class=RawTextHelpFormatter) + parser.add_argument("--stdout", "-s", action="store_true", + help="Write to stdout") + parser.add_argument("--force", "-f", action="store_true", + help="Force overwrite of existing files") + parser.add_argument("files", type=str, nargs="+", + help="C++ source file(s)") + return parser + + +if __name__ == "__main__": + arg_parser = create_arg_parser(DESCRIPTION) + args = arg_parser.parse_args() + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(__name__) + + for input_file_str in args.files: + input_file = Path(input_file_str) + if not input_file.is_file(): + logger.error(f"{input_file_str} does not exist or is not a file.") + sys.exit(-1) + + if input_file.suffix != ".cpp" and input_file.suffix != ".h": + logger.error(f"{input_file} does not appear to be a C++ file.") + sys.exit(-1) + + translated_lines = [f"# Converted from {input_file.name}\n"] + for line in input_file.read_text().split("\n"): + translated_lines.append(snippet_translate(line)) + translated = "\n".join(translated_lines) + + if args.stdout: + sys.stdout.write(translated) + else: + target_file = input_file.parent / (input_file.stem + ".py") + if target_file.exists(): + if not target_file.is_file(): + logger.error(f"{target_file} exists and is not a file.") + sys.exit(-1) + if not args.force: + logger.error(f"{target_file} exists. Use -f to overwrite.") + sys.exit(-1) + + target_file.write_text(translated) + logger.info(f"Wrote {target_file}.") diff --git a/tools/regenerate_example_resources.py b/tools/regenerate_example_resources.py new file mode 100644 index 000000000..098c58b1f --- /dev/null +++ b/tools/regenerate_example_resources.py @@ -0,0 +1,60 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +""" +regenerate_example_resources.py +=============================== + +Regenerates the QRC resource files of the PySide examples. +""" + + +import subprocess +import sys +from pathlib import Path + +RCC_COMMAND = "pyside6-rcc" +LRELEASE_COMMAND = "lrelease" + + +def prepare_linguist_example(path): + """Create the .qm files for the Linguist example which are bundled in the QRC file""" + translations_dir = path / "translations" + if not translations_dir.is_dir(): + translations_dir.mkdir(parents=True) + + for ts_file in path.glob("*.ts"): + qm_file = translations_dir / f"{ts_file.stem}.qm" + print("Regenerating ", ts_file, qm_file) + ex = subprocess.call([LRELEASE_COMMAND, ts_file, "-qm", qm_file]) + if ex != 0: + print(f"{LRELEASE_COMMAND} failed for {ts_file}", file=sys.stderr) + sys.exit(ex) + + +def generate_rc_file(qrc_file): + """Regenerate the QRC resource file.""" + dir = qrc_file.parent + if dir.name == "linguist": + prepare_linguist_example(dir) + + target_file = dir / f"{qrc_file.stem}_rc.py" + if not target_file.is_file(): # prefix naming convention + target_file2 = qrc_file.parent / f"rc_{qrc_file.stem}.py" + if target_file2.is_file(): + target_file = target_file2 + if not target_file.is_file(): + print(target_file, " does not exist.", file=sys.stderr) + return + + print("Regenerating ", qrc_file, target_file) + ex = subprocess.call([RCC_COMMAND, qrc_file, "-o", target_file]) + if ex != 0: + print(f"{RCC_COMMAND} failed for {qrc_file}", file=sys.stderr) + sys.exit(ex) + + +if __name__ == '__main__': + examples_path = Path(__file__).resolve().parent.parent / "examples" + for qrc_file in examples_path.glob("**/*.qrc"): + generate_rc_file(qrc_file) diff --git a/tools/regenerate_example_ui.py b/tools/regenerate_example_ui.py new file mode 100644 index 000000000..2e0881c07 --- /dev/null +++ b/tools/regenerate_example_ui.py @@ -0,0 +1,36 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +""" +regenerate_example_ui.py +======================== + +Regenerates the ui files of the PySide examples. +""" + + +import subprocess +import sys +from pathlib import Path + +UIC_COMMAND = "pyside6-uic" + + +def generate_ui_file(ui_file): + """Regenerate the ui file.""" + target_file = ui_file.parent / f"ui_{ui_file.stem}.py" + if not target_file.is_file(): + print(target_file, " does not exist.", file=sys.stderr) + return + + print("Regenerating ", ui_file, target_file) + ex = subprocess.call([UIC_COMMAND, ui_file, "-o", target_file]) + if ex != 0: + print(f"{UIC_COMMAND} failed for {ui_file}", file=sys.stderr) + sys.exit(ex) + + +if __name__ == '__main__': + examples_path = Path(__file__).resolve().parent.parent / "examples" + for ui_file in examples_path.glob("**/*.ui"): + generate_ui_file(ui_file) diff --git a/tools/scanqtclasses.py b/tools/scanqtclasses.py new file mode 100644 index 000000000..0f87d80bd --- /dev/null +++ b/tools/scanqtclasses.py @@ -0,0 +1,122 @@ +# Copyright (C) 2024 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +from pathlib import Path +import os +import re +import subprocess +import sys + +"""Scan the Qt C++ headers per module for classes that should be present + in the matching type system and print the missing classes.""" + + +VALUE_TYPE = re.compile(r'^\s*<value-type name="([^"]+)"') + + +OBJECT_TYPE = re.compile(r'^\s*<object-type name="([^"]+)"') + + +def query_qtpaths(keyword): + """Query qtpaths for a keyword.""" + query_cmd = ["qtpaths", "-query", keyword] + output = subprocess.check_output(query_cmd, stderr=subprocess.STDOUT, + universal_newlines=True) + return output.strip() + + +def is_class_exluded(name): + """Check for excluded classes that do not make sense in a typesystem.""" + if len(name) < 2: + return True + if "Iterator" in name or "iterator" in name: + return True + if name.startswith("If") or name.startswith("Is") or name.startswith("When"): + return True + if name[:1].islower(): + return True + if name.startswith("QOpenGLFunctions") and name.endswith("Backend"): + return True + return False + + +def class_from_header_line(line): + """Extract a class name from a C++ header line.""" + def _is_macro(token): + return "EXPORT" in token or "API" in token + + def _fix_class_name(name): + pos = name.find('<') # Some template specialization "class Name<TemplateParam>" + if pos > 0: + name = name[:pos] + if name.endswith(':'): + name = name[:-1] + return name + + if line.startswith('//') or line.endswith(';'): # comment/forward decl + return None + line = line.strip() + if not line.startswith("class ") and not line.startswith("struct "): + return None + tokens = line.split() + pos = 1 + while pos < len(tokens) and _is_macro(tokens[pos]): + pos += 1 + return _fix_class_name(tokens[pos]) if pos < len(tokens) else None + + +def classes_from_header(header): + """Extract classes from C++ header file.""" + result = [] + for line in header.read_text("utf-8").splitlines(): + name = class_from_header_line(line) + if name and not is_class_exluded(name): + result.append(name) + return sorted(result) + + +def classes_from_typesystem(typesystem): + """Extract classes from typesystem XML file.""" + result = [] + for line in typesystem.read_text("utf-8").splitlines(): + match = VALUE_TYPE.search(line) or OBJECT_TYPE.search(line) + if match: + result.append(match.group(1)) + return sorted(result) + + +def check_classes(qt_module_inc_dir, pyside_dir): + """Check classes of a module.""" + module_name = qt_module_inc_dir.name + sys.stderr.write(f"Checking {module_name} ") + cpp_classes = [] + typesystem_classes = [] + for header in qt_module_inc_dir.glob("q*.h"): + if not header.name.endswith("_p.h"): + cpp_classes.extend(classes_from_header(header)) + for typesystem in pyside_dir.glob("*.xml"): + typesystem_classes.extend(classes_from_typesystem(typesystem)) + + cpp_count = len(cpp_classes) + typesystem_count = len(typesystem_classes) + sys.stderr.write(f"found {cpp_count} C++ / {typesystem_count} typesystem classes") + if cpp_count <= typesystem_count: + sys.stderr.write(" ok\n") + else: + sys.stderr.write(f", {cpp_count-typesystem_count} missing\n") + for cpp_class in cpp_classes: + if cpp_class not in typesystem_classes: + wrapper_name = cpp_class.lower() + "_wrapper.cpp" + print(f"{module_name}:{cpp_class}:{wrapper_name}") + + +if __name__ == '__main__': + qt_version = query_qtpaths("QT_VERSION") + qt_inc_dir = Path(query_qtpaths("QT_INSTALL_HEADERS")) + print(f"Qt {qt_version} at {os.fspath(qt_inc_dir.parent)}", file=sys.stderr) + + dir = Path(__file__).parents[1].resolve() + for module_dir in (dir / "sources" / "pyside6" / "PySide6").glob("Qt*"): + qt_module_inc_dir = qt_inc_dir / module_dir.name + if qt_module_inc_dir.is_dir(): + check_classes(qt_module_inc_dir, module_dir) diff --git a/tools/snippets_translate/README.md b/tools/snippets_translate/README.md new file mode 100644 index 000000000..8d9ab86f8 --- /dev/null +++ b/tools/snippets_translate/README.md @@ -0,0 +1,183 @@ +# Snippets Translate + +To install dependencies on an activated virtual environment run +`pip install -r requirements.txt`. + +To run the tests, execute `python -m pytest`. It's important not to +run `pytest` alone to include the PYTHONPATH so the imports work. + +Here's an explanation for each file: + +* `main.py`, main file that handle the arguments, the general process + of copying/writing files into the pyside-setup/ repository. +* `converter.py`, main function that translate each line depending + on the decision-making process that use different handlers. +* `handlers.py`, functions that handle the different translation cases. +* `parse_utils.py`, some useful function that help the translation process. +* `tests/test_converter.py`, tests cases for the converter function. + +## Usage + +``` +% python main.py -h +usage: sync_snippets [-h] --qt QT_DIR --target PYSIDE_DIR [-f DIRECTORY] [-w] [-v] [-d] [-s SINGLE_SNIPPET] [--filter FILTER_SNIPPET] + +optional arguments: + -h, --help show this help message and exit + --qt QT_DIR Path to the Qt directory (QT_SRC_DIR) + --target TARGET_DIR Directory into which to generate the snippets + -w, --write Actually copy over the files to the pyside-setup directory + -v, --verbose Generate more output + -d, --debug Generate even more output + -s SINGLE_SNIPPET, --single SINGLE_SNIPPET + Path to a single file to be translated + -f, --directory DIRECTORY Path to a directory containing the snippets to be translated + --filter FILTER_SNIPPET + String to filter the snippets to be translated +``` + +For example: + +``` +python main.py --qt /home/cmaureir/dev/qt6/ --target /home/cmaureir/dev/pyside-setup -w +``` + +which will create all the snippet files in the pyside repository. The `-w` +option is in charge of actually writing the files. + + +## Pending cases + +As described at the end of the `converter.py` and `tests/test_converter.py` +files there are a couple of corner cases that are not covered like: + +* handler `std::` types and functions +* handler for `operator...` +* handler for `tr("... %1").arg(a)` +* support for lambda expressions +* there are also strange cases that cannot be properly handle with + a line-by-line approach, for example, `for ( ; it != end; ++it) {` +* interpretation of `typedef ...` (including function pointers) +* interpretation of `extern "C" ...` + +Additionally, +one could add more test cases for each handler, because at the moment +only the general converter function (which uses handlers) is being +tested as a whole. + +## Patterns for directories + +### Snippets + +Everything that has .../snippets/*, for example: + +``` + qtbase/src/corelib/doc/snippets/ + ./qtdoc/doc/src/snippets/ + +``` + +goes to: + +``` + pyside-setup/sources/pyside6/doc/codesnippets/doc/src/snippets/* +``` + +### Examples + +Everything that has .../examples/*, for example: + +``` + ./qtbase/examples/widgets/dialogs/licensewizard + ./qtbase/examples/widgets/itemviews/pixelator +``` + +goes to + +``` + pyside-setup/sources/pyside6/doc/codesnippets/examples/ + dialogs/licensewizard + itemviews/pixelator + +``` + +## Patterns for files + +Files to skip: + +``` + *.pro + *.pri + *.cmake + *.qdoc + CMakeLists.txt +``` + +which means we will be copying: + +``` + *.png + *.cpp + *.h + *.ui + *.qrc + *.xml + *.qml + *.svg + *.js + *.ts + *.xq + *.txt + etc +``` +## Files examples + +``` +[repo] qt5 + + ./qtbase/src/corelib/doc/snippets/code/src_corelib_thread_qmutexpool.cpp + ./qtbase/src/widgets/doc/snippets/code/src_gui_styles_qstyle.cpp + ./qtbase/src/network/doc/snippets/code/src_network_kernel_qhostinfo.cpp + ./qtbase/examples/sql/relationaltablemodel/relationaltablemodel.cpp + ./qtbase/src/printsupport/doc/snippets/code/src_gui_dialogs_qabstractprintdialog.cpp + ./qtdoc/doc/src/snippets/qlistview-using + ./qtbase/src/widgets/doc/snippets/layouts/layouts.cpp +``` + +``` +[repo] pyside-setup + + ./sources/pyside6/doc/codesnippets/doc/src/snippets/code/src_corelib_thread_qmutexpool.cpp + ./sources/pyside6/doc/codesnippets/doc/src/snippets/code/src_gui_styles_qstyle.cpp + ./sources/pyside6/doc/codesnippets/doc/src/snippets/code/src_network_kernel_qhostinfo.cpp + ./sources/pyside6/doc/codesnippets/examples/relationaltablemodel/relationaltablemodel.cpp + ./sources/pyside6/doc/codesnippets/doc/src/snippets/code/src_gui_dialogs_qabstractprintdialog.cpp + ./sources/pyside6/doc/codesnippets/doc/src/snippets/qlistview-using + ./sources/pyside6/doc/codesnippets/doc/src/snippets/layouts +``` + +## The `module_classes` file + +This file is being used to identify +if the `#include` from C++ have a counterpart from Python. + +The file was generated with: + +``` +from pprint import pprint +from PySide2 import * + +_out = {} +modules = {i for i in dir() if i.startswith("Q")} +for m in modules: + exec(f"import PySide2.{m}") + exec(f"m_classes = [i for i in dir(PySide2.{m}) if i.startswith('Q')]") + if len(m_classes) == 1: + try: + exec(f"from PySide2.{m} import {m}") + exec(f"m_classes = [i for i in dir({m}) if i.startswith('Q')]") + except ImportError: + pass + _out[m] = m_classes +pprint(_out) +``` diff --git a/tools/snippets_translate/converter.py b/tools/snippets_translate/converter.py new file mode 100644 index 000000000..d45bf277f --- /dev/null +++ b/tools/snippets_translate/converter.py @@ -0,0 +1,379 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import re + +from handlers import (handle_array_declarations, handle_casts, handle_class, + handle_conditions, handle_constructor_default_values, + handle_constructors, handle_cout_endl, handle_emit, + handle_for, handle_foreach, handle_functions, + handle_inc_dec, handle_include, handle_keywords, + handle_methods_return_type, handle_negate, + handle_type_var_declaration, handle_useless_qt_classes, + handle_new, + handle_void_functions, handle_qt_connects) +from parse_utils import dstrip, get_indent, remove_ref + + +VOID_METHOD_PATTERN = re.compile(r"^ *void *[\w\_]+(::)?[\w\d\_]+\(") +QT_QUALIFIER_PATTERN = re.compile(r"Q[\w]+::") +TERNARY_OPERATOR_PATTERN = re.compile(r"^.* \? .+ : .+$") +COUT_PATTERN = re.compile("^ *(std::)?cout") +FOR_PATTERN = re.compile(r"^ *for *\(") +FOREACH_PATTERN = re.compile(r"^ *foreach *\(") +ELSE_PATTERN = re.compile(r"^ *}? *else *{?") +ELSE_REPLACEMENT_PATTERN = re.compile(r"}? *else *{?") +CLASS_PATTERN = re.compile(r"^ *class ") +STRUCT_PATTERN = re.compile(r"^ *struct ") +DELETE_PATTERN = re.compile(r"^ *delete ") +VAR1_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*\&]+(\(.*?\))? ?(?!.*=|:).*$") +VAR2_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w]+::[\w\*\&]+\(.*\)$") +VAR3_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*]+ *= *[\w\.\"\']*(\(.*?\))?") +VAR4_PATTERN = re.compile(r"\w+ = [A-Z]{1}\w+") +CONSTRUCTOR_PATTERN = re.compile(r"^ *\w+::\w+\(.*?\)") +ARRAY_VAR_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*]+\[?\]? * =? *\{") +RETURN_TYPE_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w]+::[\w\*\&]+\(.*\)$") +FUNCTION_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*\&]+\(.*\)$") +ITERATOR_PATTERN = re.compile(r"(std::)?[\w]+<[\w]+>::(const_)?iterator") +SCOPE_PATTERN = re.compile(r"[\w]+::") +SWITCH_PATTERN = re.compile(r"^\s*switch\s*\(([a-zA-Z0-9_\.]+)\)\s*{.*$") +CASE_PATTERN = re.compile(r"^(\s*)case\s+([a-zA-Z0-9_:\.]+):.*$") +DEFAULT_PATTERN = re.compile(r"^(\s*)default:.*$") + + +QUALIFIERS = {"public:", "protected:", "private:", "public slots:", + "protected slots:", "private slots:", "signals:"} + + +FUNCTION_QUALIFIERS = ["virtual ", " override", "inline ", " noexcept"] + + +switch_var = None +switch_branch = 0 + + +def snippet_translate(x): + global switch_var, switch_branch + + ## Cases which are not C++ + ## TODO: Maybe expand this with lines that doesn't need to be translated + if x.strip().startswith("content-type: text/html"): + return x + + ## General Rules + + # Remove ';' at the end of the lines + has_semicolon = x.endswith(";") + if has_semicolon: + x = x[:-1] + + # Remove lines with only '{' or '}' + xs = x.strip() + if xs == "{" or xs == "}": + return "" + + # Skip lines with the snippet related identifier '//!' + if xs.startswith("//!"): + return x + + # handle lines with only comments using '//' + if xs.startswith("//"): + x = x.replace("//", "#", 1) + return x + + qt_connects = handle_qt_connects(x) + if qt_connects: + return qt_connects + + # Handle "->" + if "->" in x: + x = x.replace("->", ".") + + # handle '&&' and '||' + if "&&" in x: + x = x.replace("&&", "and") + if "||" in x: + x = x.replace("||", "or") + + # Handle lines that have comments after the ';' + if ";" in x and "//" in x: + if x.index(";") < x.index("//"): + left, right = x.split("//", 1) + left = left.replace(";", "", 1) + x = f"{left}#{right}" + + # Handle 'new ' + # This contains an extra whitespace because of some variables + # that include the string 'new' + if "new " in x: + x = handle_new(x) + + # Handle 'const' + # Some variables/functions have the word 'const' so we explicitly + # consider the cases with a whitespace before and after. + if " const" in x: + x = x.replace(" const", "") + if "const " in x: + x = x.replace("const ", "") + + # Handle 'static' + if "static " in x: + x = x.replace("static ", "") + + # Handle 'inline' + if "inline " in x: + x = x.replace("inline ", "") + + # Handle 'double' + if "double " in x: + x = x.replace("double ", "float ") + + # Handle increment/decrement operators + if "++" in x: + x = handle_inc_dec(x, "++") + if "--" in x: + x = handle_inc_dec(x, "--") + + # handle negate '!' + if "!" in x: + x = handle_negate(x) + + # Handle "this", "true", "false" but before "#" symbols + if "this" in x: + x = handle_keywords(x, "this", "self") + if "true" in x: + x = handle_keywords(x, "true", "True") + if "false" in x: + x = handle_keywords(x, "false", "False") + if "throw" in x: + x = handle_keywords(x, "throw", "raise") + + switch_match = SWITCH_PATTERN.match(x) + if switch_match: + switch_var = switch_match.group(1) + switch_branch = 0 + return "" + + switch_match = CASE_PATTERN.match(x) + if switch_match: + indent = switch_match.group(1) + value = switch_match.group(2).replace("::", ".") + cond = "if" if switch_branch == 0 else "elif" + switch_branch += 1 + return f"{indent}{cond} {switch_var} == {value}:" + + switch_match = DEFAULT_PATTERN.match(x) + if switch_match: + indent = switch_match.group(1) + return f"{indent}else:" + + # handle 'void Class::method(...)' and 'void method(...)' + if VOID_METHOD_PATTERN.search(x): + x = handle_void_functions(x) + + # 'Q*::' -> 'Q*.' + if QT_QUALIFIER_PATTERN.search(x): + x = x.replace("::", ".") + + # handle 'nullptr' + if "nullptr" in x: + x = x.replace("nullptr", "None") + + ## Special Cases Rules + xs = x.strip() + # Special case for 'main' + if xs.startswith("int main("): + return f'{get_indent(x)}if __name__ == "__main__":' + + if xs.startswith("QApplication app(argc, argv)"): + return f"{get_indent(x)}app = QApplication([])" + + # Special case for 'return app.exec()' + if xs.startswith("return app.exec"): + return x.replace("return app.exec()", "sys.exit(app.exec())") + + # Handle includes -> import + if xs.startswith("#include"): + x = handle_include(x) + return dstrip(x) + + if xs.startswith("emit "): + x = handle_emit(x) + return dstrip(x) + + # *_cast + if "_cast<" in x: + x = handle_casts(x) + xs = x.strip() + + # Handle Qt classes that needs to be removed + x = handle_useless_qt_classes(x) + + # Handling ternary operator + if TERNARY_OPERATOR_PATTERN.search(xs): + x = x.replace(" ? ", " if ") + x = x.replace(" : ", " else ") + xs = x.strip() + + # Handle 'while', 'if', and 'else if' + # line might end in ')' or ") {" + if xs.startswith(("while", "if", "else if", "} else if")): + x = handle_conditions(x) + return dstrip(x) + elif ELSE_PATTERN.search(x): + x = ELSE_REPLACEMENT_PATTERN.sub("else:", x) + return dstrip(x) + + # 'cout' and 'endl' + if COUT_PATTERN.search(x) or ("endl" in x) or xs.startswith("qDebug()"): + x = handle_cout_endl(x) + return dstrip(x) + + # 'for' loops + if FOR_PATTERN.search(xs): + return dstrip(handle_for(x)) + + # 'foreach' loops + if FOREACH_PATTERN.search(xs): + return dstrip(handle_foreach(x)) + + # 'class' and 'structs' + if CLASS_PATTERN.search(x) or STRUCT_PATTERN.search(x): + if "struct " in x: + x = x.replace("struct ", "class ") + return handle_class(x) + + # 'delete' + if DELETE_PATTERN.search(x): + return x.replace("delete", "del") + + # 'public:', etc + if xs in QUALIFIERS: + return f"# {x}".replace(":", "") + + # For expressions like: `Type var` + # which does not contain a `= something` on the right side + # should match + # Some thing + # QSome<var> thing + # QSome thing(...) + # should not match + # QSome thing = a + # QSome thing = a(...) + # def something(a, b, c) + # At the end we skip methods with the form: + # QStringView Message::body() + # to threat them as methods. + if (has_semicolon and VAR1_PATTERN.search(xs) + and not ([f for f in FUNCTION_QUALIFIERS if f in x]) + and xs.split()[0] not in ("def", "return", "and", "or") + and not VAR2_PATTERN.search(xs) + and ("{" not in x and "}" not in x)): + + # FIXME: this 'if' is a hack for a function declaration with this form: + # QString myDecoderFunc(QByteArray &localFileName) + # No idea how to check for different for variables like + # QString notAFunction(Something something) + # Maybe checking the structure of the arguments? + if "Func" not in x: + return dstrip(handle_type_var_declaration(x)) + + # For expressions like: `Type var = value`, + # considering complex right-side expressions. + # QSome thing = b + # QSome thing = b(...) + # float v = 0.1 + # QSome *thing = ... + if (VAR3_PATTERN.search(xs) + and ("{" not in x and "}" not in x)): + left, right = x.split("=", 1) + var_name = " ".join(left.strip().split()[1:]) + x = f"{get_indent(x)}{remove_ref(var_name)} = {right.strip()}" + # Special case: When having this: + # QVBoxLayout *layout = new QVBoxLayout; + # we end up like this: + # layout = QVBoxLayout + # so we need to add '()' at the end if it's just a word + # with only alpha numeric content + if VAR4_PATTERN.search(xs) and not xs.endswith(")"): + v = x.rstrip() + if (not v.endswith(" True") and not v.endswith(" False") + and not v.endswith(" None")): + x = f"{v}()" + return dstrip(x) + + # For constructors, that we now the shape is: + # ClassName::ClassName(...) + if CONSTRUCTOR_PATTERN.search(xs): + x = handle_constructors(x) + return dstrip(x) + + # For base object constructor: + # : QWidget(parent) + if ( + xs.startswith(": ") + and ("<<" not in x) + and ("::" not in x) + and not xs.endswith(";") + ): + + return handle_constructor_default_values(x) + + # Arrays declarations with the form: + # type var_name[] = {... + # type var_name {... + # if re.search(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*]+\[\] * = *\{", x.strip()): + if ARRAY_VAR_PATTERN.search(xs): + x = handle_array_declarations(x) + xs = x.strip() + + # Methods with return type + # int Class::method(...) + # QStringView Message::body() + if RETURN_TYPE_PATTERN.search(xs): + # We just need to capture the 'method name' and 'arguments' + x = handle_methods_return_type(x) + xs = x.strip() + + # Handling functions + # By this section of the function, we cover all the other cases + # So we can safely assume it's not a variable declaration + if FUNCTION_PATTERN.search(xs): + x = handle_functions(x) + xs = x.strip() + + # if it is a C++ iterator declaration, then ignore it due to dynamic typing in Python + # eg: std::vector<int> it; + # the case of iterator being used inside a for loop is already handed in handle_for(..) + # TODO: handle iterator initialization statement like it = container.begin(); + if ITERATOR_PATTERN.search(x): + x = "" + return x + + # By now all the typical special considerations of scope resolution operator should be handled + # 'Namespace*::' -> 'Namespace*.' + # TODO: In the case where a C++ class function is defined outside the class, this would be wrong + # but we do not have such a code snippet yet + if SCOPE_PATTERN.search(x): + x = x.replace("::", ".") + + # General return for no special cases + return dstrip(x) + + # TODO: + # * Lambda expressions + + # * operator overload + # void operator()(int newState) { state = newState; } + # const QDBusArgument &operator>>(const QDBusArgument &argument, MyDictionary &myDict) + # inline bool operator==(const Employee &e1, const Employee &e2) + # void *operator new[](size_t size) + + # * extern "C" ... + # extern "C" MY_EXPORT int avg(int a, int b) + + # * typedef ... + # typedef int (*AvgFunction)(int, int); + + # * function pointers + # typedef void (*MyPrototype)(); diff --git a/tools/snippets_translate/handlers.py b/tools/snippets_translate/handlers.py new file mode 100644 index 000000000..34e969a62 --- /dev/null +++ b/tools/snippets_translate/handlers.py @@ -0,0 +1,596 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import re +import sys + +from parse_utils import (dstrip, get_indent, get_qt_module_class, + parse_arguments, remove_ref, replace_main_commas) + +IF_PATTERN = re.compile(r'^\s*if\s*\(') +PARENTHESES_NONEMPTY_CONTENT_PATTERN = re.compile(r"\((.+)\)") +LOCAL_INCLUDE_PATTERN = re.compile(r'"(.*)"') +GLOBAL_INCLUDE_PATTERN = re.compile(r"<(.*)>") +IF_CONDITION_PATTERN = PARENTHESES_NONEMPTY_CONTENT_PATTERN +ELSE_IF_PATTERN = re.compile(r'^\s*}?\s*else if\s*\(') +WHILE_PATTERN = re.compile(r'^\s*while\s*\(') +CAST_PATTERN = re.compile(r"[a-z]+_cast<(.*?)>\((.*?)\)") # Non greedy match of <> +ITERATOR_LOOP_PATTERN = re.compile(r"= *(.*)egin\(") +REMOVE_TEMPLATE_PARAMETER_PATTERN = re.compile("<.*>") +PARENTHESES_CONTENT_PATTERN = re.compile(r"\((.*)\)") +CONSTRUCTOR_BODY_PATTERN = re.compile(".*{ *}.*") +CONSTRUCTOR_BODY_REPLACEMENT_PATTERN = re.compile("{ *}") +CONSTRUCTOR_BASE_PATTERN = re.compile("^ *: *") +NEGATE_PATTERN = re.compile(r"!(.)") +CLASS_TEMPLATE_PATTERN = re.compile(r".*<.*>") +EMPTY_CLASS_PATTERN = re.compile(r".*{.*}") +EMPTY_CLASS_REPLACEMENT_PATTERN = re.compile(r"{.*}") +FUNCTION_BODY_PATTERN = re.compile(r"\{(.*)\}") +ARRAY_DECLARATION_PATTERN = re.compile(r"^[a-zA-Z0-9\<\>]+ ([\w\*]+) *\[?\]?") +RETURN_TYPE_PATTERN = re.compile(r"^ *[a-zA-Z0-9]+ [\w]+::([\w\*\&]+\(.*\)$)") +CAPTURE_PATTERN = re.compile(r"^ *([a-zA-Z0-9]+) ([\w\*\&]+\(.*\)$)") +USELESS_QT_CLASSES_PATTERNS = [ + re.compile(r'QLatin1StringView\(("[^"]*")\)'), + re.compile(r'QLatin1String\(("[^"]*")\)'), + re.compile(r'QString\.fromLatin1\(("[^"]*")\)'), + re.compile(r"QLatin1Char\(('[^']*')\)"), + re.compile(r'QStringLiteral\(("[^"]*")\)'), + re.compile(r'QString\.fromUtf8\(("[^"]*")\)'), + re.compile(r'u("[^"]*")_s') +] +COMMENT1_PATTERN = re.compile(r" *# *[\w\ ]+$") +COMMENT2_PATTERN = re.compile(r" *# *(.*)$") +COUT_ENDL_PATTERN = re.compile(r"cout *<<(.*)<< *.*endl") +COUT1_PATTERN = re.compile(r" *<< *") +COUT2_PATTERN = re.compile(r".*cout *<<") +COUT_ENDL2_PATTERN = re.compile(r"<< +endl") +NEW_PATTERN = re.compile(r"new +([a-zA-Z][a-zA-Z0-9_]*)") + + +def handle_condition(x, name): + # Make sure it's not a multi line condition + x = x.replace("}", "") + if x.count("(") == x.count(")"): + comment = "" + # This handles the lines that have no ';' at the end but + # have a comment after the end of the line, like: + # while (true) // something + # { ... } + if "//" in x: + comment_content = x.split("//", 1) + comment = f" #{comment_content[-1]}" + x = x.replace(f"//{comment_content[-1]}", "") + + match = IF_CONDITION_PATTERN.search(x) + if match: + condition = match.group(1) + return f"{get_indent(x)}{name} {condition.strip()}:{comment}" + else: + print(f'snippets_translate: Warning "{x}" does not match condition pattern', + file=sys.stderr) + return x + + +def handle_keywords(x, word, pyword): + if word in x: + if "#" in x: + if x.index(word) < x.index("#"): + x = x.replace(word, pyword) + else: + x = x.replace(word, pyword) + return x + + +def handle_inc_dec(x, operator): + # Alone on a line + clean_x = x.strip() + if clean_x.startswith(operator) or clean_x.endswith(operator): + x = x.replace(operator, "") + x = f"{x} = {clean_x.replace(operator, '')} {operator[0]} 1" + return x + + +def handle_casts(x): + while True: + match = CAST_PATTERN.search(x) + if not match: + break + type_name = match.group(1).strip() + while type_name.endswith("*") or type_name.endswith("&") or type_name.endswith(" "): + type_name = type_name[:-1] + data_name = match.group(2).strip() + python_cast = f"{type_name}({data_name})" + x = x[0:match.start(0)] + python_cast + x[match.end(0):] + + return x + + +def handle_include(x): + if '"' in x: + header = LOCAL_INCLUDE_PATTERN.search(x) + if header: + header_name = header.group(1).replace(".h", "") + module_name = header_name.replace('/', '.') + x = f"from {module_name} import *" + else: + # We discard completely if there is something else + # besides '"something.h"' + x = "" + elif "<" in x and ">" in x: + name = GLOBAL_INCLUDE_PATTERN.search(x).group(1) + t = get_qt_module_class(name) + # if it's not a Qt module or class, we discard it. + if t is None: + x = "" + else: + # is a module + if t[0]: + x = f"from PySide6 import {t[1]}" + # is a class + else: + x = f"from PySide6.{t[1]} import {name}" + return x + + +def handle_conditions(x): + if WHILE_PATTERN.match(x): + x = handle_condition(x, "while") + elif IF_PATTERN.match(x): + x = handle_condition(x, "if") + elif ELSE_IF_PATTERN.match(x): + x = handle_condition(x, "else if") + x = x.replace("else if", "elif") + x = x.replace("::", ".") + return x + + +def handle_for(x): + content = PARENTHESES_CONTENT_PATTERN.search(x) + + new_x = x + if content: + # parenthesis content + content = content.group(1) + + # for (int i = 1; i < argc; ++i) + if x.count(";") == 2: + + # for (start; middle; end) + start, middle, end = content.split(";") + + # iterators + if "begin(" in x.lower() and "end(" in x.lower(): + name = ITERATOR_LOOP_PATTERN.search(start) + iterable = None + iterator = None + if name: + name = name.group(1) + # remove initial '=', and split the '.' + # because '->' was already transformed, + # and we keep the first word. + iterable = name.replace("=", "", 1).split(".")[0] + + iterator = remove_ref(start.split("=")[0].split()[-1]) + if iterator and iterable: + return f"{get_indent(x)}for {iterator} in {iterable}:" + + if ("++" in end or "--" in end) or ("+=" in end or "-=" in end): + if "," in start: + raw_var, value = start.split(",")[0].split("=") + else: + # Malformed for-loop: + # for (; pixel1 > start; pixel1 -= stride) + # We return the same line + if not start.strip() or "=" not in start: + return f"{get_indent(x)}{dstrip(x)}" + raw_var, value = start.split("=") + raw_var = raw_var.strip() + value = value.strip() + var = raw_var.split()[-1] + + end_value = None + if "+=" in end: + end_value = end.split("+=")[-1] + elif "-=" in end: + end_value = end.split("-=")[-1] + if end_value: + try: + end_value = int(end_value) + except ValueError: + end_value = None + + if "<" in middle: + limit = middle.split("<")[-1] + + if "<=" in middle: + limit = middle.split("<=")[-1] + try: + limit = int(limit) + limit += 1 + except ValueError: + limit = f"{limit} + 1" + + if end_value: + new_x = f"for {var} in range({value}, {limit}, {end_value}):" + else: + new_x = f"for {var} in range({value}, {limit}):" + elif ">" in middle: + limit = middle.split(">")[-1] + + if ">=" in middle: + limit = middle.split(">=")[-1] + try: + limit = int(limit) + limit -= 1 + except ValueError: + limit = f"{limit} - 1" + if end_value: + new_x = f"for {var} in range({limit}, {value}, -{end_value}):" + else: + new_x = f"for {var} in range({limit}, {value}, -1):" + else: + # TODO: No support if '<' or '>' is not used. + pass + + # for (const QByteArray &ext : qAsConst(extensionList)) + elif x.count(":") > 0: + iterator, iterable = content.split(":", 1) + var = iterator.split()[-1].replace("&", "").strip() + iterable = iterable.strip() + if iterable.startswith("qAsConst(") or iterable.startswith("std::as_const("): + iterable = iterable[iterable.find("(") + 1: -1] + new_x = f"for {remove_ref(var)} in {iterable}:" + return f"{get_indent(x)}{dstrip(new_x)}" + + +def handle_foreach(x): + content = PARENTHESES_CONTENT_PATTERN.search(x) + if content: + parenthesis = content.group(1) + iterator, iterable = parenthesis.split(",", 1) + # remove iterator type + it = dstrip(iterator.split()[-1]) + # remove <...> from iterable + value = REMOVE_TEMPLATE_PARAMETER_PATTERN.sub("", iterable) + return f"{get_indent(x)}for {it} in {value}:" + + +def handle_type_var_declaration(x): + # remove content between <...> + if "<" in x and ">" in x: + x = " ".join(REMOVE_TEMPLATE_PARAMETER_PATTERN.sub("", i) for i in x.split()) + content = PARENTHESES_CONTENT_PATTERN.search(x) + if content: + # this means we have something like: + # QSome thing(...) + type_name, var_name = x.split()[:2] + var_name = var_name.split("(")[0] + x = f"{get_indent(x)}{var_name} = {type_name}({content.group(1)})" + else: + # this means we have something like: + # QSome thing + type_name, var_name = x.split()[:2] + x = f"{get_indent(x)}{var_name} = {type_name}()" + return x + + +def handle_constructors(x): + arguments = PARENTHESES_CONTENT_PATTERN.search(x).group(1) + class_method = x.split("(")[0].split("::") + if len(class_method) == 2: + # Equal 'class name' and 'method name' + if len(set(class_method)) == 1: + arguments = ", ".join(remove_ref(i.split()[-1]) for i in arguments.split(",") if i) + if arguments: + return f"{get_indent(x)}def __init__(self, {arguments}):" + else: + return f"{get_indent(x)}def __init__(self):" + return dstrip(x) + + +def handle_constructor_default_values(x): + # if somehow we have a ' { } ' by the end of the line, + # we discard that section completely, since even with a single + # value, we don't need to take care of it, for example: + # ' : a(1) { } -> self.a = 1 + if CONSTRUCTOR_BODY_PATTERN.search(x): + x = CONSTRUCTOR_BODY_REPLACEMENT_PATTERN.sub("", x) + + values = "".join(x.split(":", 1)) + # Check the commas that are not inside round parenthesis + # For example: + # : QWidget(parent), Something(else, and, other), value(1) + # so we can find only the one after '(parent),' and 'other),' + # and replace them by '@' + # : QWidget(parent)@ Something(else, and, other)@ value(1) + # to be able to split the line. + values = replace_main_commas(values) + # if we have more than one expression + if "@" in values: + return_values = "" + for arg in values.split("@"): + arg = CONSTRUCTOR_BASE_PATTERN.sub("", arg).strip() + if arg.startswith("Q"): + class_name = arg.split("(")[0] + content = arg.replace(class_name, "")[1:-1] + return_values += f" super().__init__({content})\n" + elif arg: + var_name = arg.split("(")[0] + content = PARENTHESES_NONEMPTY_CONTENT_PATTERN.search(arg).group(1) + return_values += f" self.{var_name} = {content}\n" + else: + arg = CONSTRUCTOR_BASE_PATTERN.sub("", values).strip() + if arg.startswith("Q"): + class_name = arg.split("(")[0] + content = arg.replace(class_name, "")[1:-1] + return f" super().__init__({content})" + elif arg: + var_name = arg.split("(")[0] + match = PARENTHESES_NONEMPTY_CONTENT_PATTERN.search(arg) + if match: + content = match.group(1) + return f" self.{var_name} = {content}" + else: + print(f'snippets_translate: Warning "{arg}" does not match pattern', + file=sys.stderr) + return "" + return return_values.rstrip() + + +def handle_cout_endl(x): + # if comment at the end + comment = "" + if COMMENT1_PATTERN.search(x): + match = COMMENT2_PATTERN.search(x).group(1) + comment = f' # {match}' + x = x.split("#")[0] + + if "qDebug()" in x: + x = x.replace("qDebug()", "cout") + + if "cout" in x and "endl" in x: + data = COUT_ENDL_PATTERN.search(x) + if data: + data = data.group(1) + data = COUT1_PATTERN.sub(", ", data) + x = f"{get_indent(x)}print({data}){comment}" + elif "cout" in x: + data = COUT2_PATTERN.sub("", x) + data = COUT1_PATTERN.sub(", ", data) + x = f"{get_indent(x)}print({data}){comment}" + elif "endl" in x: + data = COUT_ENDL2_PATTERN.sub("", x) + data = COUT1_PATTERN.sub(", ", data) + x = f"{get_indent(x)}print({data}){comment}" + + x = x.replace("( ", "(").replace(" )", ")").replace(" ,", ",").replace("(, ", "(") + x = x.replace("Qt.endl", "").replace(", )", ")") + return x + + +def handle_negate(x): + # Skip if it's part of a comment: + if "#" in x: + if x.index("#") < x.index("!"): + return x + elif "/*" in x: + if x.index("/*") < x.index("!"): + return x + next_char = NEGATE_PATTERN.search(x).group(1) + if next_char not in ("=", '"'): + x = x.replace("!", "not ") + return x + + +def handle_emit(x): + function_call = x.replace("emit ", "").strip() + match = PARENTHESES_CONTENT_PATTERN.search(function_call) + if not match: + stmt = x.strip() + print(f'snippets_translate: Warning "{stmt}" does not match function call', + file=sys.stderr) + return '' + arguments = match.group(1) + method_name = function_call.split("(")[0].strip() + return f"{get_indent(x)}{method_name}.emit({arguments})" + + +def handle_void_functions(x): + class_method = x.replace("void ", "").split("(")[0] + first_param = "" + if "::" in class_method: + first_param = "self, " + method_name = class_method.split("::")[1] + else: + method_name = class_method.strip() + + # if the arguments are in the same line: + arguments = None + if ")" in x: + parenthesis = PARENTHESES_CONTENT_PATTERN.search(x).group(1) + arguments = dstrip(parse_arguments(parenthesis)) + elif "," in x: + arguments = dstrip(parse_arguments(x.split("(")[-1])) + + # check if includes a '{ ... }' after the method signature + after_signature = x.split(")")[-1] + re_decl = FUNCTION_BODY_PATTERN.search(after_signature) + extra = "" + if re_decl: + extra = re_decl.group(1) + if not extra: + extra = " pass" + + if arguments: + x = f"{get_indent(x)}def {method_name}({first_param}{dstrip(arguments)}):{extra}" + else: + x = f"{get_indent(x)}def {method_name}({first_param.replace(', ', '')}):{extra}" + return x + + +def handle_class(x): + # Check if there is a comment at the end of the line + comment = "" + if "//" in x: + parts = x.split("//") + x = "".join(parts[:-1]) + comment = parts[-1] + + # If the line ends with '{' + if x.rstrip().endswith("{"): + x = x[:-1] + + # Get inheritance + decl_parts = x.split(":") + class_name = decl_parts[0].rstrip() + if len(decl_parts) > 1: + bases = decl_parts[1] + bases_name = ", ".join(i.split()[-1] for i in bases.split(",") if i) + else: + bases_name = "" + + # Check if the class_name is templated, then remove it + if CLASS_TEMPLATE_PATTERN.search(class_name): + class_name = class_name.split("<")[0] + + # Special case: invalid notation for an example: + # class B() {...} -> clas B(): pass + if EMPTY_CLASS_PATTERN.search(class_name): + class_name = EMPTY_CLASS_REPLACEMENT_PATTERN.sub("", class_name).rstrip() + return f"{class_name}(): pass" + + # Special case: check if the line ends in ',' + if x.endswith(","): + x = f"{class_name}({bases_name}," + else: + x = f"{class_name}({bases_name}):" + + if comment: + return f"{x} #{comment}" + else: + return x + + +def handle_array_declarations(x): + content = ARRAY_DECLARATION_PATTERN.search(x.strip()) + if content: + var_name = content.group(1) + rest_line = "".join(x.split("{")[1:]) + x = f"{get_indent(x)}{var_name} = {{{rest_line}" + return x + + +def handle_methods_return_type(x): + capture = RETURN_TYPE_PATTERN.search(x) + if capture: + content = capture.group(1) + method_name = content.split("(")[0] + par_capture = PARENTHESES_NONEMPTY_CONTENT_PATTERN.search(x) + arguments = "(self)" + if par_capture: + arguments = f"(self, {par_capture.group(1)})" + x = f"{get_indent(x)}def {method_name}{arguments}:" + return x + + +def handle_functions(x): + capture = CAPTURE_PATTERN.search(x) + if capture: + return_type = capture.group(1) + if return_type == "return": # "return QModelIndex();" + return x + content = capture.group(2) + function_name = content.split("(")[0] + par_capture = PARENTHESES_NONEMPTY_CONTENT_PATTERN.search(x) + arguments = "" + if par_capture: + for arg in par_capture.group(1).split(","): + arguments += f"{arg.split()[-1]}," + # remove last comma + if arguments.endswith(","): + arguments = arguments[:-1] + x = f"{get_indent(x)}def {function_name}({dstrip(arguments)}):" + return x + + +def handle_useless_qt_classes(x): + for c in USELESS_QT_CLASSES_PATTERNS: + while True: + match = c.search(x) + if match: + x = x[0:match.start()] + match.group(1) + x[match.end():] + else: + break + return x.replace('"_L1', '"').replace("u'", "'") + + +def handle_new(x): + """Parse operator new() and add parentheses were needed: + func(new Foo, new Bar(x))" -> "func(Foo(), Bar(x))""" + result = "" + last_pos = 0 + for match in NEW_PATTERN.finditer(x): + end = match.end(0) + parentheses_needed = end >= len(x) or x[end] != "(" + type_name = match.group(1) + result += x[last_pos:match.start(0)] + type_name + if parentheses_needed: + result += "()" + last_pos = end + result += x[last_pos:] + return result + + +# The code below handles pairs of instance/pointer to member functions (PMF) +# which appear in Qt in connect statements like: +# "connect(fontButton, &QAbstractButton::clicked, this, &Dialog::setFont)". +# In a first pass, these pairs are replaced by: +# "connect(fontButton.clicked, self.setFont)" to be able to handle statements +# spanning lines. A 2nd pass then checks for the presence of a connect +# statement and replaces it by: +# "fontButton.clicked.connect(self.setFont)". +# To be called right after checking for comments. + + +INSTANCE_PMF_RE = re.compile(r"&?(\w+),\s*&\w+::(\w+)") + + +CONNECT_RE = re.compile(r"^(\s*)(QObject::)?connect\(([A-Za-z0-9_\.]+),\s*") + + +def handle_qt_connects(line_in): + if not INSTANCE_PMF_RE.search(line_in): + return None + # 1st pass, "fontButton, &QAbstractButton::clicked" -> "fontButton.clicked" + + is_connect = "connect(" in line_in + line = line_in + # Remove any smart pointer access, etc in connect statements + if is_connect: + line = line.replace(".get()", "").replace(".data()", "").replace("->", ".") + last_pos = 0 + result = "" + for match in INSTANCE_PMF_RE.finditer(line): + instance = match.group(1) + if instance == "this": + instance = "self" + member_fun = match.group(2) + next_pos = match.start() + result += line[last_pos:next_pos] + last_pos = match.end() + result += f"{instance}.{member_fun}" + result += line[last_pos:] + + if not is_connect: + return result + + # 2nd pass, reorder connect. + connect_match = CONNECT_RE.match(result) + if not connect_match: + return result + + space = connect_match.group(1) + signal_ = connect_match.group(3) + connect_stmt = f"{space}{signal_}.connect(" + connect_stmt += result[connect_match.end():] + return connect_stmt diff --git a/tools/snippets_translate/main.py b/tools/snippets_translate/main.py new file mode 100644 index 000000000..01ea06c5e --- /dev/null +++ b/tools/snippets_translate/main.py @@ -0,0 +1,522 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import logging +import os +import re +import sys +from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter +from enum import Enum +from pathlib import Path +from textwrap import dedent +from typing import Dict, List + +from override import python_example_snippet_mapping +from converter import snippet_translate + +HELP = """Converts Qt C++ code snippets to Python snippets. + +Ways to override Snippets: + +1) Complete snippets from local files: + To replace snippet "[1]" of "foo/bar.cpp", create a file + "sources/pyside6/doc/snippets/foo/bar_1.cpp.py" . +2) Snippets extracted from Python examples: + To use snippets from Python examples, add markers ("#! [id]") to it + and an entry to _PYTHON_EXAMPLE_SNIPPET_MAPPING. +""" + + +# Logger configuration +try: + from rich.logging import RichHandler + + logging.basicConfig( + level="NOTSET", format="%(message)s", datefmt="[%X]", handlers=[RichHandler()] + ) + have_rich = True + extra = {"markup": True} + + from rich.console import Console + from rich.table import Table + +except ModuleNotFoundError: + # 'rich' not found, falling back to default logger" + logging.basicConfig(level=logging.INFO) + have_rich = False + extra = {} + +log = logging.getLogger("snippets_translate") + +# Filter and paths configuration +SKIP_END = (".pro", ".pri", ".cmake", ".qdoc", ".yaml", ".frag", ".qsb", ".vert", "CMakeLists.txt") +SKIP_BEGIN = ("changes-", ".") +CPP_SNIPPET_PATTERN = re.compile(r"//! ?\[([^]]+)\]") +PYTHON_SNIPPET_PATTERN = re.compile(r"#! ?\[([^]]+)\]") + +ROOT_PATH = Path(__file__).parents[2] +SOURCE_PATH = ROOT_PATH / "sources" / "pyside6" / "doc" / "snippets" + + +OVERRIDDEN_SNIPPET = "# OVERRIDDEN_SNIPPET" + + +class FileStatus(Enum): + Exists = 0 + New = 1 + + +def get_parser() -> ArgumentParser: + """ + Returns a parser for the command line arguments of the script. + See README.md for more information. + """ + parser = ArgumentParser(prog="snippets_translate", + description=HELP, + formatter_class=RawDescriptionHelpFormatter) + parser.add_argument( + "--qt", + action="store", + dest="qt_dir", + required=True, + help="Path to the Qt directory (QT_SRC_DIR)", + ) + + parser.add_argument( + "--target", + action="store", + dest="target_dir", + required=True, + help="Directory into which to generate the snippets", + ) + + parser.add_argument( + "-w", + "--write", + action="store_true", + dest="write_files", + help="Actually copy over the files to the pyside-setup directory", + ) + + parser.add_argument( + "-v", + "--verbose", + action="store_true", + dest="verbose", + help="Generate more output", + ) + + parser.add_argument( + "-d", + "--debug", + action="store_true", + dest="debug", + help="Generate even more output", + ) + + parser.add_argument( + "-s", + "--single", + action="store", + dest="single_snippet", + help="Path to a single file to be translated", + ) + + parser.add_argument( + "-f", + "--directory", + action="store", + dest="single_directory", + help="Path to a single directory to be translated", + ) + + parser.add_argument( + "--filter", + action="store", + dest="filter_snippet", + help="String to filter the snippets to be translated", + ) + return parser + + +def is_directory(directory): + if not directory.is_dir(): + log.error(f"Path '{directory}' is not a directory") + return False + return True + + +def check_arguments(options): + + # Notify 'write' option + if options.write_files: + if not opt_quiet: + log.warning( + f"Files will be copied from '{options.qt_dir}':\n" f"\tto '{options.target_dir}'" + ) + else: + msg = "This is a listing only, files are not being copied" + if have_rich: + msg = f"[green]{msg}[/green]" + if not opt_quiet: + log.info(msg, extra=extra) + + # Check 'qt_dir' + return is_directory(Path(options.qt_dir)) + + +def is_valid_file(x): + file_name = x.name + # Check END + for ext in SKIP_END: + if file_name.endswith(ext): + return False + + # Check BEGIN + for ext in SKIP_BEGIN: + if file_name.startswith(ext): + return False + + # Contains 'snippets' or 'examples' as subdirectory + if not ("snippets" in x.parts or "examples" in x.parts): + return False + + return True + + +def get_snippet_ids(line: str, pattern: re.Pattern) -> List[str]: + # Extract the snippet ids for a line '//! [1] //! [2]' + result = [] + for m in pattern.finditer(line): + result.append(m.group(1)) + return result + + +def overriden_snippet_lines(lines: List[str], start_id: str) -> List[str]: + """Wrap an overridden snippet with marker and id lines.""" + id_string = f"//! [{start_id}]" + result = [OVERRIDDEN_SNIPPET, id_string] + result.extend(lines) + result.append(id_string) + return result + + +def get_snippet_override(start_id: str, rel_path: str) -> List[str]: + """Check if the snippet is overridden by a local file under + sources/pyside6/doc/snippets.""" + file_start_id = start_id.replace(' ', '_') + override_name = f"{rel_path.stem}_{file_start_id}{rel_path.suffix}.py" + override_path = SOURCE_PATH / rel_path.parent / override_name + if not override_path.is_file(): + return [] + lines = override_path.read_text().splitlines() + return overriden_snippet_lines(lines, start_id) + + +def _get_snippets(lines: List[str], + comment: str, + pattern: re.Pattern) -> Dict[str, List[str]]: + """Helper to extract (potentially overlapping) snippets from a C++ file + indicated by pattern ("//! [1]") and return them as a dict by <id>.""" + snippets: Dict[str, List[str]] = {} + snippet: List[str] + done_snippets : List[str] = [] + + i = 0 + while i < len(lines): + line = lines[i] + i += 1 + + start_ids = get_snippet_ids(line, pattern) + while start_ids: + # Start of a snippet + start_id = start_ids.pop(0) + if start_id in done_snippets: + continue + + # Reconstruct a single ID line to avoid repetitive ID lines + # by consecutive snippets with multi-ID lines like "//! [1] [2]" + id_line = f"{comment}! [{start_id}]" + done_snippets.append(start_id) + snippet = [id_line] # The snippet starts with this id + + # Find the end of the snippet + j = i + while j < len(lines): + l = lines[j] + j += 1 + + # Add the line to the snippet + snippet.append(l) + + # Check if the snippet is complete + if start_id in get_snippet_ids(l, pattern): + # End of snippet + snippet[len(snippet) - 1] = id_line + snippets[start_id] = snippet + break + + return snippets + + +def get_python_example_snippet_override(start_id: str, rel_path: str) -> List[str]: + """Check if the snippet is overridden by a python example snippet.""" + key = (os.fspath(rel_path), start_id) + value = python_example_snippet_mapping().get(key) + if not value: + return [] + path, id = value + file_lines = path.read_text().splitlines() + snippet_dict = _get_snippets(file_lines, '#', PYTHON_SNIPPET_PATTERN) + lines = snippet_dict.get(id) + if not lines: + raise RuntimeError(f'Snippet "{id}" not found in "{os.fspath(path)}"') + lines = lines[1:-1] # Strip Python snippet markers + return overriden_snippet_lines(lines, start_id) + + +def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]: + """Extract (potentially overlapping) snippets from a C++ file indicated + by '//! [1]'.""" + result = _get_snippets(lines, '//', CPP_SNIPPET_PATTERN) + id_list = result.keys() + for snippet_id in id_list: + # Check file overrides and example overrides + snippet = get_snippet_override(snippet_id, rel_path) + if not snippet: + snippet = get_python_example_snippet_override(snippet_id, rel_path) + if snippet: + result[snippet_id] = snippet + + return result.values() + + +def get_license_from_file(lines): + result = [] + spdx = len(lines) >= 2 and lines[0].startswith("//") and "SPDX" in lines[1] + if spdx: # SPDX, 6.4 + for line in lines: + if line.startswith("//"): + result.append("# " + line[3:]) + else: + break + else: # Old style, C-Header, 6.2 + for line in lines: + if line.startswith("/*") or line.startswith("**"): + result.append(line) + # End of the comment + if line.endswith("*/"): + break + if result: + # We know we have the whole block, so we can + # perform replacements to translate the comment + result[0] = result[0].replace("/*", "**").replace("*", "#") + result[-1] = result[-1].replace("*/", "**").replace("*", "#") + + for i in range(1, len(result) - 1): + result[i] = re.sub(r"^\*\*", "##", result[i]) + return "\n".join(result) + + +def translate_file(file_path, final_path, qt_path, debug, write): + lines = [] + snippets = [] + try: + with file_path.open("r", encoding="utf-8") as f: + lines = f.read().splitlines() + rel_path = file_path.relative_to(qt_path) + snippets = get_snippets(lines, rel_path) + except Exception as e: + log.error(f"Error reading {file_path}: {e}") + raise + if snippets: + # TODO: Get license header first + license_header = get_license_from_file(lines) + if debug: + if have_rich: + console = Console() + table = Table(show_header=True, header_style="bold magenta") + table.add_column("C++") + table.add_column("Python") + + translated_lines = [] + for snippet in snippets: + if snippet and snippet[0] == OVERRIDDEN_SNIPPET: + translated_lines.extend(snippet[1:]) + continue + + for line in snippet: + if not line: + continue + translated_line = snippet_translate(line) + translated_lines.append(translated_line) + + # logging + if debug: + if have_rich: + table.add_row(line, translated_line) + else: + if not opt_quiet: + print(line, translated_line) + + if debug and have_rich: + if not opt_quiet: + console.print(table) + + if write: + # Open the final file + new_suffix = ".h.py" if final_path.name.endswith(".h") else ".py" + target_file = final_path.with_suffix(new_suffix) + + # Directory where the file will be placed, if it does not exists + # we create it. The option 'parents=True' will create the parents + # directories if they don't exist, and if some of them exists, + # the option 'exist_ok=True' will ignore them. + if not target_file.parent.is_dir(): + if not opt_quiet: + log.info(f"Creating directories for {target_file.parent}") + target_file.parent.mkdir(parents=True, exist_ok=True) + + with target_file.open("w", encoding="utf-8") as out_f: + out_f.write("//! [AUTO]\n\n") + out_f.write(license_header) + out_f.write("\n\n") + + for s in translated_lines: + out_f.write(s) + out_f.write("\n") + + if not opt_quiet: + log.info(f"Written: {target_file}") + else: + if not opt_quiet: + log.warning("No snippets were found") + + +def copy_file(file_path, qt_path, out_path, write=False, debug=False): + + # Replicate the Qt path including module under the PySide snippets directory + qt_path_count = len(qt_path.parts) + final_path = out_path.joinpath(*file_path.parts[qt_path_count:]) + + # Check if file exists. + if final_path.exists(): + status_msg = " [yellow][Exists][/yellow]" if have_rich else "[Exists]" + status = FileStatus.Exists + elif final_path.with_suffix(".py").exists(): + status_msg = "[cyan][ExistsPy][/cyan]" if have_rich else "[Exists]" + status = FileStatus.Exists + else: + status_msg = " [green][New][/green]" if have_rich else "[New]" + status = FileStatus.New + + if debug: + if not opt_quiet: + log.info(f"From {file_path} to") + log.info(f"==> {final_path}") + + if not opt_quiet: + if have_rich: + log.info(f"{status_msg} {final_path}", extra={"markup": True}) + else: + log.info(f"{status_msg:10s} {final_path}") + + # Change .cpp to .py, .h to .h.py + # Translate C++ code into Python code + if final_path.name.endswith(".cpp") or final_path.name.endswith(".h"): + translate_file(file_path, final_path, qt_path, debug, write) + + return status + + +def single_directory(options, qt_path, out_path): + # Process all files in the directory + directory_path = Path(options.single_directory) + for file_path in directory_path.glob("**/*"): + if file_path.is_dir() or not is_valid_file(file_path): + continue + copy_file(file_path, qt_path, out_path, write=options.write_files, debug=options.debug) + + +def single_snippet(options, qt_path, out_path): + # Process a single file + file = Path(options.single_snippet) + if is_valid_file(file): + copy_file(file, qt_path, out_path, write=options.write_files, debug=options.debug) + + +def all_modules_in_directory(options, qt_path, out_path): + """ + Process all Qt modules in the directory. Logs how many files were processed. + """ + # New files, already existing files + valid_new, valid_exists = 0, 0 + + for module in qt_path.iterdir(): + module_name = module.name + + # Filter only Qt modules + if not module_name.startswith("qt"): + continue + + if not opt_quiet: + log.info(f"Module {module_name}") + + # Iterating everything + for f in module.glob("**/*.*"): + # Proceed only if the full path contain the filter string + if not is_valid_file(f): + continue + + if options.filter_snippet and options.filter_snippet not in str(f.absolute()): + continue + + status = copy_file(f, qt_path, out_path, write=options.write_files, debug=options.debug) + + # Stats + if status == FileStatus.New: + valid_new += 1 + elif status == FileStatus.Exists: + valid_exists += 1 + + if not opt_quiet: + log.info( + dedent( + f"""\ + Summary: + Total valid files: {valid_new + valid_exists} + New files: {valid_new} + Existing files: {valid_exists} + """ + ) + ) + + +def process_files(options: Namespace) -> None: + qt_path = Path(options.qt_dir) + out_path = Path(options.target_dir) + + # Creating directories in case they don't exist + if not out_path.is_dir(): + out_path.mkdir(parents=True) + + if options.single_directory: + single_directory(options, qt_path, out_path) + elif options.single_snippet: + single_snippet(options, qt_path, out_path) + else: + # General case: process all Qt modules in the directory + all_modules_in_directory(options, qt_path, out_path) + + +if __name__ == "__main__": + parser = get_parser() + opt: Namespace = parser.parse_args() + opt_quiet = not (opt.verbose or opt.debug) + + if not check_arguments(opt): + # Error, invalid arguments + parser.print_help() + sys.exit(-1) + + process_files(opt) diff --git a/tools/snippets_translate/module_classes.py b/tools/snippets_translate/module_classes.py new file mode 100644 index 000000000..df4c7557c --- /dev/null +++ b/tools/snippets_translate/module_classes.py @@ -0,0 +1,1484 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + + +module_classes = { + "Qt3DAnimation": [ + "QAbstractAnimation", + "QAbstractAnimationClip", + "QAbstractChannelMapping", + "QAbstractClipAnimator", + "QAbstractClipBlendNode", + "QAdditiveClipBlend", + "QAnimationAspect", + "QAnimationCallback", + "QAnimationClip", + "QAnimationClipLoader", + "QAnimationController", + "QAnimationGroup", + "QBlendedClipAnimator", + "QClipAnimator", + "QClock", + "QKeyFrame", + "QKeyframeAnimation", + "QLerpClipBlend", + "QMorphTarget", + "QMorphingAnimation", + "QSkeletonMapping", + "QVertexBlendAnimation", + ], + "Qt3DCore": [ + "QAbstractAspect", + "QAbstractSkeleton", + "QArmature", + "QAspectEngine", + "QAspectJob", + "QBackendNode", + "QComponent", + "QComponentAddedChange", + "QComponentRemovedChange", + "QDynamicPropertyUpdatedChange", + "QEntity", + "QJoint", + "QNode", + "QNodeCommand", + "QNodeCreatedChangeBase", + "QNodeDestroyedChange", + "QNodeId", + "QNodeIdTypePair", + "QPropertyNodeAddedChange", + "QPropertyNodeRemovedChange", + "QPropertyUpdatedChange", + "QPropertyUpdatedChangeBase", + "QPropertyValueAddedChange", + "QPropertyValueAddedChangeBase", + "QPropertyValueRemovedChange", + "QPropertyValueRemovedChangeBase", + "QSceneChange", + "QSkeleton", + "QSkeletonLoader", + "QStaticPropertyUpdatedChangeBase", + "QStaticPropertyValueAddedChangeBase", + "QStaticPropertyValueRemovedChangeBase", + "QTransform", + ], + "Qt3DExtras": [ + "QAbstractCameraController", + "QAbstractSpriteSheet", + "QConeGeometry", + "QConeMesh", + "QCuboidGeometry", + "QCuboidMesh", + "QCylinderGeometry", + "QCylinderMesh", + "QDiffuseMapMaterial", + "QDiffuseSpecularMapMaterial", + "QDiffuseSpecularMaterial", + "QExtrudedTextGeometry", + "QExtrudedTextMesh", + "QFirstPersonCameraController", + "QForwardRenderer", + "QGoochMaterial", + "QMetalRoughMaterial", + "QMorphPhongMaterial", + "QNormalDiffuseMapAlphaMaterial", + "QNormalDiffuseMapMaterial", + "QNormalDiffuseSpecularMapMaterial", + "QOrbitCameraController", + "QPerVertexColorMaterial", + "QPhongAlphaMaterial", + "QPhongMaterial", + "QPlaneGeometry", + "QPlaneMesh", + "QSkyboxEntity", + "QSphereGeometry", + "QSphereMesh", + "QSpriteGrid", + "QSpriteSheet", + "QSpriteSheetItem", + "QText2DEntity", + "QTextureMaterial", + "QTorusGeometry", + "QTorusMesh", + "Qt3DWindow", + ], + "Qt3DInput": [ + "QAbstractActionInput", + "QAbstractAxisInput", + "QAbstractPhysicalDevice", + "QAction", + "QActionInput", + "QAnalogAxisInput", + "QAxis", + "QAxisAccumulator", + "QAxisSetting", + "QButtonAxisInput", + "QInputAspect", + "QInputChord", + "QInputSequence", + "QInputSettings", + "QKeyEvent", + "QKeyboardDevice", + "QKeyboardHandler", + "QLogicalDevice", + "QMouseDevice", + "QMouseEvent", + "QMouseHandler", + "QWheelEvent", + ], + "Qt3DLogic": ["QFrameAction", "QLogicAspect"], + "Qt3DRender": [ + "QAbstractFunctor", + "QAbstractLight", + "QAbstractRayCaster", + "QAbstractTexture", + "QAbstractTextureImage", + "QAlphaCoverage", + "QAlphaTest", + "QAttribute", + "QBlendEquation", + "QBlendEquationArguments", + "QBlitFramebuffer", + "QBuffer", + "QBufferCapture", + "QBufferDataGenerator", + "QCamera", + "QCameraLens", + "QCameraSelector", + "QClearBuffers", + "QClipPlane", + "QColorMask", + "QComputeCommand", + "QCullFace", + "QDepthTest", + "QDirectionalLight", + "QDispatchCompute", + "QDithering", + "QEffect", + "QEnvironmentLight", + "QFilterKey", + "QFrameGraphNode", + "QFrameGraphNodeCreatedChangeBase", + "QFrontFace", + "QFrustumCulling", + "QGeometry", + "QGeometryFactory", + "QGeometryRenderer", + "QGraphicsApiFilter", + "QLayer", + "QLayerFilter", + "QLevelOfDetail", + "QLevelOfDetailBoundingSphere", + "QLevelOfDetailSwitch", + "QLineWidth", + "QMaterial", + "QMemoryBarrier", + "QMesh", + "QMultiSampleAntiAliasing", + "QNoDepthMask", + "QNoDraw", + "QNoPicking", + "QObjectPicker", + "QPaintedTextureImage", + "QParameter", + "QPickEvent", + "QPickLineEvent", + "QPickPointEvent", + "QPickTriangleEvent", + "QPickingSettings", + "QPointLight", + "QPointSize", + "QPolygonOffset", + "QProximityFilter", + "QRayCaster", + "QRayCasterHit", + "QRenderAspect", + "QRenderCapabilities", + "QRenderCapture", + "QRenderCaptureReply", + "QRenderPass", + "QRenderPassFilter", + "QRenderSettings", + "QRenderState", + "QRenderStateSet", + "QRenderSurfaceSelector", + "QRenderTarget", + "QRenderTargetOutput", + "QRenderTargetSelector", + "QSceneLoader", + "QScissorTest", + "QScreenRayCaster", + "QSeamlessCubemap", + "QSetFence", + "QShaderData", + "QShaderImage", + "QShaderProgram", + "QShaderProgramBuilder", + "QSharedGLTexture", + "QSortPolicy", + "QSpotLight", + "QStencilMask", + "QStencilOperation", + "QStencilOperationArguments", + "QStencilTest", + "QStencilTestArguments", + "QTechnique", + "QTechniqueFilter", + "QTexture1D", + "QTexture1DArray", + "QTexture2D", + "QTexture2DArray", + "QTexture2DMultisample", + "QTexture2DMultisampleArray", + "QTexture3D", + "QTextureBuffer", + "QTextureCubeMap", + "QTextureCubeMapArray", + "QTextureData", + "QTextureGenerator", + "QTextureImage", + "QTextureImageData", + "QTextureImageDataGenerator", + "QTextureLoader", + "QTextureRectangle", + "QTextureWrapMode", + "QViewport", + "QWaitFence", + ], + "QtCharts": [ + "QAbstractAxis", + "QAbstractBarSeries", + "QAbstractSeries", + "QAreaLegendMarker", + "QAreaSeries", + "QBarCategoryAxis", + "QBarLegendMarker", + "QBarModelMapper", + "QBarSeries", + "QBarSet", + "QBoxPlotLegendMarker", + "QBoxPlotModelMapper", + "QBoxPlotSeries", + "QBoxSet", + "QCandlestickLegendMarker", + "QCandlestickModelMapper", + "QCandlestickSeries", + "QCandlestickSet", + "QCategoryAxis", + "QChart", + "QChartView", + "QDateTimeAxis", + "QHBarModelMapper", + "QHBoxPlotModelMapper", + "QHCandlestickModelMapper", + "QHPieModelMapper", + "QHXYModelMapper", + "QHorizontalBarSeries", + "QHorizontalPercentBarSeries", + "QHorizontalStackedBarSeries", + "QLegend", + "QLegendMarker", + "QLineSeries", + "QLogValueAxis", + "QPercentBarSeries", + "QPieLegendMarker", + "QPieModelMapper", + "QPieSeries", + "QPieSlice", + "QPolarChart", + "QScatterSeries", + "QSplineSeries", + "QStackedBarSeries", + "QVBarModelMapper", + "QVBoxPlotModelMapper", + "QVCandlestickModelMapper", + "QVPieModelMapper", + "QVXYModelMapper", + "QValueAxis", + "QXYLegendMarker", + "QXYModelMapper", + "QXYSeries", + ], + "QtConcurrent": [ + "QFutureQString", + "QFutureVoid", + "QFutureWatcherQString", + "QFutureWatcherVoid", + "QtConcurrent", + ], + "QtCore": [ + "QAbstractAnimation", + "QAbstractEventDispatcher", + "QAbstractItemModel", + "QAbstractListModel", + "QAbstractNativeEventFilter", + "QAbstractProxyModel", + "QAbstractState", + "QAbstractTableModel", + "QAbstractTransition", + "QAnimationGroup", + "QBasicMutex", + "QBasicTimer", + "QBitArray", + "QBuffer", + "QByteArray", + "QByteArrayMatcher", + "QCalendar", + "QCborArray", + "QCborError", + "QCborKnownTags", + "QCborMap", + "QCborParserError", + "QCborSimpleType", + "QCborStreamReader", + "QCborStreamWriter", + "QCborStringResultByteArray", + "QCborStringResultString", + "QCborValue", + "QChildEvent", + "QCollator", + "QCollatorSortKey", + "QCommandLineOption", + "QCommandLineParser", + "QConcatenateTablesProxyModel", + "QCoreApplication", + "QCryptographicHash", + "QDataStream", + "QDate", + "QDateTime", + "QDeadlineTimer", + "QDir", + "QDirIterator", + "QDynamicPropertyChangeEvent", + "QEasingCurve", + "QElapsedTimer", + "QEnum", + "QEvent", + "QEventLoop", + "QEventTransition", + "QFactoryInterface", + "QFile", + "QFileDevice", + "QFileInfo", + "QFileSelector", + "QFileSystemWatcher", + "QFinalState", + "QFlag", + "QFutureInterfaceBase", + "QGenericArgument", + "QGenericReturnArgument", + "QHistoryState", + "QIODevice", + "QIdentityProxyModel", + "QItemSelection", + "QItemSelectionModel", + "QItemSelectionRange", + "QJsonArray", + "QJsonDocument", + "QJsonParseError", + "QJsonValue", + "QLibraryInfo", + "QLine", + "QLineF", + "QLocale", + "QLockFile", + "QMargins", + "QMarginsF", + "QMessageAuthenticationCode", + "QMessageLogContext", + "QMetaClassInfo", + "QMetaEnum", + "QMetaMethod", + "QMetaObject", + "QMetaProperty", + "QMimeData", + "QMimeDatabase", + "QMimeType", + "QModelIndex", + "QMutex", + "QMutexLocker", + "QObject", + "QOperatingSystemVersion", + "QParallelAnimationGroup", + "QPauseAnimation", + "QPersistentModelIndex", + "QPluginLoader", + "QPoint", + "QPointF", + "QProcess", + "QProcessEnvironment", + "QPropertyAnimation", + "QRandomGenerator", + "QRandomGenerator64", + "QReadLocker", + "QReadWriteLock", + "QRect", + "QRectF", + "QRecursiveMutex", + "QRegExp", + "QRegularExpression", + "QRegularExpressionMatch", + "QRegularExpressionMatchIterator", + "QResource", + "QRunnable", + "QSaveFile", + "QSemaphore", + "QSemaphoreReleaser", + "QSequentialAnimationGroup", + "QSettings", + "QSignalBlocker", + "QSignalMapper", + "QSignalTransition", + "QSize", + "QSizeF", + "QSocketDescriptor", + "QSocketNotifier", + "QSortFilterProxyModel", + "QStandardPaths", + "QState", + "QStateMachine", + "QStorageInfo", + "QStringListModel", + "QSysInfo", + "QSystemSemaphore", + "QT_TRANSLATE_NOOP", + "QT_TRANSLATE_NOOP3", + "QT_TRANSLATE_NOOP_UTF8", + "QT_TR_NOOP", + "QT_TR_NOOP_UTF8", + "QTemporaryDir", + "QTemporaryFile", + "QTextBoundaryFinder", + "QTextCodec", + "QTextDecoder", + "QTextEncoder", + "QTextStream", + "QTextStreamManipulator", + "QThread", + "QThreadPool", + "QTime", + "QTimeLine", + "QTimeZone", + "QTimer", + "QTimerEvent", + "QTranslator", + "QTransposeProxyModel", + "QUrl", + "QUrlQuery", + "QUuid", + "QVariantAnimation", + "QVersionNumber", + "QWaitCondition", + "QWriteLocker", + "QXmlStreamAttribute", + "QXmlStreamAttributes", + "QXmlStreamEntityDeclaration", + "QXmlStreamEntityResolver", + "QXmlStreamNamespaceDeclaration", + "QXmlStreamNotationDeclaration", + "QXmlStreamReader", + "QXmlStreamWriter", + "Qt", + "QtCriticalMsg", + "QtDebugMsg", + "QtFatalMsg", + "QtInfoMsg", + "QtMsgType", + "QtSystemMsg", + "QtWarningMsg", + ], + "QtDataVisualization": [ + "Q3DBars", + "Q3DCamera", + "Q3DInputHandler", + "Q3DLight", + "Q3DObject", + "Q3DScatter", + "Q3DScene", + "Q3DSurface", + "Q3DTheme", + "QAbstract3DAxis", + "QAbstract3DGraph", + "QAbstract3DInputHandler", + "QAbstract3DSeries", + "QAbstractDataProxy", + "QBar3DSeries", + "QBarDataItem", + "QBarDataProxy", + "QCategory3DAxis", + "QCustom3DItem", + "QCustom3DLabel", + "QCustom3DVolume", + "QHeightMapSurfaceDataProxy", + "QItemModelBarDataProxy", + "QItemModelScatterDataProxy", + "QItemModelSurfaceDataProxy", + "QLogValue3DAxisFormatter", + "QScatter3DSeries", + "QScatterDataItem", + "QScatterDataProxy", + "QSurface3DSeries", + "QSurfaceDataItem", + "QSurfaceDataProxy", + "QTouch3DInputHandler", + "QValue3DAxis", + "QValue3DAxisFormatter", + ], + "QtGui": [ + "QAbstractOpenGLFunctions", + "QAbstractTextDocumentLayout", + "QAccessible", + "QAccessibleEditableTextInterface", + "QAccessibleEvent", + "QAccessibleInterface", + "QAccessibleObject", + "QAccessibleSelectionInterface", + "QAccessibleStateChangeEvent", + "QAccessibleTableCellInterface", + "QAccessibleTableModelChangeEvent", + "QAccessibleTextCursorEvent", + "QAccessibleTextInsertEvent", + "QAccessibleTextInterface", + "QAccessibleTextRemoveEvent", + "QAccessibleTextSelectionEvent", + "QAccessibleTextUpdateEvent", + "QAccessibleValueChangeEvent", + "QAccessibleValueInterface", + "QActionEvent", + "QBackingStore", + "QBitmap", + "QBrush", + "QClipboard", + "QCloseEvent", + "QColor", + "QColorConstants", + "QColorSpace", + "QConicalGradient", + "QContextMenuEvent", + "QCursor", + "QDesktopServices", + "QDoubleValidator", + "QDrag", + "QDragEnterEvent", + "QDragLeaveEvent", + "QDragMoveEvent", + "QDropEvent", + "QEnterEvent", + "QExposeEvent", + "QFileOpenEvent", + "QFocusEvent", + "QFont", + "QFontDatabase", + "QFontInfo", + "QFontMetrics", + "QFontMetricsF", + "QGradient", + "QGuiApplication", + "QHelpEvent", + "QHideEvent", + "QHoverEvent", + "QIcon", + "QIconDragEvent", + "QIconEngine", + "QImage", + "QImageIOHandler", + "QImageReader", + "QImageWriter", + "QInputEvent", + "QInputMethod", + "QInputMethodEvent", + "QInputMethodQueryEvent", + "QIntValidator", + "QKeyEvent", + "QKeySequence", + "QLinearGradient", + "QMatrix", + "QMatrix2x2", + "QMatrix2x3", + "QMatrix2x4", + "QMatrix3x2", + "QMatrix3x3", + "QMatrix3x4", + "QMatrix4x2", + "QMatrix4x3", + "QMatrix4x4", + "QMouseEvent", + "QMoveEvent", + "QMovie", + "QNativeGestureEvent", + "QOffscreenSurface", + "QOpenGLBuffer", + "QOpenGLContext", + "QOpenGLContextGroup", + "QOpenGLDebugLogger", + "QOpenGLDebugMessage", + "QOpenGLExtraFunctions", + "QOpenGLFramebufferObject", + "QOpenGLFramebufferObjectFormat", + "QOpenGLFunctions", + "QOpenGLPixelTransferOptions", + "QOpenGLShader", + "QOpenGLShaderProgram", + "QOpenGLTexture", + "QOpenGLTextureBlitter", + "QOpenGLTimeMonitor", + "QOpenGLTimerQuery", + "QOpenGLVersionProfile", + "QOpenGLVertexArrayObject", + "QOpenGLWindow", + "QPageLayout", + "QPageSize", + "QPagedPaintDevice", + "QPaintDevice", + "QPaintDeviceWindow", + "QPaintEngine", + "QPaintEngineState", + "QPaintEvent", + "QPainter", + "QPainterPath", + "QPainterPathStroker", + "QPalette", + "QPdfWriter", + "QPen", + "QPicture", + "QPictureIO", + "QPixelFormat", + "QPixmap", + "QPixmapCache", + "QPointingDeviceUniqueId", + "QPolygon", + "QPolygonF", + "QPyTextObject", + "QQuaternion", + "QRadialGradient", + "QRasterWindow", + "QRawFont", + "QRegExpValidator", + "QRegion", + "QRegularExpressionValidator", + "QResizeEvent", + "QScreen", + "QScrollEvent", + "QScrollPrepareEvent", + "QSessionManager", + "QShortcutEvent", + "QShowEvent", + "QStandardItem", + "QStandardItemModel", + "QStaticText", + "QStatusTipEvent", + "QStyleHints", + "QSurface", + "QSurfaceFormat", + "QSyntaxHighlighter", + "QTabletEvent", + "QTextBlock", + "QTextBlockFormat", + "QTextBlockGroup", + "QTextBlockUserData", + "QTextCharFormat", + "QTextCursor", + "QTextDocument", + "QTextDocumentFragment", + "QTextDocumentWriter", + "QTextFormat", + "QTextFragment", + "QTextFrame", + "QTextFrameFormat", + "QTextImageFormat", + "QTextInlineObject", + "QTextItem", + "QTextLayout", + "QTextLength", + "QTextLine", + "QTextList", + "QTextListFormat", + "QTextObject", + "QTextObjectInterface", + "QTextOption", + "QTextTable", + "QTextTableCell", + "QTextTableCellFormat", + "QTextTableFormat", + "QToolBarChangeEvent", + "QTouchDevice", + "QTouchEvent", + "QTransform", + "QValidator", + "QVector2D", + "QVector3D", + "QVector4D", + "QWhatsThisClickedEvent", + "QWheelEvent", + "QWindow", + "QWindowStateChangeEvent", + "Qt", + ], + "QtHelp": [ + "QCompressedHelpInfo", + "QHelpContentItem", + "QHelpContentModel", + "QHelpContentWidget", + "QHelpEngine", + "QHelpEngineCore", + "QHelpFilterData", + "QHelpFilterEngine", + "QHelpFilterSettingsWidget", + "QHelpIndexModel", + "QHelpIndexWidget", + "QHelpLink", + "QHelpSearchEngine", + "QHelpSearchQuery", + "QHelpSearchQueryWidget", + "QHelpSearchResult", + "QHelpSearchResultWidget", + ], + "QtLocation": [ + "QGeoCodeReply", + "QGeoCodingManager", + "QGeoCodingManagerEngine", + "QGeoManeuver", + "QGeoRoute", + "QGeoRouteReply", + "QGeoRouteRequest", + "QGeoRouteSegment", + "QGeoRoutingManager", + "QGeoRoutingManagerEngine", + "QGeoServiceProvider", + "QGeoServiceProviderFactory", + "QGeoServiceProviderFactoryV2", + "QPlace", + "QPlaceAttribute", + "QPlaceCategory", + "QPlaceContactDetail", + "QPlaceContent", + "QPlaceContentReply", + "QPlaceContentRequest", + "QPlaceDetailsReply", + "QPlaceEditorial", + "QPlaceIcon", + "QPlaceIdReply", + "QPlaceImage", + "QPlaceManager", + "QPlaceManagerEngine", + "QPlaceMatchReply", + "QPlaceMatchRequest", + "QPlaceProposedSearchResult", + "QPlaceRatings", + "QPlaceReply", + "QPlaceResult", + "QPlaceReview", + "QPlaceSearchReply", + "QPlaceSearchRequest", + "QPlaceSearchResult", + "QPlaceSearchSuggestionReply", + "QPlaceSupplier", + "QPlaceUser", + ], + "QtMultimedia": [ + "QAbstractAudioDeviceInfo", + "QAbstractAudioInput", + "QAbstractAudioOutput", + "QAbstractVideoBuffer", + "QAbstractVideoFilter", + "QAbstractVideoSurface", + "QAudio", + "QAudioBuffer", + "QAudioDecoder", + "QAudioDecoderControl", + "QAudioDeviceInfo", + "QAudioEncoderSettings", + "QAudioEncoderSettingsControl", + "QAudioFormat", + "QAudioInput", + "QAudioInputSelectorControl", + "QAudioOutput", + "QAudioOutputSelectorControl", + "QAudioProbe", + "QAudioRecorder", + "QAudioRoleControl", + "QCamera", + "QCameraCaptureBufferFormatControl", + "QCameraCaptureDestinationControl", + "QCameraControl", + "QCameraExposure", + "QCameraExposureControl", + "QCameraFeedbackControl", + "QCameraFlashControl", + "QCameraFocus", + "QCameraFocusControl", + "QCameraFocusZone", + "QCameraImageCapture", + "QCameraImageCaptureControl", + "QCameraImageProcessing", + "QCameraImageProcessingControl", + "QCameraInfo", + "QCameraInfoControl", + "QCameraLocksControl", + "QCameraViewfinderSettings", + "QCameraViewfinderSettingsControl", + "QCameraViewfinderSettingsControl2", + "QCameraZoomControl", + "QCustomAudioRoleControl", + "QImageEncoderControl", + "QImageEncoderSettings", + "QMediaAudioProbeControl", + "QMediaAvailabilityControl", + "QMediaBindableInterface", + "QMediaContainerControl", + "QMediaContent", + "QMediaControl", + "QMediaGaplessPlaybackControl", + "QMediaNetworkAccessControl", + "QMediaObject", + "QMediaPlayer", + "QMediaPlayerControl", + "QMediaPlaylist", + "QMediaRecorder", + "QMediaRecorderControl", + "QMediaResource", + "QMediaService", + "QMediaServiceCameraInfoInterface", + "QMediaServiceDefaultDeviceInterface", + "QMediaServiceFeaturesInterface", + "QMediaServiceProviderHint", + "QMediaServiceSupportedDevicesInterface", + "QMediaServiceSupportedFormatsInterface", + "QMediaStreamsControl", + "QMediaTimeInterval", + "QMediaTimeRange", + "QMediaVideoProbeControl", + "QMetaDataReaderControl", + "QMetaDataWriterControl", + "QMultimedia", + "QRadioData", + "QRadioDataControl", + "QRadioTuner", + "QRadioTunerControl", + "QSound", + "QSoundEffect", + "QVideoDeviceSelectorControl", + "QVideoEncoderSettings", + "QVideoEncoderSettingsControl", + "QVideoFilterRunnable", + "QVideoFrame", + "QVideoProbe", + "QVideoRendererControl", + "QVideoSurfaceFormat", + "QVideoWindowControl", + ], + "QtMultimediaWidgets": [ + "QCameraViewfinder", + "QGraphicsVideoItem", + "QVideoWidget", + "QVideoWidgetControl", + ], + "QtNetwork": [ + "QAbstractNetworkCache", + "QAbstractSocket", + "QAuthenticator", + "QDnsDomainNameRecord", + "QDnsHostAddressRecord", + "QDnsLookup", + "QDnsMailExchangeRecord", + "QDnsServiceRecord", + "QDnsTextRecord", + "QDtls", + "QDtlsError", + "QHostAddress", + "QHostInfo", + "QHstsPolicy", + "QHttpMultiPart", + "QHttpPart", + "QIPv6Address", + "QLocalServer", + "QLocalSocket", + "QNetworkAccessManager", + "QNetworkAddressEntry", + "QNetworkCacheMetaData", + "QNetworkConfiguration", + "QNetworkConfigurationManager", + "QNetworkCookie", + "QNetworkCookieJar", + "QNetworkDatagram", + "QNetworkDiskCache", + "QNetworkInterface", + "QNetworkProxy", + "QNetworkProxyFactory", + "QNetworkProxyQuery", + "QNetworkReply", + "QNetworkRequest", + "QNetworkSession", + "QOcspCertificateStatus", + "QOcspResponse", + "QOcspRevocationReason", + "QPasswordDigestor", + "QSsl", + "QSslCertificate", + "QSslCertificateExtension", + "QSslCipher", + "QSslConfiguration", + "QSslDiffieHellmanParameters", + "QSslError", + "QSslKey", + "QSslPreSharedKeyAuthenticator", + "QSslSocket", + "QTcpServer", + "QTcpSocket", + "QUdpSocket", + ], + "QtOpenGL": [ + "QGL", + "QGLBuffer", + "QGLColormap", + "QGLContext", + "QGLFormat", + "QGLFramebufferObject", + "QGLFramebufferObjectFormat", + "QGLPixelBuffer", + "QGLShader", + "QGLShaderProgram", + "QGLWidget", + ], + "QtOpenGLFunctions": [ + "QOpenGLFunctions_1_0", + "QOpenGLFunctions_1_1", + "QOpenGLFunctions_1_2", + "QOpenGLFunctions_1_3", + "QOpenGLFunctions_1_4", + "QOpenGLFunctions_1_5", + "QOpenGLFunctions_2_0", + "QOpenGLFunctions_2_1", + "QOpenGLFunctions_3_0", + "QOpenGLFunctions_3_1", + "QOpenGLFunctions_3_2_Compatibility", + "QOpenGLFunctions_3_2_Core", + "QOpenGLFunctions_3_3_Compatibility", + "QOpenGLFunctions_3_3_Core", + "QOpenGLFunctions_4_0_Compatibility", + "QOpenGLFunctions_4_0_Core", + "QOpenGLFunctions_4_1_Compatibility", + "QOpenGLFunctions_4_1_Core", + "QOpenGLFunctions_4_2_Compatibility", + "QOpenGLFunctions_4_2_Core", + "QOpenGLFunctions_4_3_Compatibility", + "QOpenGLFunctions_4_3_Core", + "QOpenGLFunctions_4_4_Compatibility", + "QOpenGLFunctions_4_4_Core", + "QOpenGLFunctions_4_5_Compatibility", + "QOpenGLFunctions_4_5_Core", + ], + "QtPositioning": [ + "QGeoAddress", + "QGeoAreaMonitorInfo", + "QGeoAreaMonitorSource", + "QGeoCircle", + "QGeoCoordinate", + "QGeoLocation", + "QGeoPath", + "QGeoPolygon", + "QGeoPositionInfo", + "QGeoPositionInfoSource", + "QGeoPositionInfoSourceFactory", + "QGeoRectangle", + "QGeoSatelliteInfo", + "QGeoSatelliteInfoSource", + "QGeoShape", + "QNmeaPositionInfoSource", + ], + "QtPrintSupport": [ + "QAbstractPrintDialog", + "QPageSetupDialog", + "QPrintDialog", + "QPrintEngine", + "QPrintPreviewDialog", + "QPrintPreviewWidget", + "QPrinter", + "QPrinterInfo", + ], + "QtQml": [ + "QJSEngine", + "QJSValue", + "QJSValueIterator", + "QQmlAbstractUrlInterceptor", + "QQmlApplicationEngine", + "QQmlComponent", + "QQmlContext", + "QQmlDebuggingEnabler", + "QQmlEngine", + "QQmlError", + "QQmlExpression", + "QQmlExtensionInterface", + "QQmlExtensionPlugin", + "QQmlFile", + "QQmlFileSelector", + "QQmlImageProviderBase", + "QQmlIncubationController", + "QQmlIncubator", + "QQmlListReference", + "QQmlNetworkAccessManagerFactory", + "QQmlParserStatus", + "QQmlProperty", + "QQmlPropertyMap", + "QQmlPropertyValueSource", + "QQmlScriptString", + "QQmlTypesExtensionInterface", + "QtQml", + ], + "QtQuick": [ + "QQuickAsyncImageProvider", + "QQuickFramebufferObject", + "QQuickImageProvider", + "QQuickImageResponse", + "QQuickItem", + "QQuickItemGrabResult", + "QQuickPaintedItem", + "QQuickRenderControl", + "QQuickTextDocument", + "QQuickTextureFactory", + "QQuickTransform", + "QQuickView", + "QQuickWindow", + "QSGAbstractRenderer", + "QSGBasicGeometryNode", + "QSGClipNode", + "QSGDynamicTexture", + "QSGEngine", + "QSGGeometry", + "QSGGeometryNode", + "QSGMaterialType", + "QSGNode", + "QSGOpacityNode", + "QSGSimpleRectNode", + "QSGSimpleTextureNode", + "QSGTexture", + "QSGTextureProvider", + "QSGTransformNode", + "QSharedPointer<QQuickItemGrabResult >", + ], + "QtQuickControls2": ["QQuickStyle"], + "QtQuickWidgets": ["QQuickWidget"], + "QtRemoteObjects": [ + "QAbstractItemModelReplica", + "QRemoteObjectAbstractPersistedStore", + "QRemoteObjectDynamicReplica", + "QRemoteObjectHost", + "QRemoteObjectHostBase", + "QRemoteObjectNode", + "QRemoteObjectPendingCall", + "QRemoteObjectPendingCallWatcher", + "QRemoteObjectRegistry", + "QRemoteObjectRegistryHost", + "QRemoteObjectReplica", + "QRemoteObjectSettingsStore", + "QRemoteObjectSourceLocationInfo", + ], + "QtScript": [ + "QScriptClass", + "QScriptClassPropertyIterator", + "QScriptContext", + "QScriptContextInfo", + "QScriptEngine", + "QScriptEngineAgent", + "QScriptExtensionInterface", + "QScriptExtensionPlugin", + "QScriptProgram", + "QScriptString", + "QScriptValue", + "QScriptValueIterator", + "QScriptable", + ], + "QtScriptTools": ["QScriptEngineDebugger"], + "QtScxml": [ + "QScxmlCompiler", + "QScxmlCppDataModel", + "QScxmlDataModel", + "QScxmlDynamicScxmlServiceFactory", + "QScxmlEcmaScriptDataModel", + "QScxmlError", + "QScxmlEvent", + "QScxmlExecutableContent", + "QScxmlInvokableService", + "QScxmlInvokableServiceFactory", + "QScxmlNullDataModel", + "QScxmlStateMachine", + "QScxmlStaticScxmlServiceFactory", + "QScxmlTableData", + ], + "QtSensors": [ + "QAccelerometer", + "QAccelerometerFilter", + "QAccelerometerReading", + "QAltimeter", + "QAltimeterFilter", + "QAltimeterReading", + "QAmbientLightFilter", + "QAmbientLightReading", + "QAmbientLightSensor", + "QAmbientTemperatureFilter", + "QAmbientTemperatureReading", + "QAmbientTemperatureSensor", + "QCompass", + "QCompassFilter", + "QCompassReading", + "QDistanceFilter", + "QDistanceReading", + "QDistanceSensor", + "QGyroscope", + "QGyroscopeFilter", + "QGyroscopeReading", + "QHolsterFilter", + "QHolsterReading", + "QHolsterSensor", + "QHumidityFilter", + "QHumidityReading", + "QHumiditySensor", + "QIRProximityFilter", + "QIRProximityReading", + "QIRProximitySensor", + "QLidFilter", + "QLidReading", + "QLidSensor", + "QLightFilter", + "QLightReading", + "QLightSensor", + "QMagnetometer", + "QMagnetometerFilter", + "QMagnetometerReading", + "QOrientationFilter", + "QOrientationReading", + "QOrientationSensor", + "QPressureFilter", + "QPressureReading", + "QPressureSensor", + "QProximityFilter", + "QProximityReading", + "QProximitySensor", + "QRotationFilter", + "QRotationReading", + "QRotationSensor", + "QSensor", + "QSensorBackend", + "QSensorBackendFactory", + "QSensorChangesInterface", + "QSensorFilter", + "QSensorGestureManager", + "QSensorGesturePluginInterface", + "QSensorGestureRecognizer", + "QSensorManager", + "QSensorPluginInterface", + "QSensorReading", + "QTapFilter", + "QTapReading", + "QTapSensor", + "QTiltFilter", + "QTiltReading", + "QTiltSensor", + ], + "QtSerialPort": ["QSerialPort", "QSerialPortInfo"], + "QtSql": [ + "QSql", + "QSqlDatabase", + "QSqlDriver", + "QSqlDriverCreatorBase", + "QSqlError", + "QSqlField", + "QSqlIndex", + "QSqlQuery", + "QSqlQueryModel", + "QSqlRecord", + "QSqlRelation", + "QSqlRelationalDelegate", + "QSqlRelationalTableModel", + "QSqlResult", + "QSqlTableModel", + ], + "QtSvg": ["QGraphicsSvgItem", "QSvgGenerator", "QSvgRenderer", "QSvgWidget"], + "QtTest": ["QTest"], + "QtTextToSpeech": ["QTextToSpeech", "QTextToSpeechEngine", "QVoice"], + "QtUiTools": ["QUiLoader"], + "QtWebChannel": ["QWebChannel", "QWebChannelAbstractTransport"], + "QtWebEngine": [], + "QtWebEngineCore": [ + "QWebEngineCookieStore", + "QWebEngineHttpRequest", + "QWebEngineUrlRequestInfo", + "QWebEngineUrlRequestInterceptor", + "QWebEngineUrlRequestJob", + "QWebEngineUrlScheme", + "QWebEngineUrlSchemeHandler", + ], + "QtWebEngineWidgets": [ + "QWebEngineCertificateError", + "QWebEngineContextMenuData", + "QWebEngineDownloadItem", + "QWebEngineFullScreenRequest", + "QWebEngineHistory", + "QWebEngineHistoryItem", + "QWebEnginePage", + "QWebEngineProfile", + "QWebEngineScript", + "QWebEngineScriptCollection", + "QWebEngineSettings", + "QWebEngineView", + ], + "QtWebSockets": [ + "QMaskGenerator", + "QWebSocket", + "QWebSocketCorsAuthenticator", + "QWebSocketProtocol", + "QWebSocketServer", + ], + "QtWidgets": [ + "QAbstractButton", + "QAbstractGraphicsShapeItem", + "QAbstractItemDelegate", + "QAbstractItemView", + "QAbstractScrollArea", + "QAbstractSlider", + "QAbstractSpinBox", + "QAccessibleWidget", + "QAction", + "QActionGroup", + "QApplication", + "QBoxLayout", + "QButtonGroup", + "QCalendarWidget", + "QCheckBox", + "QColorDialog", + "QColormap", + "QColumnView", + "QComboBox", + "QCommandLinkButton", + "QCommonStyle", + "QCompleter", + "QDataWidgetMapper", + "QDateEdit", + "QDateTimeEdit", + "QDesktopWidget", + "QDial", + "QDialog", + "QDialogButtonBox", + "QDirModel", + "QDockWidget", + "QDoubleSpinBox", + "QErrorMessage", + "QFileDialog", + "QFileIconProvider", + "QFileSystemModel", + "QFocusFrame", + "QFontComboBox", + "QFontDialog", + "QFormLayout", + "QFrame", + "QGesture", + "QGestureEvent", + "QGestureRecognizer", + "QGraphicsAnchor", + "QGraphicsAnchorLayout", + "QGraphicsBlurEffect", + "QGraphicsColorizeEffect", + "QGraphicsDropShadowEffect", + "QGraphicsEffect", + "QGraphicsEllipseItem", + "QGraphicsGridLayout", + "QGraphicsItem", + "QGraphicsItemAnimation", + "QGraphicsItemGroup", + "QGraphicsLayout", + "QGraphicsLayoutItem", + "QGraphicsLineItem", + "QGraphicsLinearLayout", + "QGraphicsObject", + "QGraphicsOpacityEffect", + "QGraphicsPathItem", + "QGraphicsPixmapItem", + "QGraphicsPolygonItem", + "QGraphicsProxyWidget", + "QGraphicsRectItem", + "QGraphicsRotation", + "QGraphicsScale", + "QGraphicsScene", + "QGraphicsSceneContextMenuEvent", + "QGraphicsSceneDragDropEvent", + "QGraphicsSceneEvent", + "QGraphicsSceneHelpEvent", + "QGraphicsSceneHoverEvent", + "QGraphicsSceneMouseEvent", + "QGraphicsSceneMoveEvent", + "QGraphicsSceneResizeEvent", + "QGraphicsSceneWheelEvent", + "QGraphicsSimpleTextItem", + "QGraphicsTextItem", + "QGraphicsTransform", + "QGraphicsView", + "QGraphicsWidget", + "QGridLayout", + "QGroupBox", + "QHBoxLayout", + "QHeaderView", + "QInputDialog", + "QItemDelegate", + "QItemEditorCreatorBase", + "QItemEditorFactory", + "QKeyEventTransition", + "QKeySequenceEdit", + "QLCDNumber", + "QLabel", + "QLayout", + "QLayoutItem", + "QLineEdit", + "QListView", + "QListWidget", + "QListWidgetItem", + "QMainWindow", + "QMdiArea", + "QMdiSubWindow", + "QMenu", + "QMenuBar", + "QMessageBox", + "QMouseEventTransition", + "QOpenGLWidget", + "QPanGesture", + "QPinchGesture", + "QPlainTextDocumentLayout", + "QPlainTextEdit", + "QProgressBar", + "QProgressDialog", + "QProxyStyle", + "QPushButton", + "QRadioButton", + "QRubberBand", + "QScrollArea", + "QScrollBar", + "QScroller", + "QScrollerProperties", + "QShortcut", + "QSizeGrip", + "QSizePolicy", + "QSlider", + "QSpacerItem", + "QSpinBox", + "QSplashScreen", + "QSplitter", + "QSplitterHandle", + "QStackedLayout", + "QStackedWidget", + "QStatusBar", + "QStyle", + "QStyleFactory", + "QStyleHintReturn", + "QStyleHintReturnMask", + "QStyleHintReturnVariant", + "QStyleOption", + "QStyleOptionButton", + "QStyleOptionComboBox", + "QStyleOptionComplex", + "QStyleOptionDockWidget", + "QStyleOptionFocusRect", + "QStyleOptionFrame", + "QStyleOptionGraphicsItem", + "QStyleOptionGroupBox", + "QStyleOptionHeader", + "QStyleOptionMenuItem", + "QStyleOptionProgressBar", + "QStyleOptionRubberBand", + "QStyleOptionSizeGrip", + "QStyleOptionSlider", + "QStyleOptionSpinBox", + "QStyleOptionTab", + "QStyleOptionTabBarBase", + "QStyleOptionTabWidgetFrame", + "QStyleOptionTitleBar", + "QStyleOptionToolBar", + "QStyleOptionToolBox", + "QStyleOptionToolButton", + "QStyleOptionViewItem", + "QStylePainter", + "QStyledItemDelegate", + "QSwipeGesture", + "QSystemTrayIcon", + "QTabBar", + "QTabWidget", + "QTableView", + "QTableWidget", + "QTableWidgetItem", + "QTableWidgetSelectionRange", + "QTapAndHoldGesture", + "QTapGesture", + "QTextBrowser", + "QTextEdit", + "QTileRules", + "QTimeEdit", + "QToolBar", + "QToolBox", + "QToolButton", + "QToolTip", + "QTreeView", + "QTreeWidget", + "QTreeWidgetItem", + "QTreeWidgetItemIterator", + "QUndoCommand", + "QUndoGroup", + "QUndoStack", + "QUndoView", + "QVBoxLayout", + "QWhatsThis", + "QWidget", + "QWidgetAction", + "QWidgetItem", + "QWizard", + "QWizardPage", + ], + "QtX11Extras": ["QX11Info"], + "QtXml": [ + "QDomAttr", + "QDomCDATASection", + "QDomCharacterData", + "QDomComment", + "QDomDocument", + "QDomDocumentFragment", + "QDomDocumentType", + "QDomElement", + "QDomEntity", + "QDomEntityReference", + "QDomImplementation", + "QDomNamedNodeMap", + "QDomNode", + "QDomNodeList", + "QDomNotation", + "QDomProcessingInstruction", + "QDomText", + "QXmlAttributes", + "QXmlContentHandler", + "QXmlDTDHandler", + "QXmlDeclHandler", + "QXmlDefaultHandler", + "QXmlEntityResolver", + "QXmlErrorHandler", + "QXmlInputSource", + "QXmlLexicalHandler", + "QXmlLocator", + "QXmlNamespaceSupport", + "QXmlParseException", + "QXmlReader", + "QXmlSimpleReader", + ], + "QtXmlPatterns": [ + "QAbstractMessageHandler", + "QAbstractUriResolver", + "QAbstractXmlNodeModel", + "QAbstractXmlReceiver", + "QSourceLocation", + "QXmlFormatter", + "QXmlItem", + "QXmlName", + "QXmlNamePool", + "QXmlNodeModelIndex", + "QXmlQuery", + "QXmlResultItems", + "QXmlSchema", + "QXmlSchemaValidator", + "QXmlSerializer", + ], +} diff --git a/tools/snippets_translate/override.py b/tools/snippets_translate/override.py new file mode 100644 index 000000000..e7623d8a5 --- /dev/null +++ b/tools/snippets_translate/override.py @@ -0,0 +1,112 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +from pathlib import Path + +ROOT_PATH = Path(__file__).parents[2] +EXAMPLES_PATH = ROOT_PATH / "examples" +TUTORIAL_EXAMPLES_PATH = ROOT_PATH / "sources" / "pyside6" / "doc" / "tutorials" + + +_PYTHON_EXAMPLE_SNIPPET_MAPPING = { + ("qtbase/examples/widgets/tutorials/modelview/1_readonly/mymodel.cpp", + "Quoting ModelView Tutorial"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "1_readonly.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/2_formatting/mymodel.cpp", + "Quoting ModelView Tutorial"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "2_formatting.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp", + "quoting mymodel_QVariant"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "2"), + ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp", + "quoting mymodel_a"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp", + "quoting mymodel_b"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "3"), + ("qtbase/examples/widgets/tutorials/modelview/4_headers/mymodel.cpp", + "quoting mymodel_c"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "4_headers.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/5_edit/mymodel.cpp", + "quoting mymodel_e"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "5_edit.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/5_edit/mymodel.cpp", + "quoting mymodel_f"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "5_edit.py", "2"), + ("qtbase/examples/widgets/tutorials/modelview/6_treeview/mainwindow.cpp", + "Quoting ModelView Tutorial"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "6_treeview.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/7_selections/mainwindow.cpp", + "quoting modelview_a"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "7_selections.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/7_selections/mainwindow.cpp", + "quoting modelview_b"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "7_selections.py", "2"), + ("qtbase/src/widgets/doc/snippets/qlistview-dnd/mainwindow.cpp.cpp", "0"): + (TUTORIAL_EXAMPLES_PATH / "modelviewprogramming" / "qlistview-dnd.py", "mainwindow0") +} + + +_python_example_snippet_mapping = {} + + +def python_example_snippet_mapping(): + global _python_example_snippet_mapping + if not _python_example_snippet_mapping: + result = _PYTHON_EXAMPLE_SNIPPET_MAPPING + + qt_path = "qtbase/src/widgets/doc/snippets/simplemodel-use/main.cpp" + pyside_path = TUTORIAL_EXAMPLES_PATH / "modelviewprogramming" / "stringlistmodel.py" + for i in range(3): + snippet_id = str(i) + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtbase/src/widgets/doc/snippets/stringlistmodel/main.cpp" + pyside_path = TUTORIAL_EXAMPLES_PATH / "modelviewprogramming" / "stringlistmodel.py" + for i in range(6): + snippet_id = str(i) + result[(qt_path, snippet_id)] = pyside_path, f"main{snippet_id}" + + qt_path = "qtbase/examples/widgets/itemviews/spinboxdelegate/delegate.cpp" + pyside_path = (EXAMPLES_PATH / "widgets" / "itemviews" / "spinboxdelegate" + / "spinboxdelegate.py") + for i in range(5): + snippet_id = str(i) + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtbase/src/widgets/doc/snippets/stringlistmodel/model.cpp" + pyside_path = (TUTORIAL_EXAMPLES_PATH / "modelviewprogramming" + / "stringlistmodel.py") + for i in range(10): + snippet_id = str(i) + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtbase/src/widgets/doc/snippets/qlistview-dnd/model.cpp" + pyside_path = (TUTORIAL_EXAMPLES_PATH / "modelviewprogramming" + / "qlistview-dnd.py") + for i in range(11): + snippet_id = str(i) + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtconnectivity/examples/bluetooth/heartrate_game/devicefinder.cpp" + pyside_path = EXAMPLES_PATH / "bluetooth" / "heartrate_game" / "devicefinder.py" + for i in range(5): + snippet_id = f"devicediscovery-{i}" + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtconnectivity/examples/bluetooth/heartrate_game/devicehandler.cpp" + pyside_path = EXAMPLES_PATH / "bluetooth" / "heartrate_game" / "devicehandler.py" + for snippet_id in ["Connect-Signals-1", "Connect-Signals-2", + "Filter HeartRate service 2", "Find HRM characteristic", + "Reading value"]: + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtconnectivity/examples/bluetooth/heartrate_server/main.cpp" + pyside_path = EXAMPLES_PATH / "bluetooth" / "heartrate_server" / "heartrate_server.py" + for snippet_id in ["Advertising Data", "Start Advertising", "Service Data", + "Provide Heartbeat"]: + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + _python_example_snippet_mapping = result + + return _python_example_snippet_mapping diff --git a/tools/snippets_translate/parse_utils.py b/tools/snippets_translate/parse_utils.py new file mode 100644 index 000000000..234d1b669 --- /dev/null +++ b/tools/snippets_translate/parse_utils.py @@ -0,0 +1,109 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import re + +from module_classes import module_classes + + +def get_qt_module_class(x): + """ + Receives the name of an include: + 'QSomething' from '#include <QSomething>' + + Returns a tuple '(bool, str)' where the 'bool' is True if the name is + a module by itself, like QtCore or QtWidgets, and False if it's a class + from one of those modules. The 'str' returns the name of the module + where the class belongs, or the same module. + + In case it doesn't find the class or the module, it will return None. + """ + if "/" in x: + x = x.split("/")[-1] + + for imodule, iclasses in module_classes.items(): + if imodule == x: + return True, x + for iclass in iclasses: + if iclass == x: + return False, imodule + return None + + +def get_indent(x): + return " " * (len(x) - len(x.lstrip())) + + +# Remove more than one whitespace from the code, but not considering +# the indentation. Also removes '&', '*', and ';' from arguments. +def dstrip(x): + right = x + if re.search(r"\s+", x): + right = re.sub(" +", " ", x).strip() + if "&" in right: + right = right.replace("&", "") + + if "*" in right: + re_pointer = re.compile(r"\*(.)") + next_char = re_pointer.search(x) + if next_char: + if next_char.group(1).isalpha(): + right = right.replace("*", "") + + if right.endswith(";"): + right = right.replace(";", "") + x = f"{get_indent(x)}{right}" + + return x + + +def remove_ref(var_name): + var = var_name.strip() + while var.startswith("*") or var.startswith("&"): + var = var[1:] + return var.lstrip() + + +def parse_arguments(p): + unnamed_var = 0 + if "," in p: + v = "" + for i, arg in enumerate(p.split(",")): + if i != 0: + v += ", " + if arg: + new_value = arg.split()[-1] + # handle no variable name + if new_value.strip() == "*": + v += f"arg__{unnamed_var}" + unnamed_var += 1 + else: + v += arg.split()[-1] + elif p.strip(): + new_value = p.split()[-1] + if new_value.strip() == "*": + v = f"arg__{unnamed_var}" + else: + v = new_value + else: + v = p + + return v + + +def replace_main_commas(v): + # : QWidget(parent), Something(else, and, other), value(1) + new_v = "" + parenthesis = 0 + for c in v: + if c == "(": + parenthesis += 1 + elif c == ")": + parenthesis -= 1 + + if c == "," and parenthesis == 0: + c = "@" + + new_v += c + + return new_v diff --git a/tools/snippets_translate/requirements.txt b/tools/snippets_translate/requirements.txt new file mode 100644 index 000000000..1fb678867 --- /dev/null +++ b/tools/snippets_translate/requirements.txt @@ -0,0 +1,2 @@ +rich +pytest diff --git a/tools/snippets_translate/snippets_translate.pyproject b/tools/snippets_translate/snippets_translate.pyproject new file mode 100644 index 000000000..f660033c1 --- /dev/null +++ b/tools/snippets_translate/snippets_translate.pyproject @@ -0,0 +1,4 @@ +{ + "files": ["main.py", "converter.py", "handlers.py", "override.py", + "tests/test_converter.py", "tests/test_snippets.py"] +} diff --git a/tools/snippets_translate/tests/test_converter.py b/tools/snippets_translate/tests/test_converter.py new file mode 100644 index 000000000..084cc8a6d --- /dev/null +++ b/tools/snippets_translate/tests/test_converter.py @@ -0,0 +1,481 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +from converter import snippet_translate as st + + +def multi_st(lines): + result = [st(l) for l in lines.split("\n")] + return "\n".join(result) + + +def test_comments(): + assert st("// This is a comment") == "# This is a comment" + assert st("// double slash // inside") == "# double slash // inside" + + +def test_comments_eol(): + assert st("a = 1; // comment") == "a = 1 # comment" + assert st("while ( 1 != 1 ) { // comment") == "while 1 != 1: # comment" + + +def test_qdoc_snippets(): + assert st("//! [0]") == "//! [0]" + + +def test_arrow(): + assert st("label->setText('something')") == "label.setText('something')" + + +def test_curly_braces(): + assert st(" {") == "" + assert st("}") == "" + assert st("while (true){") == "while True:" + assert st("while (true) { ") == "while True:" + + +def test_inc_dec(): + assert st("++i;") == "i = i + 1" + assert st("i--;") == "i = i - 1" + + +def test_and_or(): + assert st("while (a && b)") == "while a and b:" + assert st("else if (a || b && c)") == "elif a or b and c:" + + +def test_while_if_elseif(): + assert st("while(a)") == "while a:" + assert st("if (condition){") == "if condition:" + assert st(" if (condition){") == " if condition:" + assert st("} else if (a) {") == " elif a:" + assert ( + st("if (!m_vbo.isCreated()) // init() failed,") + == "if not m_vbo.isCreated(): # init() failed," + ) + # Special case, second line from a condition + assert ( + st("&& event->answerRect().intersects(dropFrame->geometry()))") + == "and event.answerRect().intersects(dropFrame.geometry()))" + ) + + +def test_else(): + assert st("else") == "else:" + assert st("} else {") == "else:" + assert st("}else") == "else:" + assert st("else {") == "else:" + + +def test_new(): + assert st("a = new Something(...);") == "a = Something(...)" + assert st("a = new Something") == "a = Something()" + assert st("foo(new X, new Y(b), new Z)") == "foo(X(), Y(b), Z())" + # Class member initialization list + assert st("m_mem(new Something(p)),") == "m_mem(Something(p))," + assert st("m_mem(new Something),") == "m_mem(Something())," + + +def test_semicolon(): + assert st("a = 1;") == "a = 1" + assert st("};") == "" + + +def test_include(): + assert st('#include "something.h"') == "from something import *" + assert st("#include <QtCore>") == "from PySide6 import QtCore" + assert st("#include <QLabel>") == "from PySide6.QtWidgets import QLabel" + assert st("#include <NotQt>") == "" + assert st('#include strange"') == "" + + +def test_main(): + assert st("int main(int argc, char *argv[])") == 'if __name__ == "__main__":' + + +def test_cast(): + assert st("a = reinterpret_cast<type>(data);") == "a = type(data)" + assert st("a = reinterpret_cast<type*>(data) * 9;") == "a = type(data) * 9" + assert ( + st("elapsed = (elapsed + qobject_cast<QTimer*>(sender())->interval()) % 1000;") + == "elapsed = (elapsed + QTimer(sender()).interval()) % 1000" + ) + assert ( + st("a = qobject_cast<type*>(data) * 9 + static_cast<int>(42)") + == "a = type(data) * 9 + int(42)" + ) + + +def test_double_colon(): + assert st("Qt::Align") == "Qt.Align" + assert st('QSound::play("mysounds/bells.wav");') == 'QSound.play("mysounds/bells.wav")' + assert st("Widget::method") == "Widget.method" + + # multiline statement connect statement + # eg: connect(reply, &QNetworkReply::errorOccurred, + # this, &MyClass::slotError); + assert st("this, &MyClass::slotError);") == "self.slotError)" + + +def test_connects(): + assert ( + st("connect(button, &QPushButton::clicked, this, &MyClass::slotClicked);") + == "button.clicked.connect(self.slotClicked)" + ) + assert ( + st("connect(m_ui->button, &QPushButton::clicked, this, &MyClass::slotClicked);") + == "m_ui.button.clicked.connect(self.slotClicked)" + ) + assert ( + st("connect(button.get(), &QPushButton::clicked, this, &MyClass::slotClicked);") + == "button.clicked.connect(self.slotClicked)" + ) + + +def test_cout_endl(): + assert st("cout << 'hello' << 'world' << endl") == "print('hello', 'world')" + assert st(" cout << 'hallo' << 'welt' << endl") == " print('hallo', 'welt')" + assert st("cout << 'hi'") == "print('hi')" + assert st("'world' << endl") == "print('world')" + + assert st("cout << circ.at(i) << endl;") == "print(circ.at(i))" + assert ( + st('cout << "Element name: " << qPrintable(e.tagName()) << "\n";') + == 'print("Element name: ", qPrintable(e.tagName()), "\n")' + ) + assert ( + st('cout << "First occurrence of Harumi is at position " << i << Qt::endl;') + == 'print("First occurrence of Harumi is at position ", i)' + ) + assert st('cout << "Found Jeanette" << endl;') == 'print("Found Jeanette")' + assert st('cout << "The key: " << it.key() << Qt::endl') == 'print("The key: ", it.key())' + assert ( + st("cout << (*constIterator).toLocal8Bit().constData() << Qt::endl;") + == "print((constIterator).toLocal8Bit().constData())" + ) + assert st("cout << ba[0]; // prints H") == "print(ba[0]) # prints H" + assert ( + st('cout << "Also the value: " << (*it) << Qt::endl;') == 'print("Also the value: ", (it))' + ) + assert st('cout << "[" << *data << "]" << Qt::endl;') == 'print("[", data, "]")' + + assert st('out << "Qt rocks!" << Qt::endl;') == 'print(out, "Qt rocks!")' + assert st(' std::cout << "MyObject::MyObject()\n";') == ' print("MyObject::MyObject()\n")' + assert st('qDebug() << "Retrieved:" << retrieved;') == 'print("Retrieved:", retrieved)' + + +def test_variable_declaration(): + assert st("QLabel label;") == "label = QLabel()" + assert st('QLabel label("Hello");') == 'label = QLabel("Hello")' + assert st("Widget w;") == "w = Widget()" + assert st('QLabel *label = new QLabel("Hello");') == 'label = QLabel("Hello")' + assert st('QLabel label = a_function("Hello");') == 'label = a_function("Hello")' + assert st('QString a = "something";') == 'a = "something"' + assert st("int var;") == "var = int()" + assert st("float v = 0.1;") == "v = 0.1" + assert st("QSome<thing> var;") == "var = QSome()" + assert st("QQueue<int> queue;") == "queue = QQueue()" + assert st("QVBoxLayout *layout = new QVBoxLayout;") == "layout = QVBoxLayout()" + assert st("QPointer<QLabel> label = new QLabel;") == "label = QLabel()" + assert st("QMatrix4x4 matrix;") == "matrix = QMatrix4x4()" + assert st("QList<QImage> collage =") == "collage =" + assert st("bool b = true;") == "b = True" + assert st("Q3DBars *m_graph = nullptr;") == "m_graph = None" + # Do not fall for member function definitions + assert st("Q3DBars *Graph::bars() const") == "Q3DBars Graph.bars()" + # Do not fall for member function declarations + assert st("virtual Q3DBars *bars();") == "virtual Q3DBars bars()" + + +def test_for(): + assert st("for (int i = 0; i < 10; i++)") == "for i in range(0, 10):" + assert st(" for (int i = 0; i < 10; i+=2)") == " for i in range(0, 10, 2):" + assert st("for (int i = 10; i >= 0; i-=2)") == "for i in range(-1, 10, -2):" + assert st("for (int i = 0; i < 10; ++i)") == "for i in range(0, 10):" + assert ( + st("for (int c = 0;" "c < model.columnCount();" "++c) {") + == "for c in range(0, model.columnCount()):" + ) + assert ( + st("for (int c = 0;" "c < table->columns();" "++c) {") + == "for c in range(0, table.columns()):" + ) + assert st("for (int i = 0; i <= 10; i++)") == "for i in range(0, 11):" + assert st("for (int i = 10; i >= 0; i--)") == "for i in range(-1, 10, -1):" + + ## if contains "begin()" and "end()", do a 'for it in var' + assert ( + st( + "for (QHash<int, QString>::const_iterator it = hash.cbegin()," + "end = hash.cend(); it != end; ++it)" + ) + == "for it in hash:" + ) + assert ( + st("for (QTextBlock it = doc->begin();" "it != doc->end(); it = it.next())") + == "for it in doc:" + ) + assert st("for (auto it = map.begin(); it != map.end(); ++it) {") == "for it in map:" + assert st("for (i = future.constBegin(); i != future.constEnd(); ++i)") == "for i in future:" + assert st("for (it = block.begin(); !(it.atEnd()); ++it) {") == "for it in block:" + assert ( + st(" for (it = snippetPaths.constBegin();" "it != snippetPaths.constEnd(); ++it)") + == " for it in snippetPaths:" + ) + + assert st("for (QChar ch : s)") == "for ch in s:" + assert ( + st("for (const QByteArray &ext : " "qAsConst(extensionList))") + == "for ext in extensionList:" + ) + assert st("for (QTreeWidgetItem *item : found) {") == "for item in found:" + + # TODO: Strange cases + # for ( ; it != end; ++it) { + # for (; !elt.isNull(); elt = elt.nextSiblingElement("entry")) { + # for (int i = 0; ids[i]; ++i) + # for (int i = 0; i < (1>>20); ++i) + # for(QDomNode n = element.firstChild(); !n.isNull(); n = n.nextSibling()) + + +def test_emit(): + assert st("emit sliderPressed();") == "sliderPressed.emit()" + assert st("emit actionTriggered(action);") == "actionTriggered.emit(action)" + assert st("emit activeChanged(d->m_active);") == "activeChanged.emit(d.m_active)" + assert st("emit dataChanged(index, index);") == "dataChanged.emit(index, index)" + assert st("emit dataChanged(index, index, {role});") == "dataChanged.emit(index, index, {role})" + assert ( + st('emit dragResult(tr("The data was copied here."));') + == 'dragResult.emit(tr("The data was copied here."))' + ) + assert ( + st("emit mimeTypes(event->mimeData()->formats());") + == "mimeTypes.emit(event.mimeData().formats())" + ) + assert ( + st("emit q_ptr->averageFrequencyChanged(m_averageFrequency);") + == "q_ptr.averageFrequencyChanged.emit(m_averageFrequency)" + ) + assert st("emit q_ptr->frequencyChanged();") == "q_ptr.frequencyChanged.emit()" + assert ( + st("emit rangeChanged(d->minimum, d->maximum);") + == "rangeChanged.emit(d.minimum, d.maximum)" + ) + assert ( + st("emit sliderMoved((d->position = value));") == "sliderMoved.emit((d.position = value))" + ) + assert ( + st("emit stateChanged(QContactAction::FinishedState);") + == "stateChanged.emit(QContactAction.FinishedState)" + ) + assert st("emit textCompleted(lineEdit->text());") == "textCompleted.emit(lineEdit.text())" + assert ( + st("emit updateProgress(newstat, m_watcher->progressMaximum());") + == "updateProgress.emit(newstat, m_watcher.progressMaximum())" + ) + + +def test_void_functions(): + assert st("void Something::Method(int a, char *b) {") == "def Method(self, a, b):" + assert ( + st("void MainWindow::updateMenus(QListWidgetItem *current)") + == "def updateMenus(self, current):" + ) + assert ( + st("void MyScrollArea::scrollContentsBy(int dx, int dy)") + == "def scrollContentsBy(self, dx, dy):" + ) + assert st("void Wrapper::wrapper6() {") == "def wrapper6(self):" + assert st("void MyClass::setPriority(Priority) {}") == "def setPriority(self, Priority): pass" + assert st("void MyException::raise() const { throw *this; }") == "def raise(self): raise self" + assert st("void tst_Skip::test_data()") == "def test_data(self):" + assert st("void util_function_does_nothing()") == "def util_function_does_nothing():" + assert st("static inline void cleanup(MyCustomClass *pointer)") == "def cleanup(pointer):" + # TODO: Which name? + assert st("void RenderWindow::exposeEvent(QExposeEvent *)") == "def exposeEvent(self, arg__0):" + + +def test_classes(): + assert st("class MyWidget //: public QWidget") == "class MyWidget(): #: public QWidget" + assert st("class MyMfcView : public CView") == "class MyMfcView(CView):" + assert st("class MyGame : public QObject {") == "class MyGame(QObject):" + assert st("class tst_Skip") == "class tst_Skip():" + assert st("class A : public B, protected C") == "class A(B, C):" + assert st("class A : public B, public C") == "class A(B, C):" + assert st("class SomeTemplate<int> : public QFrame") == "class SomeTemplate(QFrame):" + # This is a tricky situation because it has a multi line dependency: + # class MyMemberSheetExtension : public QObject, + # public QDesignerMemberSheetExtension + # { + # we will use the leading comma to trust it's the previously situation. + assert st("class A : public QObject,") == "class A(QObject," + assert st("class B {...};") == "class B(): pass" + + +def test_constuctors(): + assert st("MyWidget::MyWidget(QWidget *parent)") == "def __init__(self, parent):" + assert st("Window::Window()") == "def __init__(self):" + + +def test_inheritance_init(): + assert ( + st(": QClass(fun(re, 1, 2), parent), a(1)") + == " super().__init__(fun(re, 1, 2), parent)\n self.a = 1" + ) + assert ( + st(": QQmlNdefRecord(copyFooRecord(record), parent)") + == " super().__init__(copyFooRecord(record), parent)" + ) + assert ( + st(" : QWidget(parent), helper(helper)") + == " super().__init__(parent)\n self.helper = helper" + ) + assert st(" : QWidget(parent)") == " super().__init__(parent)" + assert ( + st(": a(0), bB(99), cC2(1), p_S(10),") + == " self.a = 0\n self.bB = 99\n self.cC2 = 1\n self.p_S = 10" + ) + assert ( + st(": QAbstractFileEngineIterator(nameFilters, filters), index(0) ") + == " super().__init__(nameFilters, filters)\n self.index = 0" + ) + assert ( + st(": m_document(doc), m_text(text)") == " self.m_document = doc\n self.m_text = text" + ) + assert st(": m_size(size) { }") == " self.m_size = size" + assert ( + st(": option->palette.color(QPalette::Mid);") + == " self.option.palette.color = QPalette.Mid" + ) + assert st(": QSqlResult(driver) {}") == " super().__init__(driver)" + + +def test_arrays(): + assert st("static const GLfloat vertices[] = {") == "vertices = {" + assert st("static const char *greeting_strings[] = {") == "greeting_strings = {" + assert st("uchar arrow_bits[] = {0x3f, 0x1f, 0x0f}") == "arrow_bits = {0x3f, 0x1f, 0x0f}" + assert st("QList<int> vector { 1, 2, 3, 4 };") == "vector = { 1, 2, 3, 4 }" + + +def test_functions(): + assert st("int Class::method(a, b, c)") == "def method(self, a, b, c):" + assert st("QStringView Message::body() const") == "def body(self):" + assert st("void Ren::exEvent(QExp *)") == "def exEvent(self, arg__0):" + assert ( + st("QString myDecoderFunc(const QByteArray &localFileName);") + == "def myDecoderFunc(localFileName):" + ) + assert st("return QModelIndex();") == "return QModelIndex()" + + +def test_foreach(): + assert st("foreach (item, selected) {") == "for item in selected:" + assert st("foreach (const QVariant &v, iterable) {") == "for v in iterable:" + assert st("foreach (QObject *obj, list)") == "for obj in list:" + assert ( + st("foreach (const QContactTag& tag, contact.details<QContactTag>()) {") + == "for tag in contact.details():" + ) + + +def test_structs(): + assert st("struct ScopedPointerCustomDeleter") == "class ScopedPointerCustomDeleter():" + assert st("struct Wrapper : public QWidget {") == "class Wrapper(QWidget):" + assert st("struct Window {") == "class Window():" + + +def test_ternary_operator(): + assert st("int a = 1 ? b > 0 : 3") == "a = 1 if b > 0 else 3" + assert ( + st("if (!game.saveGame(json ? Game::Json : Game::Binary))") + == "if not game.saveGame(json if Game.Json else Game.Binary):" + ) + + +def test_useless_qt_classes(): + assert st('result += QLatin1String("; ");') == 'result += "; "' + assert st('result += QString::fromLatin1("; ");') == 'result += "; "' + assert ( + st('result = QStringLiteral("A") + QStringLiteral("B");') + == 'result = "A" + "B"') + assert st("<< QLatin1Char('\0') << endl;") == "print('\0')" + assert st('result = u"A"_s;') == 'result = "A"' + + +def test_special_cases(): + assert ( + st('http->setProxy("proxy.example.com", 3128);') + == 'http.setProxy("proxy.example.com", 3128)' + ) + assert st("delete something;") == "del something" + assert ( + st("m_program->setUniformValue(m_matrixUniform, matrix);") + == "m_program.setUniformValue(m_matrixUniform, matrix)" + ) + assert ( + st("QObject::connect(&window1, &Window::messageSent,") + == "window1.messageSent.connect(" + ) + assert st("double num;") == "num = float()" + + # Leave a comment to remember it comes from C++ + assert st("public:") == "# public" + assert st("private:") == "# private" + + #iterator declaration + assert st("std::vector<int>::iterator i;") == "" + + # TODO: Handle the existing ones with Python equivalents + # assert st("std::...") + + # FIXME: Maybe a better interpretation? + # assert st("QDebug operator<<(QDebug dbg, const Message &message)") == "def __str__(self):" + + # TODO: Maybe play with the format? + # assert st('m_o.append(tr("version: %1.%2").arg(a).arg(b))') == 'm_o.append(tr("version: {1}.{2}".format(a, b)' + + +def test_lambdas(): + # QtConcurrent::blockingMap(vector, [](int &x) { x *= 2; }); + + # QList<QImage> collage = QtConcurrent::mappedReduced(images, + # [&size](const QImage &image) { + # return image.scaled(size, size); + # }, + # addToCollage + # ).results(); + pass + + +def test_switch_case(): + source = """switch (v) { +case 1: + f1(); + break; +case ClassName::EnumValue: + f2(); + break; +default: + f3(); + break; +} +""" + expected = """ +if v == 1: + f1() + break +elif v == ClassName.EnumValue: + f2() + break +else: + f3() + break + +""" + + assert multi_st(source) == expected + + +def test_std_function(): + # std::function<QImage(const QImage &)> scale = [](const QImage &img) { + pass diff --git a/tools/snippets_translate/tests/test_snippets.py b/tools/snippets_translate/tests/test_snippets.py new file mode 100644 index 000000000..84897d815 --- /dev/null +++ b/tools/snippets_translate/tests/test_snippets.py @@ -0,0 +1,134 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +from main import _get_snippets, get_snippet_ids, CPP_SNIPPET_PATTERN + + +C_COMMENT = "//" + + +def test_stacking(): + lines = [ + "//! [A] //! [B] ", + "//! [C] //! [D] //! [E]", + "// Content", + "//! [C] //! [A] ", + "//! [B] //! [D] //! [E]", + ] + snippets = _get_snippets(lines, C_COMMENT, CPP_SNIPPET_PATTERN) + assert len(snippets) == 5 + + snippet_a = snippets["A"] + assert len(snippet_a) == 4 # A starts at line 0 and ends at line 3 + + snippet_b = snippets["B"] + assert len(snippet_b) == 5 # B starts at line 0 and ends at line 4 + + snippet_c = snippets["C"] + assert len(snippet_c) == 3 # C starts at line 1 and ends at line 3 + + snippet_d = snippets["D"] + assert len(snippet_d) == 4 # D starts at line 1 and ends at line 4 + + snippet_e = snippets["E"] + assert len(snippet_e) == 4 # E starts at line 1 and ends at line 4 + + +def test_nesting(): + lines = [ + "//! [A]", + "//! [B]", + "//! [C]", + "// Content", + "//! [A]", + "//! [C]", + "//! [B]", + ] + snippets = _get_snippets(lines, C_COMMENT, CPP_SNIPPET_PATTERN) + assert len(snippets) == 3 + + snippet_a = snippets["A"] + assert len(snippet_a) == 5 + assert snippet_a == lines[:5] + + snippet_b = snippets["B"] + assert len(snippet_b) == 6 + assert snippet_b == lines[1:] + + snippet_c = snippets["C"] + assert len(snippet_c) == 4 + assert snippet_c == lines[2:6] + + +def test_overlapping(): + a_id = "//! [A]" + b_id = "//! [B]" + lines = [ + "pretext", + a_id, + "l1", + "//! [C]", + "//! [A] //! [B]", + "l2", + "l3 // Comment", + b_id, + "posttext", + "//! [C]", + ] + snippets = _get_snippets(lines, C_COMMENT, CPP_SNIPPET_PATTERN) + assert len(snippets) == 3 + + # Simple snippet ID lines are generated + snippet_a = snippets["A"] + assert len(snippet_a) == 4 + assert snippet_a == lines[1:4] + [a_id] + + snippet_c = snippets["C"] + assert len(snippet_c) == 7 + assert snippet_c == lines[3:] + + snippet_b = snippets["B"] + assert len(snippet_b) == 4 + assert snippet_b == [b_id] + lines[5:8] + + +def test_snippets(): + a_id = "//! [A]" + b_id = "//! [B]" + + lines = [ + "pretext", + a_id, + "l1", + "//! [A] //! [B]", + "l2", + "l3 // Comment", + b_id, + "posttext" + ] + + snippets = _get_snippets(lines, C_COMMENT, CPP_SNIPPET_PATTERN) + assert len(snippets) == 2 + + snippet_a = snippets["A"] + + assert len(snippet_a) == 3 + assert snippet_a == lines[1:3] + [a_id] + + snippet_b = snippets["B"] + assert len(snippet_b) == 4 + assert snippet_b == [b_id] + lines[4:7] + + +def test_snippet_ids(): + assert get_snippet_ids("", CPP_SNIPPET_PATTERN) == [] + assert get_snippet_ids("//! ", + CPP_SNIPPET_PATTERN) == [] # Invalid id + assert get_snippet_ids("//! [some name]", + CPP_SNIPPET_PATTERN) == ["some name"] + assert get_snippet_ids("//! [some name] [some other name]", + CPP_SNIPPET_PATTERN) == ["some name"] + assert get_snippet_ids("//! [some name] //! ", + CPP_SNIPPET_PATTERN) == ["some name"] # Invalid id + assert get_snippet_ids("//! [some name] //! [some other name]", + CPP_SNIPPET_PATTERN) == ["some name", "some other name"] diff --git a/tools/uic_test.py b/tools/uic_test.py new file mode 100644 index 000000000..208536963 --- /dev/null +++ b/tools/uic_test.py @@ -0,0 +1,86 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import os +import re +import subprocess +import sys +import tempfile +from argparse import ArgumentParser, RawTextHelpFormatter +from pathlib import Path +from textwrap import dedent +from typing import Optional, Tuple + +VERSION = 6 + + +DESC = """Runs uic on a set of UI files and displays the resulting widgets.""" + + +TEMP_DIR = Path(tempfile.gettempdir()) + + +def get_class_name(file: Path) -> Tuple[Optional[str], Optional[str]]: + """Return class name and widget name of UI file.""" + pattern = re.compile(r'^\s*<widget class="(\w+)" name="(\w+)"\s*>.*$') + for line in Path(file).read_text().splitlines(): + match = pattern.match(line) + if match: + return (match.group(1), match.group(2)) + return (None, None) + + +def test_file(file: str, uic: bool = False) -> bool: + """Run uic on a UI file and show the resulting UI.""" + path = Path(file) + (klass, name) = get_class_name(path) + if not klass: + print(f'{file} does not appear to be a UI file', file=sys.stderr) + return False + py_klass = f'Ui_{name}' + py_file_basename = py_klass.lower() + py_file = TEMP_DIR / (py_file_basename + '.py') + py_main = TEMP_DIR / 'main.py' + cmd = ['uic', '-g', 'python'] if uic else [f'pyside{VERSION}-uic'] + cmd.extend(['-o', os.fspath(py_file), file]) + try: + subprocess.call(cmd) + except FileNotFoundError as e: + print(str(e) + " (try -u for uic)", file=sys.stderr) + return False + main_source = dedent(f'''\ + import sys + from PySide{VERSION}.QtWidgets import QApplication, {klass} + from {py_file_basename} import {py_klass} + + if __name__ == "__main__": + app = QApplication(sys.argv) + ui = {py_klass}() + widget = {klass}() + ui.setupUi(widget) + widget.show() + sys.exit(app.exec())''') + py_main.write_text(main_source) + exit_code = subprocess.call([sys.executable, os.fspath(py_main)]) + py_main.unlink() + py_file.unlink() + return exit_code == 0 + + +if __name__ == '__main__': + argument_parser = ArgumentParser(description=DESC, + formatter_class=RawTextHelpFormatter) + argument_parser.add_argument('--uic', '-u', action='store_true', + help='Use uic instead of pyside-uic') + argument_parser.add_argument("files", help="UI Files", + nargs='+', type=str) + options = argument_parser.parse_args() + failed = 0 + count = len(options.files) + for i, file in enumerate(options.files): + print(f'{i+1}/{count} {file}') + if not test_file(file, options.uic): + failed += 1 + if failed != 0: + print(f'{failed}/{count} failed.') + sys.exit(failed) |