diff options
Diffstat (limited to 'tools')
43 files changed, 3163 insertions, 2652 deletions
diff --git a/tools/checklibs.py b/tools/checklibs.py index 18aa11e93..9a53beade 100644 --- a/tools/checklibs.py +++ b/tools/checklibs.py @@ -1,41 +1,5 @@ -############################################################################# -## -## Copyright (C) 2017 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #!/usr/bin/env python # @@ -49,7 +13,12 @@ # # -import subprocess, sys, re, os.path, optparse, collections +import collections +import optparse +import re +import subprocess +import sys +from pathlib import Path from pprint import pprint @@ -214,23 +183,23 @@ class MachOFile: if recorded_path.startswith(ImagePath.EXECUTABLE_PATH_TOKEN): executable_image_path = self.executable_path() if executable_image_path: - path.resolved_path = os.path.normpath( + path.resolved_path = Path( recorded_path.replace( ImagePath.EXECUTABLE_PATH_TOKEN, - os.path.dirname(executable_image_path.resolved_path))) + Path(executable_image_path.resolved_path).parent)) # handle @loader_path elif recorded_path.startswith(ImagePath.LOADER_PATH_TOKEN): - path.resolved_path = os.path.normpath(recorded_path.replace( + path.resolved_path = Path(recorded_path.replace( ImagePath.LOADER_PATH_TOKEN, - os.path.dirname(self.image_path.resolved_path))) + Path(self.image_path.resolved_path).parent)) # handle @rpath elif recorded_path.startswith(ImagePath.RPATH_TOKEN): for rpath in self.all_rpaths(): - resolved_path = os.path.normpath(recorded_path.replace( + resolved_path = Path(recorded_path.replace( ImagePath.RPATH_TOKEN, rpath.resolved_path)) - if os.path.exists(resolved_path): + if resolved_path.exists(): path.resolved_path = resolved_path path.rpath_source = rpath.rpath_source break @@ -333,7 +302,7 @@ class ImagePath: return description def exists(self): - return self.resolved_path and os.path.exists(self.resolved_path) + return self.resolved_path and Path(self.resolved_path).exists() def resolved_equals_recorded(self): return (self.resolved_path and self.recorded_path and diff --git a/tools/create_changelog.py b/tools/create_changelog.py index b93d16ae2..6c24f417f 100644 --- a/tools/create_changelog.py +++ b/tools/create_changelog.py @@ -1,46 +1,13 @@ -############################################################################# -## -## Copyright (C) 2019 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import re +import os import sys +import textwrap from argparse import ArgumentParser, Namespace, RawTextHelpFormatter -from subprocess import check_output, Popen, PIPE +from pathlib import Path +from subprocess import PIPE, Popen, check_output from typing import Dict, List, Tuple content_header = """Qt for Python @VERSION is a @TYPE release. @@ -68,6 +35,48 @@ shiboken_header = """*********************************************************** **************************************************************************** """ +description = """ +PySide6 changelog tool + +Example usage: +tools/create_changelog.py -v -r 6.5.3 +""" + + +def change_log(version: list) -> Path: + """Return path of the changelog of the version.""" + name = f"changes-{version[0]}.{version[1]}.{version[2]}" + return Path(__file__).parents[1] / "doc" / "changelogs" / name + + +def is_lts_version(version: list) -> bool: + return version[0] == 5 or version[1] in (2, 5) + + +def version_tag(version: list) -> str: + """Return the version tag.""" + tag = f"v{version[0]}.{version[1]}.{version[2]}" + return tag + "-lts" if is_lts_version(version) else tag + + +def revision_range(version: list) -> str: + """Determine a git revision_range from the version. Either log from + the previous version tag or since the last update to the changelog.""" + changelog = change_log(version) + if changelog.is_file(): + output = check_output(["git", "log", "-n", "1", "--format=%H", + os.fspath(changelog)]) + if output: + return output.strip().decode("utf-8") + "..HEAD" + + last_version = version.copy() + if version[2] == 0: + adjust_idx = 0 if version[1] == 0 else 1 + else: + adjust_idx = 2 + last_version[adjust_idx] -= 1 + return version_tag(last_version) + "..HEAD" + def parse_options() -> Namespace: tag_msg = ("Tags, branches, or SHA to compare\n" @@ -75,7 +84,7 @@ def parse_options() -> Namespace: " v5.12.0..v5.12.1\n" " cebc32a5..5.12") - options = ArgumentParser(description="PySide6 changelog tool", + options = ArgumentParser(description=description, formatter_class=RawTextHelpFormatter) options.add_argument("-d", "--directory", @@ -84,8 +93,7 @@ def parse_options() -> Namespace: options.add_argument("-v", "--versions", type=str, - help=tag_msg, - required=True) + help=tag_msg) options.add_argument("-r", "--release", type=str, @@ -94,8 +102,7 @@ def parse_options() -> Namespace: options.add_argument("-t", "--type", type=str, - help="Release type: bug-fix, minor, or major", - default="bug-fix") + help="Release type: bug-fix, minor, or major") options.add_argument("-e", "--exclude", @@ -104,14 +111,43 @@ def parse_options() -> Namespace: default=False) args = options.parse_args() + + release_version = list(int(v) for v in args.release.split(".")) + if len(release_version) != 3: + print("Error: --release must be of form major.minor.patch") + sys.exit(-1) + + # Some auto-detection smartness + if not args.type: + if release_version[2] == 0: + args.type = "major" if release_version[1] == 0 else "minor" + else: + args.type = "bug-fix" + # For major/minor releases, skip all fixes with "Pick-to: " since they + # appear in bug-fix releases. + if args.type != "bug-fix": + args.exclude = True + print(f'Assuming "{args.type}" version', file=sys.stderr) + if args.type not in ("bug-fix", "minor", "major"): - print("Error:" + print("Error: " "-y/--type needs to be: bug-fix (default), minor, or major") sys.exit(-1) + if not args.versions: + args.versions = revision_range(release_version) + print(f"Assuming range {args.versions}", file=sys.stderr) + + args.release_version = release_version return args +def format_text(text: str) -> str: + """Format an entry with a leading dash, 80 columns""" + return textwrap.fill(text, width=77, initial_indent=" - ", + subsequent_indent=" ") + + def check_tag(tag: str) -> bool: output = False @@ -146,6 +182,7 @@ def get_commit_content(sha: str) -> str: print(err, file=sys.stderr) return out.decode("utf-8") + def git_get_sha1s(versions: List[str], pattern: str): """Return a list of SHA1s matching a pattern""" command = "git rev-list --reverse --grep '^{}'".format(pattern) @@ -173,7 +210,6 @@ def git_get_sha1s(versions: List[str], pattern: str): print(err, file=sys.stderr) pick_to_sha1 = out_e_sha1.splitlines() - return [s.decode("utf-8") for s in out_sha1.splitlines() if s not in pick_to_sha1] @@ -213,18 +249,19 @@ def create_task_log(versions: List[str]) -> None: git_command(versions, "Task-number: ") -def extract_change_log(commit_message: List[str]) -> Tuple[str, List[str]]: - """Extract a tuple of (component, change log lines) from a commit message - of the form [ChangeLog][shiboken6] description...""" - result = [] +def extract_change_log(commit_message: List[str]) -> Tuple[str, int, str]: + """Extract a tuple of (component, task-number, change log paragraph) + from a commit message of the form [ChangeLog][shiboken6] description...""" + result = '' component = 'pyside' within_changelog = False + task_nr = '' for line in commit_message: if within_changelog: if line: - result.append(' ' + line.strip()) + result += ' ' + line.strip() else: - break + within_changelog = False else: if line.startswith('[ChangeLog]'): log_line = line[11:] @@ -233,38 +270,61 @@ def extract_change_log(commit_message: List[str]) -> Tuple[str, List[str]]: if end > 0: component = log_line[1:end] log_line = log_line[end + 1:] - result.append(' - ' + log_line.strip()) + result = log_line.strip() within_changelog = True - return (component, result) + elif line.startswith("Fixes: ") or line.startswith("Task-number: "): + task_nr = line.split(":")[1].strip() + + task_nr_int = -1 + if task_nr: + result = f"[{task_nr}] {result}" + dash = task_nr.find('-') # "PYSIDE-627" + task_nr_int = int(task_nr[dash + 1:]) + + return (component, task_nr_int, format_text(result)) def create_change_log(versions: List[str]) -> None: for sha in git_get_sha1s(versions, r"\[ChangeLog\]"): change_log = extract_change_log(get_commit_content(sha).splitlines()) - if change_log[0].startswith('shiboken'): - shiboken6_changelogs.extend(change_log[1]) + component, task_nr, text = change_log + if component.startswith('shiboken'): + shiboken6_changelogs.append((task_nr, text)) else: - pyside6_changelogs.extend(change_log[1]) + pyside6_changelogs.append((task_nr, text)) + + +def format_commit_msg(entry: Dict[str, str]) -> str: + task = entry["task"].replace("Fixes: ", "").replace("Task-number: ", "") + title = entry["title"] + if title.startswith("shiboken6: "): + title = title[11:] + elif title.startswith("PySide6: "): + title = title[9:] + return format_text(f"[{task}] {title}") def gen_list(d: Dict[str, Dict[str, str]]) -> str: - def clean_task(s): - return s.replace("Fixes: ", "").replace("Task-number: ", "") - return "".join(" - [{}] {}\n".format(clean_task(v["task"]), v["title"]) - for _, v in d.items()) + return "\n".join(format_commit_msg(v) + for _, v in d.items()) def sort_dict(d: Dict[str, Dict[str, str]]) -> Dict[str, Dict[str, str]]: return dict(sorted(d.items(), key=lambda kv: kv[1]['task-number'])) +def sort_changelog(c: List[Tuple[int, str]]) -> List[Tuple[int, str]]: + return sorted(c, key=lambda task_text_tuple: task_text_tuple[0]) + + if __name__ == "__main__": args = parse_options() pyside6_commits: Dict[str, Dict[str, str]] = {} shiboken6_commits: Dict[str, Dict[str, str]] = {} - pyside6_changelogs: List[str] = [] - shiboken6_changelogs: List[str] = [] + # Changelogs are tuples of task number/formatted text + pyside6_changelogs: List[Tuple[int, str]] = [] + shiboken6_changelogs: List[Tuple[int, str]] = [] exclude_pick_to = args.exclude @@ -280,16 +340,20 @@ if __name__ == "__main__": # Sort commits pyside6_commits = sort_dict(pyside6_commits) shiboken6_commits = sort_dict(shiboken6_commits) + pyside6_changelogs = sort_changelog(pyside6_changelogs) + shiboken6_changelogs = sort_changelog(shiboken6_changelogs) # Generate message print(content_header.replace("@VERSION", args.release). replace("@TYPE", args.type)) - print('\n'.join(pyside6_changelogs)) + for c in pyside6_changelogs: + print(c[1]) print(gen_list(pyside6_commits)) if not pyside6_changelogs and not pyside6_commits: print(" - No changes") print(shiboken_header) - print('\n'.join(shiboken6_changelogs)) + for c in shiboken6_changelogs: + print(c[1]) print(gen_list(shiboken6_commits)) if not shiboken6_changelogs and not shiboken6_commits: print(" - No changes") diff --git a/tools/cross_compile_android/android_utilities.py b/tools/cross_compile_android/android_utilities.py new file mode 100644 index 000000000..039fa9431 --- /dev/null +++ b/tools/cross_compile_android/android_utilities.py @@ -0,0 +1,256 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import logging +import shutil +import os +import stat +import sys +import subprocess + +from urllib import request +from pathlib import Path +from typing import List +from packaging import version +from tqdm import tqdm + +# the tag number does not matter much since we update the sdk later +DEFAULT_SDK_TAG = 6514223 +ANDROID_NDK_VERSION = "26b" + + +def run_command(command: List[str], cwd: str = None, ignore_fail: bool = False, + dry_run: bool = False, accept_prompts: bool = False, show_stdout: bool = False, + capture_stdout: bool = False): + + if capture_stdout and not show_stdout: + raise RuntimeError("capture_stdout should always be used together with show_stdout") + + if dry_run: + print(" ".join(command)) + return + + input = None + if accept_prompts: + input = str.encode("y") + + if show_stdout: + stdout = None + else: + stdout = subprocess.DEVNULL + + result = subprocess.run(command, cwd=cwd, input=input, stdout=stdout, + capture_output=capture_stdout) + + if result.returncode != 0 and not ignore_fail: + sys.exit(result.returncode) + + if capture_stdout and not result.returncode: + return result.stdout.decode("utf-8") + + return None + + +class DownloadProgressBar(tqdm): + def update_to(self, b=1, bsize=1, tsize=None): + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) + + +class SdkManager: + def __init__(self, android_sdk_dir: Path, dry_run: bool = False): + self._sdk_manager = android_sdk_dir / "tools" / "bin" / "sdkmanager" + + if not self._sdk_manager.exists(): + raise RuntimeError(f"Unable to find SdkManager in {str(self._sdk_manager)}") + + if not os.access(self._sdk_manager, os.X_OK): + current_permissions = stat.S_IMODE(os.lstat(self._sdk_manager).st_mode) + os.chmod(self._sdk_manager, current_permissions | stat.S_IEXEC) + + self._android_sdk_dir = android_sdk_dir + self._dry_run = dry_run + + def list_packages(self): + command = [self._sdk_manager, f"--sdk_root={self._android_sdk_dir}", "--list"] + return run_command(command=command, dry_run=self._dry_run, show_stdout=True, + capture_stdout=True) + + def install(self, *args, accept_license: bool = False, show_stdout=False): + command = [str(self._sdk_manager), f"--sdk_root={self._android_sdk_dir}", *args] + run_command(command=command, dry_run=self._dry_run, + accept_prompts=accept_license, show_stdout=show_stdout) + + +def _unpack(zip_file: Path, destination: Path): + """ + Unpacks the zip_file into destination preserving all permissions + + TODO: Try to use zipfile module. Currently we cannot use zipfile module here because + extractAll() does not preserve permissions. + + In case `unzip` is not available, the user is requested to install it manually + """ + unzip = shutil.which("unzip") + if not unzip: + raise RuntimeError("Unable to find program unzip. Use `sudo apt-get install unzip`" + "to install it") + + command = [unzip, zip_file, "-d", destination] + run_command(command=command, show_stdout=True) + + +def _download(url: str, destination: Path): + """ + Download url to destination + """ + headers, download_path = None, None + # https://github.com/tqdm/tqdm#hooks-and-callbacks + with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: + download_path, headers = request.urlretrieve(url=url, filename=destination, + reporthook=t.update_to) + assert headers["Content-Type"] == "application/zip" + assert Path(download_path).resolve() == destination + + +def download_android_ndk(ndk_path: Path): + """ + Downloads the given ndk_version into ndk_path + """ + ndk_path = ndk_path / "android-ndk" + ndk_zip_path = ndk_path / f"android-ndk-r{ANDROID_NDK_VERSION}-linux.zip" + ndk_version_path = ndk_path / f"android-ndk-r{ANDROID_NDK_VERSION}" + + if ndk_version_path.exists(): + print(f"NDK path found in {str(ndk_version_path)}") + else: + ndk_path.mkdir(parents=True, exist_ok=True) + url = (f"https://dl.google.com/android/repository" + f"/android-ndk-r{ANDROID_NDK_VERSION}-linux.zip") + + print(f"Downloading Android Ndk version r{ANDROID_NDK_VERSION}") + _download(url=url, destination=ndk_zip_path) + + print("Unpacking Android Ndk") + _unpack(zip_file=(ndk_path / f"android-ndk-r{ANDROID_NDK_VERSION}-linux.zip"), + destination=ndk_path) + + return ndk_version_path + + +def download_android_commandlinetools(android_sdk_dir: Path): + """ + Downloads Android commandline tools into cltools_path. + """ + android_sdk_dir = android_sdk_dir / "android-sdk" + url = ("https://dl.google.com/android/repository/" + f"commandlinetools-linux-{DEFAULT_SDK_TAG}_latest.zip") + cltools_zip_path = android_sdk_dir / f"commandlinetools-linux-{DEFAULT_SDK_TAG}_latest.zip" + cltools_path = android_sdk_dir / "tools" + + if cltools_path.exists(): + print(f"Command-line tools found in {str(cltools_path)}") + else: + android_sdk_dir.mkdir(parents=True, exist_ok=True) + + print("Download Android Command Line Tools: " + f"commandlinetools-linux-{DEFAULT_SDK_TAG}_latest.zip") + _download(url=url, destination=cltools_zip_path) + + print("Unpacking Android Command Line Tools") + _unpack(zip_file=cltools_zip_path, destination=android_sdk_dir) + + return android_sdk_dir + + +def android_list_build_tools_versions(sdk_manager: SdkManager): + """ + List all the build-tools versions available for download + """ + available_packages = sdk_manager.list_packages() + build_tools_versions = [] + lines = available_packages.split('\n') + + for line in lines: + if not line.strip().startswith('build-tools;'): + continue + package_name = line.strip().split(' ')[0] + if package_name.count(';') != 1: + raise RuntimeError(f"Unable to parse build-tools version: {package_name}") + ver = package_name.split(';')[1] + + build_tools_versions.append(version.Version(ver)) + + return build_tools_versions + + +def find_installed_buildtools_version(build_tools_dir: Path): + """ + It is possible that the user has multiple build-tools installed. The newer version is generally + used. This function find the newest among the installed build-tools + """ + versions = [version.Version(bt_dir.name) for bt_dir in build_tools_dir.iterdir() + if bt_dir.is_dir()] + return max(versions) + + +def find_latest_buildtools_version(sdk_manager: SdkManager): + """ + Uses sdk manager to find the latest build-tools version + """ + available_build_tools_v = android_list_build_tools_versions(sdk_manager=sdk_manager) + + if not available_build_tools_v: + raise RuntimeError('Unable to find any build tools available for download') + + return max(available_build_tools_v) + + +def install_android_packages(android_sdk_dir: Path, android_api: str, dry_run: bool = False, + accept_license: bool = False, skip_update: bool = False): + """ + Use the sdk manager to install build-tools, platform-tools and platform API + """ + tools_dir = android_sdk_dir / "tools" + if not tools_dir.exists(): + raise RuntimeError("Unable to find Android command-line tools in " + f"{str(tools_dir)}") + + # incase of --verbose flag + show_output = (logging.getLogger().getEffectiveLevel() == logging.INFO) + + sdk_manager = SdkManager(android_sdk_dir=android_sdk_dir, dry_run=dry_run) + + # install/upgrade platform-tools + if not (android_sdk_dir / "platform-tools").exists(): + print("Installing/Updating Android platform-tools") + sdk_manager.install("platform-tools", accept_license=accept_license, + show_stdout=show_output) + # The --update command is only relevant for platform tools + if not skip_update: + sdk_manager.install("--update", show_stdout=show_output) + + # install/upgrade build-tools + buildtools_dir = android_sdk_dir / "build-tools" + + if not buildtools_dir.exists(): + latest_build_tools_v = find_latest_buildtools_version(sdk_manager=sdk_manager) + print(f"Installing Android build-tools version {latest_build_tools_v}") + sdk_manager.install(f"build-tools;{latest_build_tools_v}", show_stdout=show_output) + else: + if not skip_update: + latest_build_tools_v = find_latest_buildtools_version(sdk_manager=sdk_manager) + installed_build_tools_v = find_installed_buildtools_version(buildtools_dir) + if latest_build_tools_v > installed_build_tools_v: + print(f"Updating Android build-tools version to {latest_build_tools_v}") + sdk_manager.install(f"build-tools;{latest_build_tools_v}", show_stdout=show_output) + installed_build_tools_v = latest_build_tools_v + + # install the platform API + platform_api_dir = android_sdk_dir / "platforms" / f"android-{android_api}" + if not platform_api_dir.exists(): + print(f"Installing Android platform API {android_api}") + sdk_manager.install(f"platforms;android-{android_api}", show_stdout=show_output) + + print("Android packages installation done") diff --git a/tools/cross_compile_android/main.py b/tools/cross_compile_android/main.py new file mode 100644 index 000000000..b68fd5031 --- /dev/null +++ b/tools/cross_compile_android/main.py @@ -0,0 +1,308 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import sys +import logging +import argparse +import stat +import warnings +import shutil +from dataclasses import dataclass + +from pathlib import Path +from git import Repo, RemoteProgress +from tqdm import tqdm +from jinja2 import Environment, FileSystemLoader + +from android_utilities import (run_command, download_android_commandlinetools, + download_android_ndk, install_android_packages) + +# Note: Does not work with PyEnv. Your Host Python should contain openssl. +PYTHON_VERSION = "3.11" + +SKIP_UPDATE_HELP = ("skip the updation of SDK packages build-tools, platform-tools to" + " latest version") + +ACCEPT_LICENSE_HELP = (''' +Accepts license automatically for Android SDK installation. Otherwise, +accept the license manually through command line. +''') + +CLEAN_CACHE_HELP = (''' +Cleans cache stored in $HOME/.pyside6_deploy_cache. +Options: + +1. all - all the cache including Android Ndk, Android Sdk and Cross-compiled Python are deleted. +2. ndk - Only the Android Ndk is deleted. +3. sdk - Only the Android Sdk is deleted. +4. python - The cross compiled Python for all platforms, the cloned CPython, the cross compilation + scripts for all platforms are deleted. +5. toolchain - The CMake toolchain file required for cross-compiling Qt for Python, for all + platforms are deleted. + +If --clean-cache is used and no explicit value is suppied, then `all` is used as default. +''') + + +@dataclass +class PlatformData: + plat_name: str + api_level: str + android_abi: str + qt_plat_name: str + gcc_march: str + plat_bits: str + + +def occp_exists(): + ''' + check if '--only-cross-compile-python' exists in command line arguments + ''' + return "-occp" in sys.argv or "--only-cross-compile-python" in sys.argv + + +class CloneProgress(RemoteProgress): + def __init__(self): + super().__init__() + self.pbar = tqdm() + + def update(self, op_code, cur_count, max_count=None, message=""): + self.pbar.total = max_count + self.pbar.n = cur_count + self.pbar.refresh() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="This tool cross builds CPython for Android and uses that Python to cross build" + "Android Qt for Python wheels", + formatter_class=argparse.RawTextHelpFormatter, + ) + + parser.add_argument("-p", "--plat-name", type=str, nargs="*", + choices=["aarch64", "armv7a", "i686", "x86_64"], + default=["aarch64", "armv7a", "i686", "x86_64"], dest="plat_names", + help="Android target platforms") + + parser.add_argument("-v", "--verbose", help="run in verbose mode", action="store_const", + dest="loglevel", const=logging.INFO) + parser.add_argument("--api-level", type=str, default="26", + help="Minimum Android API level to use") + parser.add_argument("--ndk-path", type=str, help="Path to Android NDK (Preferred r25c)") + # sdk path is needed to compile all the Qt Java Acitivity files into Qt6AndroidBindings.jar + parser.add_argument("--sdk-path", type=str, help="Path to Android SDK") + parser.add_argument("--qt-install-path", type=str, required=not occp_exists(), + help="Qt installation path eg: /home/Qt/6.5.0") + + parser.add_argument("-occp", "--only-cross-compile-python", action="store_true", + help="Only cross compiles Python for the specified Android platform") + + parser.add_argument("--dry-run", action="store_true", help="show the commands to be run") + + parser.add_argument("--skip-update", action="store_true", + help=SKIP_UPDATE_HELP) + + parser.add_argument("--auto-accept-license", action="store_true", + help=ACCEPT_LICENSE_HELP) + + parser.add_argument("--clean-cache", type=str, nargs="?", const="all", + choices=["all", "python", "ndk", "sdk", "toolchain"], + help=CLEAN_CACHE_HELP) + + args = parser.parse_args() + + logging.basicConfig(level=args.loglevel) + pyside_setup_dir = Path(__file__).parents[2].resolve() + qt_install_path = args.qt_install_path + ndk_path = args.ndk_path + sdk_path = args.sdk_path + only_py_cross_compile = args.only_cross_compile_python + android_abi = None + gcc_march = None + plat_bits = None + dry_run = args.dry_run + plat_names = args.plat_names + api_level = args.api_level + skip_update = args.skip_update + auto_accept_license = args.auto_accept_license + clean_cache = args.clean_cache + + # auto download Android NDK and SDK + pyside6_deploy_cache = Path.home() / ".pyside6_android_deploy" + logging.info(f"Cache created at {str(pyside6_deploy_cache.resolve())}") + pyside6_deploy_cache.mkdir(exist_ok=True) + + if pyside6_deploy_cache.exists() and clean_cache: + if clean_cache == "all": + shutil.rmtree(pyside6_deploy_cache) + elif clean_cache == "ndk": + cached_ndk_dir = pyside6_deploy_cache / "android-ndk" + if cached_ndk_dir.exists(): + shutil.rmtree(cached_ndk_dir) + elif clean_cache == "sdk": + cached_sdk_dir = pyside6_deploy_cache / "android-sdk" + if cached_sdk_dir.exists(): + shutil.rmtree(cached_sdk_dir) + elif clean_cache == "python": + cached_cpython_dir = pyside6_deploy_cache / "cpython" + if cached_cpython_dir.exists(): + shutil.rmtree(pyside6_deploy_cache / "cpython") + for cc_python_path in pyside6_deploy_cache.glob("Python-*"): + if cc_python_path.is_dir(): + shutil.rmtree(cc_python_path) + elif clean_cache == "toolchain": + for toolchain_path in pyside6_deploy_cache.glob("toolchain_*"): + if toolchain_path.is_file(): + toolchain_path.unlink() + + if not ndk_path: + # Download android ndk + ndk_path = download_android_ndk(pyside6_deploy_cache) + + if not sdk_path: + # download and unzip command-line tools + sdk_path = download_android_commandlinetools(pyside6_deploy_cache) + # install and update required android packages + install_android_packages(android_sdk_dir=sdk_path, android_api=api_level, dry_run=dry_run, + accept_license=auto_accept_license, skip_update=skip_update) + + templates_path = Path(__file__).parent / "templates" + + for plat_name in plat_names: + # for armv7a the API level dependent binaries like clang are named + # armv7a-linux-androideabi27-clang, as opposed to other platforms which + # are named like x86_64-linux-android27-clang + platform_data = None + if plat_name == "armv7a": + platform_data = PlatformData("armv7a", api_level, "armeabi-v7a", "armv7", + "armv7", "32") + elif plat_name == "aarch64": + platform_data = PlatformData("aarch64", api_level, "arm64-v8a", "arm64_v8a", "armv8-a", + "64") + elif plat_name == "i686": + platform_data = PlatformData("i686", api_level, "x86", "x86", "i686", "32") + else: # plat_name is x86_64 + platform_data = PlatformData("x86_64", api_level, "x86_64", "x86_64", "x86-64", "64") + + # python path is valid, if Python for android installation exists in python_path + python_path = (pyside6_deploy_cache + / f"Python-{platform_data.plat_name}-linux-android" / "_install") + valid_python_path = python_path.exists() + if Path(python_path).exists(): + expected_dirs = ["lib", "include"] + for expected_dir in expected_dirs: + if not (Path(python_path) / expected_dir).is_dir(): + valid_python_path = False + warnings.warn( + f"{str(python_path.resolve())} is corrupted. New Python for {plat_name} " + f"android will be cross-compiled into {str(pyside6_deploy_cache.resolve())}" + ) + break + + environment = Environment(loader=FileSystemLoader(templates_path)) + if not valid_python_path: + # clone cpython and checkout 3.10 + cpython_dir = pyside6_deploy_cache / "cpython" + python_ccompile_script = cpython_dir / f"cross_compile_{plat_name}.sh" + + if not cpython_dir.exists(): + logging.info(f"cloning cpython {PYTHON_VERSION}") + Repo.clone_from( + "https://github.com/python/cpython.git", + cpython_dir, + progress=CloneProgress(), + branch=PYTHON_VERSION, + ) + + if not python_ccompile_script.exists(): + host_system_config_name = run_command("./config.guess", cwd=cpython_dir, + dry_run=dry_run, show_stdout=True, + capture_stdout=True).strip() + + # use jinja2 to create cross_compile.sh script + template = environment.get_template("cross_compile.tmpl.sh") + content = template.render( + plat_name=platform_data.plat_name, + ndk_path=ndk_path, + api_level=platform_data.api_level, + android_py_install_path_prefix=pyside6_deploy_cache, + host_python_path=sys.executable, + python_version=PYTHON_VERSION, + host_system_name=host_system_config_name, + host_platform_name=sys.platform + ) + + logging.info(f"Writing Python cross compile script into {python_ccompile_script}") + with open(python_ccompile_script, mode="w", encoding="utf-8") as ccompile_script: + ccompile_script.write(content) + + # give run permission to cross compile script + python_ccompile_script.chmod(python_ccompile_script.stat().st_mode | stat.S_IEXEC) + + # clean built files + logging.info("Cleaning CPython built files") + run_command(["make", "distclean"], cwd=cpython_dir, dry_run=dry_run, ignore_fail=True) + + # run the cross compile script + logging.info(f"Running Python cross-compile for platform {platform_data.plat_name}") + run_command([f"./{python_ccompile_script.name}"], cwd=cpython_dir, dry_run=dry_run, + show_stdout=True) + + logging.info( + f"Cross compile Python for Android platform {platform_data.plat_name}. " + f"Final installation in {python_path}" + ) + + if only_py_cross_compile: + continue + + if only_py_cross_compile: + requested_platforms = ",".join(plat_names) + print(f"Python for Android platforms: {requested_platforms} cross compiled " + f"to {str(pyside6_deploy_cache)}") + sys.exit(0) + + qfp_toolchain = pyside6_deploy_cache / f"toolchain_{platform_data.plat_name}.cmake" + + if not qfp_toolchain.exists(): + template = environment.get_template("toolchain_default.tmpl.cmake") + content = template.render( + ndk_path=ndk_path, + sdk_path=sdk_path, + api_level=platform_data.api_level, + qt_install_path=qt_install_path, + plat_name=platform_data.plat_name, + android_abi=platform_data.android_abi, + qt_plat_name=platform_data.qt_plat_name, + gcc_march=platform_data.gcc_march, + plat_bits=platform_data.plat_bits, + python_version=PYTHON_VERSION, + target_python_path=python_path + ) + + logging.info(f"Writing Qt for Python toolchain file into {qfp_toolchain}") + with open(qfp_toolchain, mode="w", encoding="utf-8") as ccompile_script: + ccompile_script.write(content) + + # give run permission to cross compile script + qfp_toolchain.chmod(qfp_toolchain.stat().st_mode | stat.S_IEXEC) + + if sys.platform == "linux": + host_qt_install_suffix = "gcc_64" + elif sys.platform == "darwin": + host_qt_install_suffix = "macos" + else: + raise RuntimeError("Qt for Python cross compilation not supported on this platform") + + # run the cross compile script + logging.info(f"Running Qt for Python cross-compile for platform {platform_data.plat_name}") + qfp_ccompile_cmd = [sys.executable, "setup.py", "bdist_wheel", "--parallel=9", + "--standalone", + f"--cmake-toolchain-file={str(qfp_toolchain.resolve())}", + f"--qt-host-path={qt_install_path}/{host_qt_install_suffix}", + f"--plat-name=android_{platform_data.plat_name}", + f"--python-target-path={python_path}", + (f"--qt-target-path={qt_install_path}/" + f"android_{platform_data.qt_plat_name}"), + "--no-qt-tools"] + run_command(qfp_ccompile_cmd, cwd=pyside_setup_dir, dry_run=dry_run, show_stdout=True) diff --git a/tools/cross_compile_android/requirements.txt b/tools/cross_compile_android/requirements.txt new file mode 100644 index 000000000..62e8ee3b0 --- /dev/null +++ b/tools/cross_compile_android/requirements.txt @@ -0,0 +1,3 @@ +gitpython +Jinja2 +tqdm diff --git a/tools/cross_compile_android/templates/cross_compile.tmpl.sh b/tools/cross_compile_android/templates/cross_compile.tmpl.sh new file mode 100644 index 000000000..784e822ca --- /dev/null +++ b/tools/cross_compile_android/templates/cross_compile.tmpl.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +set -x -e +export HOST_ARCH={{ plat_name }}-linux-android +export TOOLCHAIN={{ ndk_path }}/toolchains/llvm/prebuilt/{{ host_platform_name }}-x86_64/bin +export TOOL_PREFIX=$TOOLCHAIN/$HOST_ARCH +export PLATFORM_API={{ api_level }} +{% if plat_name == "armv7a" -%} +export CXX=${TOOL_PREFIX}eabi${PLATFORM_API}-clang++ +export CPP="${TOOL_PREFIX}eabi${PLATFORM_API}-clang++ -E" +export CC=${TOOL_PREFIX}eabi${PLATFORM_API}-clang +{% else %} +export CXX=${TOOL_PREFIX}${PLATFORM_API}-clang++ +export CPP="${TOOL_PREFIX}${PLATFORM_API}-clang++ -E" +export CC=${TOOL_PREFIX}${PLATFORM_API}-clang +{% endif %} +export AR=$TOOLCHAIN/llvm-ar +export RANLIB=$TOOLCHAIN/llvm-ranlib +export LD=$TOOLCHAIN/ld +export READELF=$TOOLCHAIN/llvm-readelf +export CFLAGS='-fPIC -DANDROID' +./configure --host=$HOST_ARCH --target=$HOST_ARCH --build={{ host_system_name }} \ +--with-build-python={{ host_python_path }} --enable-shared \ +--enable-ipv6 ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no --without-ensurepip \ +ac_cv_little_endian_double=yes +make BLDSHARED="$CC -shared" CROSS-COMPILE=$TOOL_PREFIX- CROSS_COMPILE_TARGET=yes \ +INSTSONAME=libpython{{ python_version }}.so +make install prefix={{ android_py_install_path_prefix }}/Python-$HOST_ARCH/_install diff --git a/tools/cross_compile_android/templates/toolchain_default.tmpl.cmake b/tools/cross_compile_android/templates/toolchain_default.tmpl.cmake new file mode 100644 index 000000000..3c9752f43 --- /dev/null +++ b/tools/cross_compile_android/templates/toolchain_default.tmpl.cmake @@ -0,0 +1,73 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +# toolchain file to cross compile Qt for Python wheels for Android +cmake_minimum_required(VERSION 3.23) +include_guard(GLOBAL) +set(CMAKE_SYSTEM_NAME Android) +{% if plat_name == "armv7a" -%} +set(CMAKE_SYSTEM_PROCESSOR armv7-a) +{% else %} +set(CMAKE_SYSTEM_PROCESSOR {{ plat_name }}) +{% endif %} +set(CMAKE_ANDROID_API {{ api_level }}) +set(CMAKE_ANDROID_NDK {{ ndk_path }}) +set(CMAKE_ANDROID_ARCH_ABI {{ android_abi }}) +set(CMAKE_ANDROID_NDK_TOOLCHAIN_VERSION clang) +set(CMAKE_ANDROID_STL_TYPE c++_shared) +if(NOT DEFINED ANDROID_PLATFORM AND NOT DEFINED ANDROID_NATIVE_API_LEVEL) + set(ANDROID_PLATFORM "android-{{ api_level }}" CACHE STRING "") +endif() +set(ANDROID_SDK_ROOT {{ sdk_path }}) +{% if plat_name == "armv7a" -%} +set(_TARGET_NAME_ENDING "eabi{{ api_level }}") +{% else %} +set(_TARGET_NAME_ENDING "{{ api_level }}") +{% endif %} +set(QT_COMPILER_FLAGS "--target={{ plat_name }}-linux-android${_TARGET_NAME_ENDING} \ + -fomit-frame-pointer \ + -march={{ gcc_march }} \ + -msse4.2 \ + -mpopcnt \ + -m{{ plat_bits }} \ + -fPIC \ + -I{{ target_python_path }}/include/python{{ python_version }} \ + -Wno-unused-command-line-argument") +set(QT_COMPILER_FLAGS_RELEASE "-O2 -pipe") + +# FIXME +# https://gitlab.kitware.com/cmake/cmake/-/issues/23670 +# The CMake Android toolchain does not allow RPATHS. Hence CMAKE_INSTALL_RPATH does not work. +# Currently the linker flags are set directly as -Wl,-rpath='$ORIGIN' -Wl,-rpath='$ORIGIN/Qt/lib' +# set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) +# set(CMAKE_INSTALL_RPATH "$ORIGIN") + +set(QT_LINKER_FLAGS "-Wl,-O1 -Wl,--hash-style=gnu -Wl,-rpath='$ORIGIN' -Wl,-rpath='$ORIGIN/Qt/lib' \ + -Wl,--as-needed -L{{ qt_install_path }}/android_{{ qt_plat_name }}/lib \ + -L{{ qt_install_path }}/android_{{ qt_plat_name }}/plugins/platforms \ + -L{{ target_python_path }}/lib \ + -lpython{{ python_version }}") +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +add_compile_definitions(ANDROID) + +include(CMakeInitializeConfigs) +function(cmake_initialize_per_config_variable _PREFIX _DOCSTRING) + if (_PREFIX MATCHES "CMAKE_(C|CXX|ASM)_FLAGS") + set(CMAKE_${CMAKE_MATCH_1}_FLAGS_INIT "${QT_COMPILER_FLAGS}") + foreach (config DEBUG RELEASE MINSIZEREL RELWITHDEBINFO) + if (DEFINED QT_COMPILER_FLAGS_${config}) + set(CMAKE_${CMAKE_MATCH_1}_FLAGS_${config}_INIT "${QT_COMPILER_FLAGS_${config}}") + endif() + endforeach() + endif() + if (_PREFIX MATCHES "CMAKE_(SHARED|MODULE|EXE)_LINKER_FLAGS") + foreach (config SHARED MODULE EXE) + set(CMAKE_${config}_LINKER_FLAGS_INIT "${QT_LINKER_FLAGS}") + endforeach() + endif() + _cmake_initialize_per_config_variable(${ARGV}) +endfunction() diff --git a/tools/debug_renamer.py b/tools/debug_renamer.py index db3ba5040..ec777388b 100644 --- a/tools/debug_renamer.py +++ b/tools/debug_renamer.py @@ -1,32 +1,12 @@ -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the test suite of Qt for Python. -## -## $QT_BEGIN_LICENSE:GPL-EXCEPT$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3 as published by the Free Software -## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 -""" +import re +import sys +from argparse import ArgumentParser, FileType, RawTextHelpFormatter +from collections import OrderedDict + +DESC = """ debug_renamer.py ================ @@ -53,39 +33,15 @@ The Python output lines can be freely formatted. Any line which contains "0x.." followed by some name will be changed, all others are left alone. -We name these fields `object_id` and `typename`. - - -Operation ---------- - -The script reads from <stdin> until EOF. It produces output where the -`object_id` field is removed and some text is combined with `typename` -to produce a unique object name. - - -Example -------- - -You can create reference debugging output by using the modified interpreter at - - https://github.com/ctismer/cpython/tree/3.9-refdebug - -and pipe the error output through this script. -This is work in flux that might change quite often. To Do List ---------- Names of objects which are already deleted should be monitored and -not by chance be re-used. +not by chance be re-used. We need to think of a way to specify deletion. """ -import re -import sys -from collections import OrderedDict - def make_name(typename, name_pos): """ @@ -101,19 +57,57 @@ known_types = {} pattern = r"0x\w+\s+\S+" # hex word followed by non-WS rex = re.compile(pattern, re.IGNORECASE) -while True: - line = sys.stdin.readline() - if not line: - break + +def rename_hexval(line): if not (res := rex.search(line)): - print(line.rstrip()) - continue + return line start_pos, end_pos = res.start(), res.end() - beg, mid, end = line[:start_pos], line[start_pos : end_pos], line[end_pos:].rstrip() + beg, mid, end = line[:start_pos], line[start_pos:end_pos], line[end_pos:] object_id, typename = mid.split() + if int(object_id, 16) == 0: + return(f"{beg}{typename}_NULL{end}") if typename not in known_types: known_types[typename] = OrderedDict() obj_store = known_types[typename] if object_id not in obj_store: obj_store[object_id] = make_name(typename, len(obj_store)) - print(f"{beg}{obj_store[object_id]}{end}") + return(f"{beg}{obj_store[object_id]}{end}") + + +def hide_floatval(line): + return re.sub(r"\d+\.\d+", "<float>", line) + + +def process_all_lines(options): + """ + Process all lines from fin to fout. + The caller is responsible of opening and closing files if at all. + """ + fi, fo = options.input, options.output + rename = options.rename + float_ = options.float + while line := fi.readline(): + if rename: + line = rename_hexval(line) + if float_: + line = hide_floatval(line) + fo.write(line) + + +def create_argument_parser(desc): + parser = ArgumentParser(description=desc, formatter_class=RawTextHelpFormatter) + parser.add_argument("--rename", "-r", action="store_true", + help="Rename hex value and following word to a readable f'{word}_{anum}'") + parser.add_argument("--float", "-f", action="store_true", + help="Replace timing numbers by '<float>' (for comparing ctest output)") + parser.add_argument("--input", "-i", nargs="?", type=FileType("r"), default=sys.stdin, + help="Use the specified file instead of sys.stdin") + parser.add_argument("--output", "-o", nargs="?", type=FileType("w"), default=sys.stdout, + help="Use the specified file instead of sys.stdout") + return parser + + +if __name__ == "__main__": + argument_parser = create_argument_parser(DESC) + options = argument_parser.parse_args() + process_all_lines(options) diff --git a/tools/debug_windows.py b/tools/debug_windows.py index de3ddf445..832282895 100644 --- a/tools/debug_windows.py +++ b/tools/debug_windows.py @@ -1,43 +1,21 @@ -############################################################################# -## -## Copyright (C) 2018 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############### +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -""" +import argparse +import ctypes +import logging +import re +import subprocess +import sys +from os import path +from textwrap import dedent + +is_win = sys.platform == "win32" +if is_win: + import winreg + + +EPILOG = """ This is a troubleshooting script that assists finding out which DLLs or which symbols in a DLL are missing when executing a PySide6 python script. @@ -61,37 +39,21 @@ https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk """ -import sys -import re -import subprocess -import ctypes -import logging -import argparse -from os import path -from textwrap import dedent - -is_win = sys.platform == "win32" -is_py_3 = sys.version_info[0] == 3 -if is_win: - if is_py_3: - import winreg - else: - import _winreg as winreg - import exceptions - def get_parser_args(): desc_msg = "Run an executable under cdb with loader snaps set." help_msg = "Pass the executable and the arguments passed to it as a list." - parser = argparse.ArgumentParser(description=desc_msg) + parser = argparse.ArgumentParser(description=desc_msg, + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=EPILOG) parser.add_argument('args', nargs='*', help=help_msg) # Prepend -- so that python options like '-c' are ignored by # argparse. - massaged_args = ['--'] + sys.argv[1:] - return parser.parse_args(massaged_args) + help_requested = '-h' in sys.argv or '--help' in sys.argv + massaged_args = ['--'] + sys.argv[1:] if not help_requested else sys.argv + return parser, parser.parse_args(massaged_args) -parser_args = get_parser_args() verbose_log_file_name = path.join(path.dirname(path.abspath(__file__)), 'log_debug_windows.txt') @@ -187,10 +149,10 @@ def get_appropriate_kit(kits): log.info("Found Windows kits are: {}".format(kits)) chosen_kit = {'version': "0", 'value': None} for kit in kits: - if (kit['version'] > chosen_kit['version'] and + if (kit['version'] > chosen_kit['version'] # version 8.1 is actually '81', so consider everything # above version 20, as '2.0', etc. - kit['version'] < "20"): + and kit['version'] < "20"): chosen_kit = kit first_kit = kits[0] return first_kit @@ -203,7 +165,8 @@ def get_cdb_and_gflags_path(kits): bits = 'x64' if (sys.maxsize > 2 ** 32) else 'x32' debuggers_path = path.join(first_path_path, 'Debuggers', bits) cdb_path = path.join(debuggers_path, 'cdb.exe') - if not path.exists(cdb_path): # Try for older "Debugging Tools" packages + # Try for older "Debugging Tools" packages + if not path.exists(cdb_path): debuggers_path = "C:\\Program Files\\Debugging Tools for Windows (x64)" cdb_path = path.join(debuggers_path, 'cdb.exe') @@ -232,7 +195,7 @@ def toggle_loader_snaps(executable_name, gflags_path, enable=True): output = subprocess.check_output(gflags_args, stderr=subprocess.STDOUT, universal_newlines=True) log.info(output) - except exceptions.WindowsError as e: + except WindowsError as e: log.error("\nRunning {} exited with exception: " "\n{}".format(gflags_args, e)) exit(1) @@ -247,7 +210,7 @@ def find_error_like_snippets(content): lines = content.splitlines() context_lines = 4 - def error_predicate(l): + def error_predicate(line): # A list of mostly false positives are filtered out. # For deeper inspection, the full log exists. errors = {'errorhandling', @@ -265,8 +228,8 @@ def find_error_like_snippets(content): 'ERR_get_error', ('ERROR: Module load completed but symbols could ' 'not be loaded')} - return (re.search('error', l, re.IGNORECASE) - and all(e not in l for e in errors)) + return (re.search('error', line, re.IGNORECASE) + and all(e not in line for e in errors)) for i in range(1, len(lines)): line = lines[i] @@ -342,7 +305,7 @@ print(">>>>>>>>>>>>>>>>>>>>>>> QtCore object instance: {}".format(PySide6.QtCore call_command_under_cdb_with_gflags(sys.executable, ["-c", python_code]) -def handle_args(): +def handle_args(parser_args): if not parser_args.args: test_run_import_qt_core_under_cdb_with_gflags() else: @@ -355,9 +318,12 @@ if __name__ == '__main__': log.error("This script only works on Windows.") exit(1) + parser, parser_args = get_parser_args() + if is_admin(): - handle_args() + handle_args(parser_args) else: log.error("Please rerun the script with administrator privileges. " "It is required for gflags.exe to work. ") + parser.print_help() exit(1) diff --git a/tools/doc_modules.py b/tools/doc_modules.py new file mode 100644 index 000000000..d46f4db02 --- /dev/null +++ b/tools/doc_modules.py @@ -0,0 +1,209 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import os +import subprocess +import sys +from argparse import ArgumentParser, RawTextHelpFormatter +from pathlib import Path +import xml.sax +from xml.sax.handler import ContentHandler + +DESC = """Print a list of module short names ordered by typesystem dependencies +for which documentation can be built by intersecting the PySide6 modules with +the modules built in Qt.""" + + +ROOT_DIR = Path(__file__).parents[1].resolve() +SOURCE_DIR = ROOT_DIR / "sources" / "pyside6" / "PySide6" + + +qt_version = None +qt_include_dir = None + + +class TypeSystemContentHandler(ContentHandler): + """XML SAX content handler that extracts required modules from the + "load-typesystem" elements of the typesystem_file. Nodes that start + with Qt and are marked as generate == "no" are considered required.""" + + def __init__(self): + self.required_modules = [] + + def startElement(self, name, attrs): + if name == "load-typesystem": + generate = attrs.get("generate", "").lower() + if generate == "no" or generate == "false": + load_file_name = attrs.get("name") # "QtGui/typesystem_gui.xml" + if load_file_name.startswith("Qt"): + slash = load_file_name.find("/") + if slash > 0: + self.required_modules.append(load_file_name[:slash]) + + +def required_typesystems(module): + """Determine the required Qt modules by looking at the "load-typesystem" + elements of the typesystem_file.""" + name = module[2:].lower() + typesystem_file = SOURCE_DIR / module / f"typesystem_{name}.xml" + # Use a SAX parser since that works despite undefined entity + # errors for typesystem entities. + handler = TypeSystemContentHandler() + try: + parser = xml.sax.make_parser() + parser.setContentHandler(handler) + parser.parse(typesystem_file) + except Exception as e: + print(f"Warning: XML error parsing {typesystem_file}: {e}", file=sys.stderr) + return handler.required_modules + + +def query_qtpaths(keyword): + query_cmd = ["qtpaths", "-query", keyword] + output = subprocess.check_output(query_cmd, stderr=subprocess.STDOUT, + universal_newlines=True) + return output.strip() + + +def sort_modules(dependency_dict): + """Sort the modules by dependencies using brute force: Keep adding + modules all of whose requirements are present to the result list + until done.""" + result = [] + while True: + found = False + for module, dependencies in dependency_dict.items(): + if module not in result: + if all(dependency in result for dependency in dependencies): + result.append(module) + found = True + if not found: + break + + if len(result) < len(dependency_dict) and verbose: + for desired_module in dependency_dict.keys(): + if desired_module not in result: + print(f"Not documenting {desired_module} (missing dependency)", + file=sys.stderr) + return result + + +def _write_type_system(modules, file): + """Helper to write the type system for shiboken. It needs to be in + dependency order to prevent shiboken from loading the included + typesystems with generate="no", which causes those modules to be + missing.""" + for module in modules: + name = module[2:].lower() + filename = f"{module}/typesystem_{name}.xml" + print(f' <load-typesystem name="{filename}" generate="yes"/>', + file=file) + print("</typesystem>", file=file) + + +def write_type_system(modules, filename): + """Write the type system for shiboken in dependency order.""" + if filename == "-": + _write_type_system(modules, sys.stdout) + else: + path = Path(filename) + exists = path.exists() + with path.open(mode="a") as f: + if not exists: + print('<typesystem package="PySide">', file=f) + _write_type_system(modules, f) + + +def _write_global_header(modules, file): + """Helper to write the global header for shiboken.""" + for module in modules: + print(f"#include <{module}/{module}>", file=file) + + +def write_global_header(modules, filename): + """Write the global header for shiboken.""" + if filename == "-": + _write_global_header(modules, sys.stdout) + else: + with Path(filename).open(mode="a") as f: + _write_global_header(modules, f) + + +def _write_docconf(modules, file): + """Helper to write the include paths for the .qdocconf file.""" + # @TODO fix this for macOS frameworks. + for module in modules: + root = f" -I/{qt_include_dir}/{module}" + print(f"{root} \\", file=file) + print(f"{root}/{qt_version} \\", file=file) + print(f"{root}/{qt_version}/{module} \\", file=file) + + +def write_docconf(modules, filename): + """Write the include paths for the .qdocconf file.""" + if filename == "-": + _write_docconf(modules, sys.stdout) + else: + with Path(filename).open(mode="a") as f: + _write_docconf(modules, f) + + +if __name__ == "__main__": + argument_parser = ArgumentParser(description=DESC, + formatter_class=RawTextHelpFormatter) + argument_parser.add_argument("--verbose", "-v", action="store_true", + help="Verbose") + argument_parser.add_argument("qt_include_dir", help="Qt Include dir", + nargs='?', type=str) + argument_parser.add_argument("qt_version", help="Qt version string", + nargs='?', type=str) + argument_parser.add_argument("--typesystem", "-t", help="Typesystem file to write", + action="store", type=str) + argument_parser.add_argument("--global-header", "-g", help="Global header to write", + action="store", type=str) + argument_parser.add_argument("--docconf", "-d", help="docconf file to write", + action="store", type=str) + + options = argument_parser.parse_args() + verbose = options.verbose + if options.qt_include_dir: + qt_include_dir = Path(options.qt_include_dir) + if not qt_include_dir.is_dir(): + print(f"Invalid include directory passed: {options.qt_include_dir}", + file=sys.stderr) + sys.exit(-1) + else: + verbose = True # Called by hand to find out about available modules + query_cmd = ["qtpaths", "-query", "QT_INSTALL_HEADERS"] + qt_include_dir = Path(query_qtpaths("QT_INSTALL_HEADERS")) + if not qt_include_dir.is_dir(): + print("Cannot determine include directory", file=sys.stderr) + sys.exit(-1) + + qt_version = options.qt_version if options.qt_version else query_qtpaths("QT_VERSION") + + # Build a typesystem dependency dict of the available modules in order + # to be able to sort_modules by dependencies. This is required as + # otherwise shiboken will read the required typesystems with + # generate == "no" and thus omit modules. + module_dependency_dict = {} + for m in SOURCE_DIR.glob("Qt*"): + module = m.name + # QtGraphs duplicates symbols from QtDataVisualization causing shiboken errors + if module == "QtDataVisualization": + continue + qt_include_path = qt_include_dir / module + if qt_include_path.is_dir(): + module_dependency_dict[module] = required_typesystems(module) + elif verbose: + print(f"Not documenting {module} (not built)", file=sys.stderr) + + modules = sort_modules(module_dependency_dict) + print(" ".join([m[2:] for m in modules])) + + if options.typesystem: + write_type_system(modules, options.typesystem) + if options.global_header: + write_global_header(modules, options.global_header) + if options.docconf: + write_docconf(modules, options.docconf) diff --git a/tools/dump_metaobject.py b/tools/dump_metaobject.py index 96acc189c..6898e9317 100644 --- a/tools/dump_metaobject.py +++ b/tools/dump_metaobject.py @@ -1,69 +1,40 @@ -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only """Helper functions for formatting information on QMetaObject""" -from PySide6.QtCore import (QMetaClassInfo, QMetaEnum, QMetaMethod, - QMetaProperty, QMetaObject, QObject) +from PySide6.QtCore import QMetaMethod def _qbytearray_to_string(b): return bytes(b.data()).decode('utf-8') +def _format_metatype(meta_type): + return meta_type.id() if meta_type.isValid() else '<invalid>' + + def _dump_metaobject_helper(meta_obj, indent): - print('{}class {}:'.format(indent, meta_obj.className())) + meta_id = 0 + # FIXME: Otherwise crashes in Qt + if meta_obj.propertyOffset() < meta_obj.propertyCount(): + meta_id = _format_metatype(meta_obj.metaType()) + print(f'{indent}class {meta_obj.className()}/{meta_id}:') indent += ' ' info_offset = meta_obj.classInfoOffset() info_count = meta_obj.classInfoCount() if info_offset < info_count: - print('{}Info:'.format(indent)) + print(f'{indent}Info:') for i in range(info_offset, info_count): name = meta_obj.classInfo(i).name() value = meta_obj.classInfo(i).value() - print('{}{:4d} {}+{}'.format(indent, i, name, value)) + print(f'{indent}{i:4d} {name}+{value}') enumerator_offset = meta_obj.enumeratorOffset() enumerator_count = meta_obj.enumeratorCount() if enumerator_offset < enumerator_count: - print('{}Enumerators:'.format(indent)) + print(f'{indent}Enumerators:') for e in range(enumerator_offset, enumerator_count): meta_enum = meta_obj.enumerator(e) name = meta_enum.name() @@ -73,27 +44,27 @@ def _dump_metaobject_helper(meta_obj, indent): descr += ' flag' if meta_enum.isScoped(): descr += ' scoped' - for k in range(0, meta_enum.keyCount()): + for k in range(meta_enum.keyCount()): if k > 0: value_str += ', ' - value_str += '{} = {}'.format(meta_enum.key(k), - meta_enum.value(k)) - print('{}{:4d} {}{} ({})'.format(indent, e, name, descr, - value_str)) + key = meta_enum.key(k) + value = meta_enum.value(k) + value_str += f'{key} = {value}' + print(f'{indent}{e:4d} {name}{descr} ({value_str})') property_offset = meta_obj.propertyOffset() property_count = meta_obj.propertyCount() if property_offset < property_count: - print('{}Properties:'.format(indent)) + print(f'{indent}Properties:') for p in range(property_offset, property_count): meta_property = meta_obj.property(p) name = meta_property.name() desc = '' if meta_property.isConstant(): desc += ', constant' - if meta_property.isDesignable: + if meta_property.isDesignable(): desc += ', designable' - if meta_property.isFlagType: + if meta_property.isFlagType(): desc += ', flag' if meta_property.isEnumType(): desc += ', enum' @@ -101,13 +72,15 @@ def _dump_metaobject_helper(meta_obj, indent): desc += ', stored' if meta_property.isWritable(): desc += ', writable' - if meta_property.isResettable: + if meta_property.isResettable(): desc += ', resettable' if meta_property.hasNotifySignal(): - notify_name = meta_property.notifySignal().name() - desc += ', notify={}'.format(_qbytearray_to_string(notify_name)) - print('{}{:4d} {} {}{}'.format(indent, p, meta_property.typeName(), - name, desc)) + notify_name_b = meta_property.notifySignal().name() + notify_name = _qbytearray_to_string(notify_name_b) + desc += f', notify="{notify_name}"' + meta_id = _format_metatype(meta_property.metaType()) + type_name = meta_property.typeName() + print(f'{indent}{p:4d} {type_name}/{meta_id} "{name}"{desc}') method_offset = meta_obj.methodOffset() method_count = meta_obj.methodCount() @@ -129,17 +102,18 @@ def _dump_metaobject_helper(meta_obj, indent): typeString = ' (Slot)' elif type == QMetaMethod.Constructor: typeString = ' (Ct)' - desc = '{}{:4d} {}{} {}{}'.format(indent, m, access, - method.typeName(), signature, - typeString) + type_name = method.typeName() + desc = f'{indent}{m:4d} {access}{type_name} "{signature}"{typeString}' parameter_names = method.parameterNames() if parameter_names: parameter_types = method.parameterTypes() desc += ' Parameters:' for p, bname in enumerate(parameter_names): name = _qbytearray_to_string(bname) - type = _qbytearray_to_string(parameter_types[p]) - desc += ' {}: {}'.format(name if name else '<unnamed>', type) + type_name = _qbytearray_to_string(parameter_types[p]) + if not name: + name = '<unnamed>' + desc += f' "{name}": {type_name}' print(desc) diff --git a/tools/example_gallery/main.py b/tools/example_gallery/main.py index 1c37a8670..b5aa632c0 100644 --- a/tools/example_gallery/main.py +++ b/tools/example_gallery/main.py @@ -1,41 +1,5 @@ -############################################################################# -## -## Copyright (C) 2021 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############### +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only """ @@ -48,33 +12,83 @@ For the usage, simply run: since there is no special requirements. """ -from argparse import ArgumentParser, RawTextHelpFormatter import json import math +import os import shutil +import zipfile +import sys +from argparse import ArgumentParser, RawTextHelpFormatter +from dataclasses import dataclass +from enum import IntEnum, Enum from pathlib import Path from textwrap import dedent +class Format(Enum): + RST = 0 + MD = 1 + + +class ModuleType(IntEnum): + ESSENTIALS = 0 + ADDONS = 1 + M2M = 2 + + +SUFFIXES = {Format.RST: "rst", Format.MD: "md"} + + opt_quiet = False + + +LITERAL_INCLUDE = ".. literalinclude::" + + +IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".svgz", ".webp") + + +IGNORED_SUFFIXES = IMAGE_SUFFIXES + (".pdf", ".pyc", ".obj", ".mesh") + + suffixes = { + ".h": "cpp", + ".cpp": "cpp", + ".md": "markdown", ".py": "py", ".qml": "js", ".conf": "ini", ".qrc": "xml", ".ui": "xml", ".xbel": "xml", + ".xml": "xml", } +BASE_CONTENT = """\ +Examples +======== + + A collection of examples are provided with |project| to help new users + to understand different use cases of the module. + + You can find all these examples inside the + `pyside-setup <https://code.qt.io/cgit/pyside/pyside-setup.git/>`_ repository + on the `examples <https://code.qt.io/cgit/pyside/pyside-setup.git/tree/examples>`_ + directory. + +""" + + def ind(x): return " " * 4 * x -def get_lexer(suffix): - if suffix in suffixes: - return suffixes[suffix] - return "text" +def get_lexer(path): + if path.name == "CMakeLists.txt": + return "cmake" + lexer = suffixes.get(path.suffix) + return lexer if lexer else "text" def add_indent(s, level): @@ -87,6 +101,110 @@ def add_indent(s, level): return new_s +def check_img_ext(i): + """Check whether path is an image.""" + return i.suffix in IMAGE_SUFFIXES + + +@dataclass +class ModuleDescription: + """Specifies a sort key and type for a Qt module.""" + sort_key: int = 0 + module_type: ModuleType = ModuleType.ESSENTIALS + description: str = '' + + +MODULE_DESCRIPTIONS = { + "async": ModuleDescription(16, ModuleType.ESSENTIALS, ''), + "corelib": ModuleDescription(15, ModuleType.ESSENTIALS, ''), + "dbus": ModuleDescription(22, ModuleType.ESSENTIALS, ''), + "designer": ModuleDescription(11, ModuleType.ESSENTIALS, ''), + "gui": ModuleDescription(25, ModuleType.ESSENTIALS, ''), + "network": ModuleDescription(20, ModuleType.ESSENTIALS, ''), + "opengl": ModuleDescription(26, ModuleType.ESSENTIALS, ''), + "qml": ModuleDescription(0, ModuleType.ESSENTIALS, ''), + "quick": ModuleDescription(1, ModuleType.ESSENTIALS, ''), + "quickcontrols": ModuleDescription(2, ModuleType.ESSENTIALS, ''), + "samplebinding": ModuleDescription(30, ModuleType.ESSENTIALS, ''), + "scriptableapplication": ModuleDescription(30, ModuleType.ESSENTIALS, ''), + "sql": ModuleDescription(21, ModuleType.ESSENTIALS, ''), + "uitools": ModuleDescription(12, ModuleType.ESSENTIALS, ''), + "widgetbinding": ModuleDescription(30, ModuleType.ESSENTIALS, ''), + "widgets": ModuleDescription(10, ModuleType.ESSENTIALS, ''), + "xml": ModuleDescription(24, ModuleType.ESSENTIALS, ''), + "Qt Demos": ModuleDescription(0, ModuleType.ADDONS, ''), # from Qt repos + "3d": ModuleDescription(30, ModuleType.ADDONS, ''), + "axcontainer": ModuleDescription(20, ModuleType.ADDONS, ''), + "bluetooth": ModuleDescription(20, ModuleType.ADDONS, ''), + "charts": ModuleDescription(12, ModuleType.ADDONS, ''), + "datavisualization": ModuleDescription(11, ModuleType.ADDONS, ''), + "demos": ModuleDescription(0, ModuleType.ADDONS, ''), + "external": ModuleDescription(20, ModuleType.ADDONS, ''), + "graphs": ModuleDescription(10, ModuleType.ADDONS, ''), + "httpserver": ModuleDescription(0, ModuleType.ADDONS, ''), + "location": ModuleDescription(20, ModuleType.ADDONS, ''), + "multimedia": ModuleDescription(12, ModuleType.ADDONS, ''), + "networkauth": ModuleDescription(20, ModuleType.ADDONS, ''), + "pdf": ModuleDescription(20, ModuleType.ADDONS, ''), + "pdfwidgets": ModuleDescription(20, ModuleType.ADDONS, ''), + "quick3d": ModuleDescription(20, ModuleType.ADDONS, ''), + "remoteobjects": ModuleDescription(20, ModuleType.ADDONS, ''), + "serialbus": ModuleDescription(30, ModuleType.ADDONS, ''), + "serialport": ModuleDescription(30, ModuleType.ADDONS, ''), + "spatialaudio": ModuleDescription(20, ModuleType.ADDONS, ''), + "speech": ModuleDescription(20, ModuleType.ADDONS, ''), + "statemachine": ModuleDescription(30, ModuleType.ADDONS, ''), + "webchannel": ModuleDescription(30, ModuleType.ADDONS, ''), + "webenginequick": ModuleDescription(15, ModuleType.ADDONS, ''), + "webenginewidgets": ModuleDescription(16, ModuleType.ADDONS, ''), + "coap": ModuleDescription(0, ModuleType.M2M, ''), + "mqtt": ModuleDescription(0, ModuleType.M2M, ''), + "opcua": ModuleDescription(0, ModuleType.M2M, '') +} + + +def module_sort_key(name): + """Return key for sorting modules.""" + description = MODULE_DESCRIPTIONS.get(name) + module_type = int(description.module_type) if description else 5 + sort_key = description.sort_key if description else 100 + return f"{module_type}:{sort_key:04}:{name}" + + +def module_title(name): + """Return title for a module.""" + result = name.title() + description = MODULE_DESCRIPTIONS.get(name) + if description: + if description.description: + result += " - " + description.description + if description.module_type == ModuleType.M2M: + result += " (M2M)" + elif description.module_type == ModuleType.ADDONS: + result += " (Add-ons)" + else: + result += " (Essentials)" + return result + + +@dataclass +class ExampleData: + """Example data for formatting the gallery.""" + + def __init__(self): + self.headline = "" + + example: str + module: str + extra: str + doc_file: str + file_format: Format + abs_path: str + has_doc: bool + img_doc: Path + headline: str + + def get_module_gallery(examples): """ This function takes a list of dictionaries, that contain examples @@ -94,43 +212,40 @@ def get_module_gallery(examples): """ gallery = ( - ".. panels::\n" - f"{ind(1)}:container: container-lg pb-3\n" - f"{ind(1)}:column: col-lg-3 col-md-6 col-sm-6 col-xs-12 p-2\n\n" + ".. grid:: 1 4 4 4\n" + f"{ind(1)}:gutter: 2\n\n" ) # Iteration per rows for i in range(math.ceil(len(examples))): e = examples[i] - url = e["rst"].replace(".rst", ".html") - name = e["example"] - underline = f'{e["module"]}' - + suffix = SUFFIXES[e.file_format] + url = e.doc_file.replace(f".{suffix}", ".html") + name = e.example + underline = e.module - if e["extra"]: - underline += f'/{e["extra"]}' + if e.extra: + underline += f"/{e.extra}" if i > 0: - gallery += f"{ind(1)}---\n" - elif e["img_doc"]: - gallery += f"{ind(1)}---\n" - - if e["img_doc"]: - img_name = e['img_doc'].name - else: - img_name = "../example_no_image.png" + gallery += "\n" + img_name = e.img_doc.name if e.img_doc else "../example_no_image.png" - gallery += f"{ind(1)}:img-top: {img_name}\n" - gallery += f"{ind(1)}:img-top-cls: + d-flex align-self-center\n\n" + # Fix long names + if name.startswith("chapter"): + name = name.replace("chapter", "c") + elif name.startswith("advanced"): + name = name.replace("advanced", "a") + desc = e.headline + if not desc: + desc = f"found in the ``{underline}`` directory." - gallery += f"{ind(1)}`{name} <{url}>`_\n" - gallery += f"{ind(1)}+++\n" - gallery += f"{ind(1)}{underline}\n" - gallery += f"\n{ind(1)}.. link-button:: {url}\n" - gallery += f"{ind(2)}:type: url\n" - gallery += f"{ind(2)}:text: Go to Example\n" - gallery += f"{ind(2)}:classes: btn-qt btn-block stretched-link\n" + gallery += f"{ind(1)}.. grid-item-card:: {name}\n" + gallery += f"{ind(2)}:class-item: cover-img\n" + gallery += f"{ind(2)}:link: {url}\n" + gallery += f"{ind(2)}:img-top: {img_name}\n\n" + gallery += f"{ind(2)}{desc}\n" return f"{gallery}\n" @@ -144,34 +259,99 @@ def remove_licenses(s): return "\n".join(new_s) -def get_code_tabs(files, project_file): +def make_zip_archive(zip_name, src, skip_dirs=None): + src_path = Path(src).expanduser().resolve(strict=True) + if skip_dirs is None: + skip_dirs = [] + if not isinstance(skip_dirs, list): + print("Error: A list needs to be passed for 'skip_dirs'") + return + with zipfile.ZipFile(src_path.parents[0] / Path(zip_name), 'w', zipfile.ZIP_DEFLATED) as zf: + for file in src_path.rglob('*'): + skip = False + _parts = file.relative_to(src_path).parts + for sd in skip_dirs: + if sd in _parts: + skip = True + break + if not skip: + zf.write(file, file.relative_to(src_path.parent)) + + +def doc_file(project_dir, project_file_entry): + """Return the (optional) .rstinc file describing a source file.""" + rst_file = project_dir + if rst_file.name != "doc": # Special case: Dummy .pyproject file in doc dir + rst_file /= "doc" + rst_file /= Path(project_file_entry).name + ".rstinc" + return rst_file if rst_file.is_file() else None + + +def get_code_tabs(files, project_dir, file_format): content = "\n" + # Prepare ZIP file, and copy to final destination + zip_name = f"{project_dir.name}.zip" + make_zip_archive(zip_name, project_dir, skip_dirs=["doc"]) + zip_src = f"{project_dir}.zip" + zip_dst = EXAMPLES_DOC / zip_name + shutil.move(zip_src, zip_dst) + + if file_format == Format.RST: + content += f":download:`Download this example <{zip_name}>`\n\n" + else: + content += f"{{download}}`Download this example <{zip_name}>`\n\n" + content += "```{eval-rst}\n" + for i, project_file in enumerate(files): + if i == 0: + content += ".. tab-set::\n\n" + pfile = Path(project_file) - if pfile.suffix in (".png", ".pyc"): + if pfile.suffix in IGNORED_SUFFIXES: continue - content += f".. tabbed:: {project_file}\n\n" + content += f"{ind(1)}.. tab-item:: {project_file}\n\n" + + doc_rstinc_file = doc_file(project_dir, project_file) + if doc_rstinc_file: + indent = ind(2) + for line in doc_rstinc_file.read_text("utf-8").split("\n"): + content += indent + line + "\n" + content += "\n" - lexer = get_lexer(pfile.suffix) - content += add_indent(f".. code-block:: {lexer}", 1) + lexer = get_lexer(pfile) + content += add_indent(f"{ind(1)}.. code-block:: {lexer}", 1) content += "\n" - _path = f_path.resolve().parents[0] / project_file + _path = project_dir / project_file _file_content = "" - with open(_path, "r") as _f: - _file_content = remove_licenses(_f.read()) - - content += add_indent(_file_content, 2) + try: + with open(_path, "r", encoding="utf-8") as _f: + _file_content = remove_licenses(_f.read()) + except UnicodeDecodeError as e: + print(f"example_gallery: error decoding {project_dir}/{_path}:{e}", + file=sys.stderr) + raise + except FileNotFoundError as e: + print(f"example_gallery: error opening {project_dir}/{_path}:{e}", + file=sys.stderr) + raise + + content += add_indent(_file_content, 3) content += "\n\n" + + if file_format == Format.MD: + content += "```" + return content -def get_header_title(f_path): - _title = f_path.stem - url_name = "/".join(f_path.parts[f_path.parts.index("examples")+1:-1]) - url = f"{BASE_URL}/{url_name}" +def get_header_title(example_dir): + _index = example_dir.parts.index("examples") + rel_path = "/".join(example_dir.parts[_index:]) + _title = rel_path + url = f"{BASE_URL}/{rel_path}" return ( "..\n This file was auto-generated by the 'examples_gallery' " "script.\n Any change will be lost!\n\n" @@ -181,19 +361,271 @@ def get_header_title(f_path): ) +def rel_path(from_path, to_path): + """Determine relative paths for paths that are not subpaths (where + relative_to() fails) via a common root.""" + common = Path(*os.path.commonprefix([from_path.parts, to_path.parts])) + up_dirs = len(from_path.parts) - len(common.parts) + prefix = up_dirs * "../" + rel_to_common = os.fspath(to_path.relative_to(common)) + return f"{prefix}{rel_to_common}" + + +def read_rst_file(project_dir, project_files, doc_rst): + """Read the example .rst file and expand literal includes to project files + by relative paths to the example directory. Note: sphinx does not + handle absolute paths as expected, they need to be relative.""" + content = "" + with open(doc_rst, encoding="utf-8") as doc_f: + content = doc_f.read() + if LITERAL_INCLUDE not in content: + return content + + result = [] + path_to_example = rel_path(EXAMPLES_DOC, project_dir) + for line in content.split("\n"): + if line.startswith(LITERAL_INCLUDE): + file = line[len(LITERAL_INCLUDE) + 1:].strip() + if file in project_files: + line = f"{LITERAL_INCLUDE} {path_to_example}/{file}" + result.append(line) + return "\n".join(result) + + +def get_headline(text, file_format): + """Find the headline in the .rst file.""" + if file_format == Format.RST: + underline = text.find("\n====") + if underline != -1: + start = text.rfind("\n", 0, underline - 1) + return text[start + 1:underline] + elif file_format == Format.MD: + headline = text.find("# ") + if headline != -1: + new_line = text.find("\n", headline + 1) + if new_line != -1: + return text[headline + 2:new_line].strip() + return "" + + +def get_doc_source_file(original_doc_dir, example_name): + """Find the doc source file, return (Path, Format).""" + if original_doc_dir.is_dir(): + for file_format in (Format.RST, Format.MD): + suffix = SUFFIXES[file_format] + result = original_doc_dir / f"{example_name}.{suffix}" + if result.is_file(): + return result, file_format + return None, Format.RST + + +def get_screenshot(image_dir, example_name): + """Find screen shot: We look for an image with the same + example_name first, if not, we select the first.""" + if not image_dir.is_dir(): + return None + images = [i for i in image_dir.glob("*") if i.is_file() and check_img_ext(i)] + example_images = [i for i in images if i.name.startswith(example_name)] + if example_images: + return example_images[0] + if images: + return images[0] + return None + + +def write_resources(src_list, dst): + """Write a list of example resource paths to the dst path.""" + for src in src_list: + resource_written = shutil.copy(src, dst / src.name) + if not opt_quiet: + print("Written resource:", resource_written) + + +@dataclass +class ExampleParameters: + """Parameters obtained from scanning the examples directory.""" + + def __init__(self): + self.file_format = Format.RST + self.src_doc_dir = self.src_doc_file_path = self.src_screenshot = None + self.extra_names = "" + + example_dir: Path + module_name: str + example_name: str + extra_names: str + file_format: Format + target_doc_file: str + src_doc_dir: Path + src_doc_file_path: Path + src_screenshot: Path + + +def detect_pyside_example(example_root, pyproject_file): + """Detemine parameters of a PySide example.""" + p = ExampleParameters() + + p.example_dir = pyproject_file.parent + if p.example_dir.name == "doc": # Dummy pyproject in doc dir (scriptableapplication) + p.example_dir = p.example_dir.parent + + parts = p.example_dir.parts[len(example_root.parts):] + p.module_name = parts[0] + p.example_name = parts[-1] + # handling subdirectories besides the module level and the example + p.extra_names = "" if len(parts) == 2 else "_".join(parts[1:-1]) + + # Check for a 'doc' directory inside the example + src_doc_dir = p.example_dir / "doc" + + if src_doc_dir.is_dir(): + src_doc_file_path, fmt = get_doc_source_file(src_doc_dir, p.example_name) + if src_doc_file_path: + p.src_doc_file_path = src_doc_file_path + p.file_format = fmt + p.src_doc_dir = src_doc_dir + p.src_screenshot = get_screenshot(src_doc_dir, p.example_name) + + target_suffix = SUFFIXES[p.file_format] + doc_file = f"example_{p.module_name}_{p.extra_names}_{p.example_name}.{target_suffix}" + p.target_doc_file = doc_file.replace("__", "_") + return p + + +def detect_qt_example(example_root, pyproject_file): + """Detemine parameters of an example from a Qt repository.""" + p = ExampleParameters() + + p.example_dir = pyproject_file.parent + p.module_name = "Qt Demos" + p.example_name = p.example_dir.name + # Check for a 'doc' directory inside the example (qdoc) + doc_root = p.example_dir / "doc" + if doc_root.is_dir(): + src_doc_file_path, fmt = get_doc_source_file(doc_root / "src", p.example_name) + if src_doc_file_path: + p.src_doc_file_path = src_doc_file_path + p.file_format = fmt + p.src_doc_dir = doc_root + p.src_screenshot = get_screenshot(doc_root / "images", p.example_name) + + target_suffix = SUFFIXES[p.file_format] + p.target_doc_file = f"example_qtdemos_{p.example_name}.{target_suffix}" + return p + + +def write_example(example_root, pyproject_file, pyside_example=True): + """Read the project file and documentation, create the .rst file and + copy the data. Return a tuple of module name and a dict of example data.""" + p = (detect_pyside_example(example_root, pyproject_file) if pyside_example + else detect_qt_example(example_root, pyproject_file)) + + result = ExampleData() + result.example = p.example_name + result.module = p.module_name + result.extra = p.extra_names + result.doc_file = p.target_doc_file + result.file_format = p.file_format + result.abs_path = str(p.example_dir) + result.has_doc = bool(p.src_doc_file_path) + result.img_doc = p.src_screenshot + + files = [] + try: + with pyproject_file.open("r", encoding="utf-8") as pyf: + pyproject = json.load(pyf) + # iterate through the list of files in .pyproject and + # check if they exist, before appending to the list. + for f in pyproject["files"]: + if not Path(f).exists: + print(f"example_gallery: {f} listed in {pyproject_file} does not exist") + raise FileNotFoundError + else: + files.append(f) + except (json.JSONDecodeError, KeyError, FileNotFoundError) as e: + print(f"example_gallery: error reading {pyproject_file}: {e}") + raise + + headline = "" + if files: + doc_file = EXAMPLES_DOC / p.target_doc_file + with open(doc_file, "w", encoding="utf-8") as out_f: + if p.src_doc_file_path: + content_f = read_rst_file(p.example_dir, files, p.src_doc_file_path) + headline = get_headline(content_f, p.file_format) + if not headline: + print(f"example_gallery: No headline found in {doc_file}", + file=sys.stderr) + + # Copy other files in the 'doc' directory, but + # excluding the main '.rst' file and all the + # directories. + resources = [] + if pyside_example: + for _f in p.src_doc_dir.glob("*"): + if _f != p.src_doc_file_path and not _f.is_dir(): + resources.append(_f) + else: # Qt example: only use image. + if p.src_screenshot: + resources.append(p.src_screenshot) + write_resources(resources, EXAMPLES_DOC) + else: + content_f = get_header_title(p.example_dir) + content_f += get_code_tabs(files, pyproject_file.parent, p.file_format) + out_f.write(content_f) + + if not opt_quiet: + print(f"Written: {doc_file}") + else: + if not opt_quiet: + print("Empty '.pyproject' file, skipping") + + result.headline = headline + + return (p.module_name, result) + + +def example_sort_key(example: ExampleData): + name = example.example + return "AAA" + name if "gallery" in name else name + + +def sort_examples(example): + result = {} + for module in example.keys(): + result[module] = sorted(example.get(module), key=example_sort_key) + return result + + +def scan_examples_dir(examples_dir, pyside_example=True): + """Scan a directory of examples.""" + for pyproject_file in examples_dir.glob("**/*.pyproject"): + if pyproject_file.name != "examples.pyproject": + module_name, data = write_example(examples_dir, pyproject_file, + pyside_example) + if module_name not in examples: + examples[module_name] = [] + examples[module_name].append(data) + + if __name__ == "__main__": # Only examples with a '.pyproject' file will be listed. DIR = Path(__file__).parent - EXAMPLES_DOC = Path(f"{DIR}/../../sources/pyside6/doc/examples") - EXAMPLES_DIR = Path(f"{DIR}/../../examples/") - BASE_URL = "https://code.qt.io/cgit/pyside/pyside-setup.git/tree/examples" + EXAMPLES_DOC = Path(f"{DIR}/../../sources/pyside6/doc/examples").resolve() + EXAMPLES_DIR = Path(f"{DIR}/../../examples/").resolve() + BASE_URL = "https://code.qt.io/cgit/pyside/pyside-setup.git/tree" columns = 5 gallery = "" parser = ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter) + TARGET_HELP = f"Directory into which to generate Doc files (default: {str(EXAMPLES_DOC)})" + parser.add_argument("--target", "-t", action="store", dest="target_dir", help=TARGET_HELP) + parser.add_argument("--qt-src-dir", "-s", action="store", help="Qt source directory") parser.add_argument("--quiet", "-q", action="store_true", help="Quiet") options = parser.parse_args() opt_quiet = options.quiet + if options.target_dir: + EXAMPLES_DOC = Path(options.target_dir).resolve() # This main loop will be in charge of: # * Getting all the .pyproject files, @@ -203,114 +635,22 @@ if __name__ == "__main__": examples = {} # Create the 'examples' directory if it doesn't exist - if not EXAMPLES_DOC.is_dir(): - EXAMPLES_DOC.mkdir() - - for f_path in EXAMPLES_DIR.glob("**/*.pyproject"): - if str(f_path).endswith("examples.pyproject"): - continue - - parts = f_path.parts[len(EXAMPLES_DIR.parts):-1] - - module_name = parts[0] - example_name = parts[-1] - # handling subdirectories besides the module level and the example - extra_names = "" if len(parts) == 2 else "_".join(parts[1:-1]) - - rst_file = f"example_{module_name}_{extra_names}_{example_name}.rst" - - def check_img_ext(i): - EXT = (".png", ".jpg", ".jpeg") - if i.suffix in EXT: - return True - return False - - # Check for a 'doc' directory inside the example - has_doc = False - img_doc = None - original_doc_dir = Path(f_path.parent / "doc") - if original_doc_dir.is_dir(): - has_doc = True - images = [i for i in original_doc_dir.glob("*") if i.is_file() and check_img_ext(i)] - if len(images) > 0: - # We look for an image with the same example_name first, if not, we select the first - image_path = [i for i in images if example_name in str(i)] - if not image_path: - image_path = images[0] - else: - img_doc = image_path[0] - - if module_name not in examples: - examples[module_name] = [] - - examples[module_name].append( - { - "example": example_name, - "module": module_name, - "extra": extra_names, - "rst": rst_file, - "abs_path": str(f_path), - "has_doc": has_doc, - "img_doc": img_doc, - } - ) - - pyproject = "" - with open(str(f_path), "r") as pyf: - pyproject = json.load(pyf) - - if pyproject: - rst_file_full = EXAMPLES_DOC / rst_file - - with open(rst_file_full, "w") as out_f: - if has_doc: - doc_path = Path(f_path.parent) / "doc" - doc_rst = doc_path / f"{example_name}.rst" - - with open(doc_rst) as doc_f: - content_f = doc_f.read() - - # Copy other files in the 'doc' directory, but - # excluding the main '.rst' file and all the - # directories. - for _f in doc_path.glob("*"): - if _f == doc_rst or _f.is_dir(): - continue - src = _f - dst = EXAMPLES_DOC / _f.name - - resource_written = shutil.copy(src, dst) - if not opt_quiet: - print("Written resource:", resource_written) - else: - content_f = get_header_title(f_path) - content_f += get_code_tabs(pyproject["files"], out_f) - out_f.write(content_f) - - if not opt_quiet: - print(f"Written: {EXAMPLES_DOC}/{rst_file}") - else: - if not opt_quiet: - print("Empty '.pyproject' file, skipping") - - base_content = dedent( - """\ - .. - This file was auto-generated from the 'pyside-setup/tools/example_gallery' - All editions in this file will be lost. - - |project| Examples - =================== - - A collection of examples are provided with |project| to help new users - to understand different use cases of the module. - - You can find all these examples inside the ``pyside-setup`` on the ``examples`` - directory, or you can access them after installing |pymodname| from ``pip`` - inside the ``site-packages/PySide6/examples`` directory. - - """ - ) + # If it does exist, remove it and create a new one to start fresh + if EXAMPLES_DOC.is_dir(): + shutil.rmtree(EXAMPLES_DOC, ignore_errors=True) + if not opt_quiet: + print("WARNING: Deleted old html directory") + EXAMPLES_DOC.mkdir(exist_ok=True) + + scan_examples_dir(EXAMPLES_DIR) + if options.qt_src_dir: + qt_src = Path(options.qt_src_dir) + if not qt_src.is_dir(): + print("Invalid Qt source directory: {}", file=sys.stderr) + sys.exit(-1) + scan_examples_dir(qt_src.parent / "qtdoc", pyside_example=False) + + examples = sort_examples(examples) # We generate a 'toctree' at the end of the file, to include the new # 'example' rst files, so we get no warnings, and also that users looking @@ -329,12 +669,14 @@ if __name__ == "__main__": # Writing the main example rst file. index_files = [] with open(f"{EXAMPLES_DOC}/index.rst", "w") as f: - f.write(base_content) - for module_name, e in sorted(examples.items()): + f.write(BASE_CONTENT) + for module_name in sorted(examples.keys(), key=module_sort_key): + e = examples.get(module_name) for i in e: - index_files.append(i["rst"]) - f.write(f"{module_name.title()}\n") - f.write(f"{'*' * len(module_name.title())}\n") + index_files.append(i.doc_file) + title = module_title(module_name) + f.write(f"{title}\n") + f.write(f"{'*' * len(title)}\n") f.write(get_module_gallery(e)) f.write("\n\n") f.write(footer_index) diff --git a/tools/leak_finder.py b/tools/leak_finder.py index 5b5102887..8a21c2337 100644 --- a/tools/leak_finder.py +++ b/tools/leak_finder.py @@ -1,30 +1,5 @@ -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the test suite of Qt for Python. -## -## $QT_BEGIN_LICENSE:GPL-EXCEPT$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3 as published by the Free Software -## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 """ leak_finder.py @@ -90,11 +65,10 @@ These objects are real leaks if their number is growing with the probe size. For analysis, the number of new objects per type is counted. """ -import sys -import gc import array +import gc +import sys import unittest - # this comes from Python, too from test import support diff --git a/tools/license_check.py b/tools/license_check.py index 052c41ca5..4b12a05fd 100644 --- a/tools/license_check.py +++ b/tools/license_check.py @@ -1,47 +1,10 @@ -############################################################################# -## -## Copyright (C) 2021 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import os -from pathlib import Path import subprocess import sys - +from pathlib import Path """Tool to run a license check diff --git a/tools/metaobject_dump.py b/tools/metaobject_dump.py index db61ccc4b..b6cde13ef 100644 --- a/tools/metaobject_dump.py +++ b/tools/metaobject_dump.py @@ -1,49 +1,12 @@ -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import sys -from dump_metaobject import dump_metaobject +from dump_metaobject import dump_metaobject # Import all widget classes to enable instantiating them by type name from PySide6.QtWidgets import * - DESC = """ metaobject_dump.py <class_name> @@ -62,6 +25,6 @@ if __name__ == '__main__': type_name = sys.argv[1] type_instance = eval(type_name) if not type_instance: - print('Invalid type {}'.format(type_name)) + print(f'Invalid type {type_name}') sys.exit(1) dump_metaobject(type_instance.staticMetaObject) diff --git a/tools/missing_bindings-requirements.txt b/tools/missing_bindings-requirements.txt deleted file mode 100644 index bbe8e7ac2..000000000 --- a/tools/missing_bindings-requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -pyside6 -pyqt5 -beautifulsoup4 -pyqt3d -pyqtchart -pyqtdatavisualization -pyqtwebengine diff --git a/tools/missing_bindings/config.py b/tools/missing_bindings/config.py index 3419dfdb1..ddaf20685 100644 --- a/tools/missing_bindings/config.py +++ b/tools/missing_bindings/config.py @@ -1,104 +1,74 @@ -############################################################################# -## -## Copyright (C) 2021 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only modules_to_test = { # 6.0 - 'QtCore': 'qtcore-module.html', - 'QtGui': 'qtgui-module.html', - 'QtNetwork': 'qtnetwork-module.html', - 'QtQml': 'qtqml-module.html', - 'QtQuick': 'qtquick-module.html', - 'QtQuickWidgets': 'qtquickwidgets-module.html', - 'QtQuickControls2': 'qtquickcontrols2-module.html', - #QtQuick3D - no python bindings - 'QtSql': 'qtsql-module.html', - 'QtWidgets': 'qtwidgets-module.html', - 'QtConcurrent': 'qtconcurrent-module.html', - #QtDBUS - no python bindings - 'QtHelp': 'qthelp-module.html', - 'QtOpenGL': 'qtopengl-module.html', - 'QtPrintSupport': 'qtprintsupport-module.html', - 'QtSvg': 'qtsvg-module.html', - 'QtUiTools': 'qtuitools-module.html', - 'QtXml': 'qtxml-module.html', - 'QtTest': 'qttest-module.html', - #'QtXmlPatterns': 'qtxmlpatterns-module.html', # in Qt5 compat - 'Qt3DCore': 'qt3dcore-module.html', - 'Qt3DInput': 'qt3dinput-module.html', - 'Qt3DLogic': 'qt3dlogic-module.html', - 'Qt3DRender': 'qt3drender-module.html', - 'Qt3DAnimation': 'qt3danimation-module.html', - 'Qt3DExtras': 'qt3dextras-module.html', - #'QtNetworkAuth': 'qtnetworkauth-module.html', # no python bindings - #'QtCoAp' -- TODO - #'QtMqtt' -- TODO - #'QtOpcUA' -- TODO + 'QtCore': 'qtcore-module.html', + 'QtGui': 'qtgui-module.html', + 'QtNetwork': 'qtnetwork-module.html', + 'QtQml': 'qtqml-module.html', + 'QtQuick': 'qtquick-module.html', + 'QtQuickWidgets': 'qtquickwidgets-module.html', + # Broken in 6.5.0 + #'QtQuickControls2': 'qtquickcontrols-module.html', + 'QtSql': 'qtsql-module.html', + 'QtWidgets': 'qtwidgets-module.html', + 'QtConcurrent': 'qtconcurrent-module.html', + 'QtDBus': 'qtdbus-module.html', + 'QtHelp': 'qthelp-module.html', + 'QtOpenGL': 'qtopengl-module.html', + 'QtPrintSupport': 'qtprintsupport-module.html', + 'QtSvg': 'qtsvg-module.html', + 'QtSvgWidgets': 'qtsvgwidgets-module.html', + 'QtUiTools': 'qtuitools-module.html', + 'QtXml': 'qtxml-module.html', + 'QtTest': 'qttest-module.html', + 'Qt3DCore': 'qt3dcore-module.html', + 'Qt3DInput': 'qt3dinput-module.html', + 'Qt3DLogic': 'qt3dlogic-module.html', + 'Qt3DRender': 'qt3drender-module.html', + 'Qt3DAnimation': 'qt3danimation-module.html', + 'Qt3DExtras': 'qt3dextras-module.html', + 'QtNetworkAuth': 'qtnetworkauth-module.html', + 'QtStateMachine': 'qtstatemachine-module.html', + # 'QtCoAp' -- TODO + # 'QtMqtt' -- TODO + # 'QtOpcUA' -- TODO # 6.1 - #'QtScxml': 'qtscxml-module.html', - #'QtCharts': 'qtcharts-module.html', - #'QtDataVisualization': 'qtdatavisualization-module.html', + 'QtScxml': 'qtscxml-module.html', + 'QtCharts': 'qtcharts-module.html', + 'QtDataVisualization': 'qtdatavisualization-module.html', # 6.2 'QtBluetooth': 'qtbluetooth-module.html', - #'QtPositioning': 'qtpositioning-module.html', - #'QtMultimedia': 'qtmultimedia-module.html', - #'QtRemoteObjects': 'qtremoteobjects-module.html', - #'QtSensors': 'qtsensors-module.html', - #'QtSerialPort': 'qtserialport-module.html', - #'QtWebChannel': 'qtwebchannel-module.html', - #'QtWebEngine': 'qtwebengine-module.html', - #'QtWebEngineCore': 'qtwebenginecore-module.html', - #'QtWebEngineWidgets': 'qtwebenginewidgets-module.html', - #'QtWebSockets': 'qtwebsockets-module.html', + 'QtPositioning': 'qtpositioning-module.html', + 'QtMultimedia': 'qtmultimedia-module.html', + 'QtRemoteObjects': 'qtremoteobjects-module.html', + 'QtSensors': 'qtsensors-module.html', + 'QtSerialPort': 'qtserialport-module.html', + 'QtWebChannel': 'qtwebchannel-module.html', + 'QtWebEngineCore': 'qtwebenginecore-module.html', + 'QtWebEngineQuick': 'qtwebenginequick-module.html', + 'QtWebEngineWidgets': 'qtwebenginewidgets-module.html', + 'QtWebSockets': 'qtwebsockets-module.html', + 'QtHttpServer': 'qthttpserver-module.html', - # 6.x + # 6.3 #'QtSpeech': 'qtspeech-module.html', - #'QtMultimediaWidgets': 'qtmultimediawidgets-module.html', - #'QtLocation': 'qtlocation-module.html', + 'QtMultimediaWidgets': 'qtmultimediawidgets-module.html', + 'QtNfc': 'qtnfc-module.html', + 'QtQuick3D': 'qtquick3d-module.html', + + # 6.4 + 'QtPdf': 'qtpdf-module.html', # this include qtpdfwidgets + 'QtSpatialAudio': 'qtspatialaudio-module.html', + + # 6.5 + 'QtSerialBus': 'qtserialbus-module.html', + 'QtTextToSpeech': 'qttexttospeech-module.html', + 'QtLocation': 'qtlocation-module.html', - # Not in 6 - #'QtScriptTools': 'qtscripttools-module.html', - #'QtMacExtras': 'qtmacextras-module.html', - #'QtX11Extras': 'qtx11extras-module.html', - #'QtWinExtras': 'qtwinextras-module.html', } types_to_ignore = { diff --git a/tools/missing_bindings/main.py b/tools/missing_bindings/main.py index 7390687ff..4c223050d 100644 --- a/tools/missing_bindings/main.py +++ b/tools/missing_bindings/main.py @@ -1,41 +1,5 @@ -############################################################################# -## -## Copyright (C) 2021 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only # This script is used to generate a summary of missing types / classes # which are present in C++ Qt6, but are missing in PySide6. @@ -48,25 +12,26 @@ # PySide6. # # Example invocation of script: -# python missing_bindings.py --qt-version 6.0 -w all +# python missing_bindings.py --qt-version 6.3 -w all # --qt-version - specify which version of qt documentation to load. # -w - if PyQt6 is an installed package, check if the tested # class also exists there. import argparse -import os.path import sys from textwrap import dedent from time import gmtime, strftime from urllib import request +from pathlib import Path from bs4 import BeautifulSoup - from config import modules_to_test, types_to_ignore +import pandas as pd +import matplotlib.pyplot as plt qt_documentation_website_prefixes = { - "6.0": "http://doc.qt.io/qt-6/", - "dev": "http://doc-snapshots.qt.io/qt5-dev/", + "6.5": "https://doc.qt.io/qt-6/", + "dev": "https://doc-snapshots.qt.io/qt6-dev/", } @@ -94,8 +59,8 @@ def get_parser(): parser.add_argument( "--qt-version", "-v", - default="6.0", - choices=["6.0", "dev"], + default="6.5", + choices=["6.5", "dev"], type=str, dest="version", help="the Qt version to use to check for types", @@ -104,10 +69,16 @@ def get_parser(): "--which-missing", "-w", default="all", - choices=["all", "in-pyqt", "not-in-pyqt"], + choices=["all", "in-pyqt", "not-in-pyqt", "in-pyside-not-in-pyqt"], type=str, dest="which_missing", - help="Which missing types to show (all, or just those " "that are not present in PyQt)", + help="Which missing types to show (all, or just those that are not present in PyQt)", + ) + parser.add_argument( + "--plot", + action="store_true", + help="Create module-wise bar plot comparisons for the missing bindings comparisons" + " between Qt, PySide6 and PyQt6", ) return parser @@ -115,30 +86,28 @@ def get_parser(): def wikilog(*pargs, **kw): print(*pargs) - computed_str = "" - for arg in pargs: - computed_str += str(arg) + computed_str = "".join(str(arg) for arg in pargs) style = "text" if "style" in kw: style = kw["style"] if style == "heading1": - computed_str = "= " + computed_str + " =" + computed_str = f"= {computed_str} =" elif style == "heading5": - computed_str = "===== " + computed_str + " =====" + computed_str = f"===== {computed_str} =====" elif style == "with_newline": - computed_str += "\n" + computed_str = f"{computed_str}\n" elif style == "bold_colon": computed_str = computed_str.replace(":", ":'''") - computed_str += "'''" - computed_str += "\n" + computed_str = f"{computed_str}'''\n" elif style == "error": - computed_str = "''" + computed_str.strip("\n") + "''\n" + computed_str = computed_str.strip("\n") + computed_str = f"''{computed_str}''\n" elif style == "text_with_link": computed_str = computed_str elif style == "code": - computed_str = " " + computed_str + computed_str = f" {computed_str}" elif style == "end": return @@ -157,9 +126,12 @@ if __name__ == "__main__": pyside_package_name = "PySide6" pyqt_package_name = "PyQt6" + data = {"module": [], "qt": [], "pyside": [], "pyqt": []} total_missing_types_count = 0 total_missing_types_count_compared_to_pyqt = 0 total_missing_modules_count = 0 + total_missing_pyqt_types_count = 0 + total_missing_pyqt_modules_count = 0 wiki_file = open("missing_bindings_for_wiki_qt_io.txt", "w") wiki_file.truncate() @@ -185,11 +157,11 @@ if __name__ == "__main__": ) wikilog( - "Similar report:\n" "https://gist.github.com/ethanhs/6c626ca4e291f3682589699296377d3a", + "Similar report:\n https://gist.github.com/ethanhs/6c626ca4e291f3682589699296377d3a", style="text_with_link", ) - python_executable = os.path.basename(sys.executable or "") + python_executable = Path(sys.executable).name or "" command_line_arguments = " ".join(sys.argv) report_date = strftime("%Y-%m-%d %H:%M:%S %Z", gmtime()) @@ -227,8 +199,6 @@ if __name__ == "__main__": try: pyqt_module_name = module_name - if module_name == "QtCharts": - pyqt_module_name = module_name[:-1] pyqt_tested_module = getattr( __import__(pyqt_package_name, fromlist=[pyqt_module_name]), pyqt_module_name @@ -240,6 +210,7 @@ if __name__ == "__main__": f"Received error: {e_str}.\n", style="error", ) + total_missing_pyqt_modules_count += 1 # Get C++ class list from documentation page. page = request.urlopen(url) @@ -250,66 +221,89 @@ if __name__ == "__main__": types_on_html_page = [] for link in links: - link_text = link.text - link_text = link_text.replace("::", ".") + link_text = link.text.replace("::", ".") if link_text not in types_to_ignore: types_on_html_page.append(link_text) - wikilog(f"Number of types in {module_name}: {len(types_on_html_page)}", style="bold_colon") + total_qt_types = len(types_on_html_page) + wikilog(f"Number of types in {module_name}: {total_qt_types}", style="bold_colon") - missing_types_count = 0 + missing_pyside_types_count = 0 + missing_pyqt_types_count = 0 missing_types_compared_to_pyqt = 0 missing_types = [] for qt_type in types_on_html_page: - try: - pyside_qualified_type = "pyside_tested_module." + is_present_in_pyqt = False + is_present_in_pyside = False + missing_type = None - if "QtCharts" == module_name: - pyside_qualified_type += "QtCharts." - elif "DataVisualization" in module_name: - pyside_qualified_type += "QtDataVisualization." + try: + pyqt_qualified_type = f"pyqt_tested_module.{qt_type}" + eval(pyqt_qualified_type) + is_present_in_pyqt = True + except Exception as e: + print(f"{type(e).__name__}: {e}") + missing_pyqt_types_count += 1 + total_missing_pyqt_types_count += 1 - pyside_qualified_type += qt_type + try: + pyside_qualified_type = f"pyside_tested_module.{qt_type}" eval(pyside_qualified_type) - except: + is_present_in_pyside = True + except Exception as e: + print("Failed eval-in pyside qualified types") + print(f"{type(e).__name__}: {e}") missing_type = qt_type - missing_types_count += 1 + missing_pyside_types_count += 1 total_missing_types_count += 1 - is_present_in_pyqt = False - try: - pyqt_qualified_type = "pyqt_tested_module." - - if "Charts" in module_name: - pyqt_qualified_type += "QtCharts." - elif "DataVisualization" in module_name: - pyqt_qualified_type += "QtDataVisualization." - - pyqt_qualified_type += qt_type - eval(pyqt_qualified_type) - missing_type += " (is present in PyQt6)" + if is_present_in_pyqt: + missing_type = f"{missing_type} (is present in PyQt6)" missing_types_compared_to_pyqt += 1 total_missing_types_count_compared_to_pyqt += 1 - is_present_in_pyqt = True - except: - pass + # missing in PySide + if not is_present_in_pyside: if args.which_missing == "all": missing_types.append(missing_type) + message = f"Missing types in PySide (all) {module_name}:" + # missing in PySide and present in pyqt elif args.which_missing == "in-pyqt" and is_present_in_pyqt: missing_types.append(missing_type) + message = f"Missing types in PySide6 (but present in PyQt6) {module_name}:" + # missing in both PyQt and PySide elif args.which_missing == "not-in-pyqt" and not is_present_in_pyqt: missing_types.append(missing_type) + message = f"Missing types in PySide6 (also missing in PyQt6) {module_name}:" + elif ( + args.which_missing == "in-pyside-not-in-pyqt" + and not is_present_in_pyqt + ): + missing_types.append(qt_type) + message = f"Missing types in PyQt6 (but present in PySide6) {module_name}:" if len(missing_types) > 0: - wikilog(f"Missing types in {module_name}:", style="with_newline") + wikilog(message, style="with_newline") missing_types.sort() for missing_type in missing_types: wikilog(missing_type, style="code") wikilog("") + if args.which_missing != "in-pyside-not-in-pyqt": + missing_types_count = missing_pyside_types_count + else: + missing_types_count = missing_pyqt_types_count + + if args.plot: + total_pyside_types = total_qt_types - missing_pyside_types_count + total_pyqt_types = total_qt_types - missing_pyqt_types_count + data["module"].append(module_name) + data["qt"].append(total_qt_types) + data["pyside"].append(total_pyside_types) + data["pyqt"].append(total_pyqt_types) + wikilog(f"Number of missing types: {missing_types_count}", style="bold_colon") - if len(missing_types) > 0: + if len(missing_types) > 0 and args.which_missing != "in-pyside-not-in-pyqt": wikilog( "Number of missing types that are present in PyQt6: " f"{missing_types_compared_to_pyqt}", @@ -319,12 +313,37 @@ if __name__ == "__main__": else: wikilog("", style="end") + if args.plot: + df = pd.DataFrame(data=data, columns=["module", "qt", "pyside", "pyqt"]) + df.set_index("module", inplace=True) + df.plot(kind="bar", title="Qt API Coverage plot") + plt.legend() + plt.xticks(rotation=45) + plt.ylabel("Types Count") + figure = plt.gcf() + figure.set_size_inches(32, 18) # set to full_screen + plt.savefig("missing_bindings_comparison_plot.png", bbox_inches='tight') + print(f"Plot saved in {Path.cwd() / 'missing_bindings_comparison_plot.png'}\n") + wikilog("Summary", style="heading5") - wikilog(f"Total number of missing types: {total_missing_types_count}", style="bold_colon") - wikilog( - "Total number of missing types that are present in PyQt6: " - f"{total_missing_types_count_compared_to_pyqt}", - style="bold_colon", - ) - wikilog(f"Total number of missing modules: {total_missing_modules_count}", style="bold_colon") + + if args.which_missing != "in-pyside-not-in-pyqt": + wikilog(f"Total number of missing types: {total_missing_types_count}", style="bold_colon") + wikilog( + "Total number of missing types that are present in PyQt6: " + f"{total_missing_types_count_compared_to_pyqt}", + style="bold_colon", + ) + wikilog( + f"Total number of missing modules: {total_missing_modules_count}", style="bold_colon" + ) + else: + wikilog( + f"Total number of missing types in PyQt6: {total_missing_pyqt_types_count}", + style="bold_colon", + ) + wikilog( + f"Total number of missing modules in PyQt6: {total_missing_pyqt_modules_count}", + style="bold_colon", + ) wiki_file.close() diff --git a/tools/missing_bindings/requirements.txt b/tools/missing_bindings/requirements.txt index 732522d26..08aa0a024 100644 --- a/tools/missing_bindings/requirements.txt +++ b/tools/missing_bindings/requirements.txt @@ -1,4 +1,6 @@ beautifulsoup4 +pandas +matplotlib # PySide PySide6 @@ -6,3 +8,7 @@ PySide6 # PyQt PyQt6 PyQt6-3D +PyQt6-Charts +PyQt6-DataVisualization +PyQt6-NetworkAuth +PyQt6-WebEngine diff --git a/tools/qtcpp2py.py b/tools/qtcpp2py.py new file mode 100644 index 000000000..e4e381675 --- /dev/null +++ b/tools/qtcpp2py.py @@ -0,0 +1,63 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +import logging +import os +import sys +from argparse import ArgumentParser, RawTextHelpFormatter +from pathlib import Path + +sys.path.append(os.fspath(Path(__file__).parent / "snippets_translate")) + +from converter import snippet_translate + +DESCRIPTION = "Tool to convert C++ to Python based on snippets_translate" + + +def create_arg_parser(desc): + parser = ArgumentParser(description=desc, + formatter_class=RawTextHelpFormatter) + parser.add_argument("--stdout", "-s", action="store_true", + help="Write to stdout") + parser.add_argument("--force", "-f", action="store_true", + help="Force overwrite of existing files") + parser.add_argument("files", type=str, nargs="+", + help="C++ source file(s)") + return parser + + +if __name__ == "__main__": + arg_parser = create_arg_parser(DESCRIPTION) + args = arg_parser.parse_args() + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(__name__) + + for input_file_str in args.files: + input_file = Path(input_file_str) + if not input_file.is_file(): + logger.error(f"{input_file_str} does not exist or is not a file.") + sys.exit(-1) + + if input_file.suffix != ".cpp" and input_file.suffix != ".h": + logger.error(f"{input_file} does not appear to be a C++ file.") + sys.exit(-1) + + translated_lines = [f"# Converted from {input_file.name}\n"] + for line in input_file.read_text().split("\n"): + translated_lines.append(snippet_translate(line)) + translated = "\n".join(translated_lines) + + if args.stdout: + sys.stdout.write(translated) + else: + target_file = input_file.parent / (input_file.stem + ".py") + if target_file.exists(): + if not target_file.is_file(): + logger.error(f"{target_file} exists and is not a file.") + sys.exit(-1) + if not args.force: + logger.error(f"{target_file} exists. Use -f to overwrite.") + sys.exit(-1) + + target_file.write_text(translated) + logger.info(f"Wrote {target_file}.") diff --git a/tools/qtpy2cpp.py b/tools/qtpy2cpp.py deleted file mode 100644 index 52bff787d..000000000 --- a/tools/qtpy2cpp.py +++ /dev/null @@ -1,99 +0,0 @@ -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -from argparse import ArgumentParser, RawTextHelpFormatter -import logging -import os -import sys -from qtpy2cpp_lib.visitor import ConvertVisitor - - -DESCRIPTION = "Tool to convert Python to C++" - - -def create_arg_parser(desc): - parser = ArgumentParser(description=desc, - formatter_class=RawTextHelpFormatter) - parser.add_argument('--debug', '-d', action='store_true', - help='Debug') - parser.add_argument('--stdout', '-s', action='store_true', - help='Write to stdout') - parser.add_argument('--force', '-f', action='store_true', - help='Force overwrite of existing files') - parser.add_argument('file', type=str, help='Python source file') - return parser - - -if __name__ == '__main__': - if sys.version_info < (3, 6, 0): - raise Exception("This script requires Python 3.6") - logging.basicConfig(level=logging.INFO) - logger = logging.getLogger(__name__) - arg_parser = create_arg_parser(DESCRIPTION) - args = arg_parser.parse_args() - ConvertVisitor.debug = args.debug - - input_file = args.file - if not os.path.isfile(input_file): - logger.error(f'{input_file} does not exist or is not a file.') - sys.exit(-1) - file_root, ext = os.path.splitext(input_file) - if ext != '.py': - logger.error(f'{input_file} does not appear to be a Python file.') - sys.exit(-1) - - ast_tree = ConvertVisitor.create_ast(input_file) - if args.stdout: - sys.stdout.write(f'// Converted from {input_file}\n') - ConvertVisitor(sys.stdout).visit(ast_tree) - sys.exit(0) - - target_file = file_root + '.cpp' - if os.path.exists(target_file): - if not os.path.isfile(target_file): - logger.error(f'{target_file} exists and is not a file.') - sys.exit(-1) - if not args.force: - logger.error(f'{target_file} exists. Use -f to overwrite.') - sys.exit(-1) - - with open(target_file, "w") as file: - file.write(f'// Converted from {input_file}\n') - ConvertVisitor(file).visit(ast_tree) - logger.info(f"Wrote {target_file} ...") diff --git a/tools/qtpy2cpp.pyproject b/tools/qtpy2cpp.pyproject deleted file mode 100644 index a9d223a4d..000000000 --- a/tools/qtpy2cpp.pyproject +++ /dev/null @@ -1,6 +0,0 @@ -{ - "files": ["qtpy2cpp.py", - "qtpy2cpp_lib/formatter.py", "qtpy2cpp_lib/visitor.py", "qtpy2cpp_lib/nodedump.py", - "qtpy2cpp_lib/astdump.py", "qtpy2cpp_lib/tokenizer.py", - "qtpy2cpp_lib/test_baseline/basic_test.py", "qtpy2cpp_lib/test_baseline/uic.py"] -} diff --git a/tools/qtpy2cpp_lib/astdump.py b/tools/qtpy2cpp_lib/astdump.py deleted file mode 100644 index ea37590c2..000000000 --- a/tools/qtpy2cpp_lib/astdump.py +++ /dev/null @@ -1,149 +0,0 @@ -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -"""Tool to dump a Python AST""" - - -from argparse import ArgumentParser, RawTextHelpFormatter -import ast -from enum import Enum -import sys -import tokenize - - -from nodedump import debug_format_node - -DESCRIPTION = "Tool to dump a Python AST" - - -_source_lines = [] -_opt_verbose = False - - -def first_non_space(s): - for i, c in enumerate(s): - if c != ' ': - return i - return 0 - - -class NodeType(Enum): - IGNORE = 1 - PRINT_ONE_LINE = 2 # Print as a one liner, do not visit children - PRINT = 3 # Print with opening closing tag, visit children - PRINT_WITH_SOURCE = 4 # Like PRINT, but print source line above - - -def get_node_type(node): - if isinstance(node, (ast.Load, ast.Store, ast.Delete)): - return NodeType.IGNORE - if isinstance(node, (ast.Add, ast.alias, ast.arg, ast.Eq, ast.Gt, ast.Lt, - ast.Mult, ast.Name, ast.NotEq, ast.NameConstant, ast.Not, - ast.Num, ast.Str)): - return NodeType.PRINT_ONE_LINE - if not hasattr(node, 'lineno'): - return NodeType.PRINT - if isinstance(node, (ast.Attribute)): - return NodeType.PRINT_ONE_LINE if isinstance(node.value, ast.Name) else NodeType.PRINT - return NodeType.PRINT_WITH_SOURCE - - -class DumpVisitor(ast.NodeVisitor): - def __init__(self): - ast.NodeVisitor.__init__(self) - self._indent = 0 - self._printed_source_lines = {-1} - - def generic_visit(self, node): - node_type = get_node_type(node) - if _opt_verbose and node_type in (NodeType.IGNORE, NodeType.PRINT_ONE_LINE): - node_type = NodeType.PRINT - if node_type == NodeType.IGNORE: - return - self._indent = self._indent + 1 - indent = ' ' * self._indent - - if node_type == NodeType.PRINT_WITH_SOURCE: - line_number = node.lineno - 1 - if line_number not in self._printed_source_lines: - self._printed_source_lines.add(line_number) - line = _source_lines[line_number] - non_space = first_non_space(line) - print('{:04d} {}{}'.format(line_number, '_' * non_space, - line[non_space:])) - - if node_type == NodeType.PRINT_ONE_LINE: - print(indent, debug_format_node(node)) - else: - print(indent, '>', debug_format_node(node)) - ast.NodeVisitor.generic_visit(self, node) - print(indent, '<', type(node).__name__) - - self._indent = self._indent - 1 - - -def parse_ast(filename): - node = None - with tokenize.open(filename) as f: - global _source_lines - source = f.read() - _source_lines = source.split('\n') - node = ast.parse(source, mode="exec") - return node - - -def create_arg_parser(desc): - parser = ArgumentParser(description=desc, - formatter_class=RawTextHelpFormatter) - parser.add_argument('--verbose', '-v', action='store_true', - help='Verbose') - parser.add_argument('source', type=str, help='Python source') - return parser - - -if __name__ == '__main__': - arg_parser = create_arg_parser(DESCRIPTION) - options = arg_parser.parse_args() - _opt_verbose = options.verbose - title = f'AST tree for {options.source}' - print('=' * len(title)) - print(title) - print('=' * len(title)) - tree = parse_ast(options.source) - DumpVisitor().visit(tree) diff --git a/tools/qtpy2cpp_lib/formatter.py b/tools/qtpy2cpp_lib/formatter.py deleted file mode 100644 index a1e8c69db..000000000 --- a/tools/qtpy2cpp_lib/formatter.py +++ /dev/null @@ -1,264 +0,0 @@ -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -"""C++ formatting helper functions and formatter class""" - - -import ast -import sys - - -CLOSING = {"{": "}", "(": ")", "[": "]"} # Closing parenthesis for C++ - - -def to_string(node): - """Helper to retrieve a string from the (Lists of)Name/Attribute - aggregated into some nodes""" - if isinstance(node, ast.Name): - return node.id - if isinstance(node, ast.Attribute): - return node.attr - return '' - - -def format_inheritance(class_def_node): - """Returns inheritance specification of a class""" - result = '' - for base in class_def_node.bases: - name = to_string(base) - if name != 'object': - result += ', public ' if result else ' : public ' - result += name - return result - - -def format_for_target(target_node): - if isinstance(target_node, ast.Tuple): # for i,e in enumerate() - result = '' - for i, el in enumerate(target_node): - if i > 0: - result += ', ' - result += format_reference(el) - return result - return format_reference(target_node) - - -def format_for_loop(f_node): - """Format a for loop - This applies some heuristics to detect: - 1) "for a in [1,2])" -> "for (f: {1, 2}) {" - 2) "for i in range(5)" -> "for (i = 0; i < 5; ++i) {" - 3) "for i in range(2,5)" -> "for (i = 2; i < 5; ++i) {" - - TODO: Detect other cases, maybe including enumerate(). - """ - loop_vars = format_for_target(f_node.target) - result = 'for (' + loop_vars - if isinstance(f_node.iter, ast.Call): - f = format_reference(f_node.iter.func) - if f == 'range': - start = 0 - end = -1 - if len(f_node.iter.args) == 2: - start = format_literal(f_node.iter.args[0]) - end = format_literal(f_node.iter.args[1]) - elif len(f_node.iter.args) == 1: - end = format_literal(f_node.iter.args[0]) - result += f' = {start}; {loop_vars} < {end}; ++{loop_vars}' - elif isinstance(f_node.iter, ast.List): - # Range based for over list - result += ': ' + format_literal_list(f_node.iter) - result += ') {' - return result - - -def format_literal(node): - """Returns the value of number/string literals""" - if isinstance(node, ast.Num): - return str(node.n) - if isinstance(node, ast.Str): - # Fixme: escaping - return f'"{node.s}"' - return '' - - -def format_literal_list(l_node, enclosing='{'): - """Formats a list/tuple of number/string literals as C++ initializer list""" - result = enclosing - for i, el in enumerate(l_node.elts): - if i > 0: - result += ', ' - result += format_literal(el) - result += CLOSING[enclosing] - return result - - -def format_member(attrib_node, qualifier='auto'): - """Member access foo->member() is expressed as an attribute with - further nested Attributes/Names as value""" - n = attrib_node - result = '' - # Black magic: Guess '::' if name appears to be a class name - if qualifier == 'auto': - qualifier = '::' if n.attr[0:1].isupper() else '->' - while isinstance(n, ast.Attribute): - result = n.attr if not result else n.attr + qualifier + result - n = n.value - if isinstance(n, ast.Name) and n.id != 'self': - result = n.id + qualifier + result - return result - - -def format_reference(node, qualifier='auto'): - """Format member reference or free item""" - return node.id if isinstance(node, ast.Name) else format_member(node, qualifier) - - -def format_function_def_arguments(function_def_node): - """Formats arguments of a function definition""" - # Default values is a list of the last default values, expand - # so that indexes match - argument_count = len(function_def_node.args.args) - default_values = function_def_node.args.defaults - while len(default_values) < argument_count: - default_values.insert(0, None) - result = '' - for i, a in enumerate(function_def_node.args.args): - if result: - result += ', ' - if a.arg != 'self': - result += a.arg - if default_values[i]: - result += ' = ' - result += format_literal(default_values[i]) - return result - - -def format_start_function_call(call_node): - """Format a call of a free or member function""" - return format_reference(call_node.func) + '(' - - -def write_import(file, i_node): - """Print an import of a Qt class as #include""" - for alias in i_node.names: - if alias.name.startswith('Q'): - file.write(f'#include <{alias.name}>\n') - - -def write_import_from(file, i_node): - """Print an import from Qt classes as #include sequence""" - # "from PySide6.QtGui import QGuiApplication" or - # "from PySide6 import QtGui" - mod = i_node.module - if not mod.startswith('PySide') and not mod.startswith('PyQt'): - return - dot = mod.find('.') - qt_module = mod[dot + 1:] + '/' if dot >= 0 else '' - for i in i_node.names: - if i.name.startswith('Q'): - file.write(f'#include <{qt_module}{i.name}>\n') - - -class Indenter: - """Helper for Indentation""" - - def __init__(self, output_file): - self._indent_level = 0 - self._indentation = '' - self._output_file = output_file - - def indent_string(self, string): - """Start a new line by a string""" - self._output_file.write(self._indentation) - self._output_file.write(string) - - def indent_line(self, line): - """Write an indented line""" - self._output_file.write(self._indentation) - self._output_file.write(line) - self._output_file.write('\n') - - def INDENT(self): - """Write indentation""" - self._output_file.write(self._indentation) - - def indent(self): - """Increase indentation level""" - self._indent_level = self._indent_level + 1 - self._indentation = ' ' * self._indent_level - - def dedent(self): - """Decrease indentation level""" - self._indent_level = self._indent_level - 1 - self._indentation = ' ' * self._indent_level - - -class CppFormatter(Indenter): - """Provides helpers for formatting multi-line C++ constructs""" - - def __init__(self, output_file): - Indenter.__init__(self, output_file) - - def write_class_def(self, class_node): - """Print a class definition with inheritance""" - self._output_file.write('\n') - inherits = format_inheritance(class_node) - self.indent_line(f'class {class_node.name}{inherits}') - self.indent_line('{') - self.indent_line('public:') - - def write_function_def(self, f_node, class_context): - """Print a function definition with arguments""" - self._output_file.write('\n') - arguments = format_function_def_arguments(f_node) - warn = True - if f_node.name == '__init__' and class_context: # Constructor - name = class_context - warn = len(arguments) > 0 - elif f_node.name == '__del__' and class_context: # Destructor - name = '~' + class_context - warn = False - else: - name = 'void ' + f_node.name - self.indent_string(f'{name}({arguments})') - if warn: - self._output_file.write(' /* FIXME: types */') - self._output_file.write('\n') - self.indent_line('{') diff --git a/tools/qtpy2cpp_lib/nodedump.py b/tools/qtpy2cpp_lib/nodedump.py deleted file mode 100644 index 5cb7c3f2d..000000000 --- a/tools/qtpy2cpp_lib/nodedump.py +++ /dev/null @@ -1,86 +0,0 @@ -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -"""Helper to dump AST nodes for debugging""" - - -import ast - - -def to_string(node): - """Helper to retrieve a string from the (Lists of )Name/Attribute - aggregated into some nodes""" - if isinstance(node, ast.Name): - return node.id - if isinstance(node, ast.Attribute): - return node.attr - return '' - - -def debug_format_node(node): - """Format AST node for debugging""" - if isinstance(node, ast.alias): - return f'alias("{node.name}")' - if isinstance(node, ast.arg): - return f'arg({node.arg})' - if isinstance(node, ast.Attribute): - if isinstance(node.value, ast.Name): - nested_name = debug_format_node(node.value) - return f'Attribute("{node.attr}", {nested_name})' - return f'Attribute("{node.attr}")' - if isinstance(node, ast.Call): - return 'Call({}({}))'.format(to_string(node.func), len(node.args)) - if isinstance(node, ast.ClassDef): - base_names = [to_string(base) for base in node.bases] - bases = ': ' + ','.join(base_names) if base_names else '' - return f'ClassDef({node.name}{bases})' - if isinstance(node, ast.ImportFrom): - return f'ImportFrom("{node.module}")' - if isinstance(node, ast.FunctionDef): - arg_names = [a.arg for a in node.args.args] - return 'FunctionDef({}({}))'.format(node.name, ', '.join(arg_names)) - if isinstance(node, ast.Name): - return 'Name("{}", Ctx={})'.format(node.id, type(node.ctx).__name__) - if isinstance(node, ast.NameConstant): - return f'NameConstant({node.value})' - if isinstance(node, ast.Num): - return f'Num({node.n})' - if isinstance(node, ast.Str): - return f'Str("{node.s}")' - return type(node).__name__ diff --git a/tools/qtpy2cpp_lib/test_baseline/basic_test.py b/tools/qtpy2cpp_lib/test_baseline/basic_test.py deleted file mode 100644 index e5dc92f9f..000000000 --- a/tools/qtpy2cpp_lib/test_baseline/basic_test.py +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the test suite of Qt for Python. -## -## $QT_BEGIN_LICENSE:GPL-EXCEPT$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3 as published by the Free Software -## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -a = 7 - -if a > 5: - for f in [1, 2]: - print(f) -else: - for i in range(5): - print(i) - for i in range(2, 5): - print(i) diff --git a/tools/qtpy2cpp_lib/test_baseline/uic.py b/tools/qtpy2cpp_lib/test_baseline/uic.py deleted file mode 100644 index 73e3ca540..000000000 --- a/tools/qtpy2cpp_lib/test_baseline/uic.py +++ /dev/null @@ -1,208 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the test suite of Qt for Python. -## -## $QT_BEGIN_LICENSE:GPL-EXCEPT$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3 as published by the Free Software -## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -from PySide6.QtCore import (QCoreApplication, QMetaObject, QObject, QPoint, - QRect, QSize, QUrl, Qt) -from PySide6.QtGui import (QBrush, QColor, QConicalGradient, QFont, - QFontDatabase, QIcon, QLinearGradient, QPalette, QPainter, QPixmap, - QRadialGradient) -from PySide6.QtWidgets import * - -class Ui_ImageDialog(object): - def setupUi(self, dialog): - if dialog.objectName(): - dialog.setObjectName(u"dialog") - dialog.setObjectName(u"ImageDialog") - dialog.resize(320, 180) - self.vboxLayout = QVBoxLayout(dialog) -#ifndef Q_OS_MAC - self.vboxLayout.setSpacing(6) -#endif -#ifndef Q_OS_MAC - self.vboxLayout.setContentsMargins(9, 9, 9, 9) -#endif - self.vboxLayout.setObjectName(u"vboxLayout") - self.vboxLayout.setObjectName(u"") - self.gridLayout = QGridLayout() -#ifndef Q_OS_MAC - self.gridLayout.setSpacing(6) -#endif - self.gridLayout.setContentsMargins(1, 1, 1, 1) - self.gridLayout.setObjectName(u"gridLayout") - self.gridLayout.setObjectName(u"") - self.widthLabel = QLabel(dialog) - self.widthLabel.setObjectName(u"widthLabel") - self.widthLabel.setObjectName(u"widthLabel") - self.widthLabel.setGeometry(QRect(1, 27, 67, 22)) - self.widthLabel.setFrameShape(QFrame.NoFrame) - self.widthLabel.setFrameShadow(QFrame.Plain) - self.widthLabel.setTextFormat(Qt.AutoText) - - self.gridLayout.addWidget(self.widthLabel, 1, 0, 1, 1) - - self.heightLabel = QLabel(dialog) - self.heightLabel.setObjectName(u"heightLabel") - self.heightLabel.setObjectName(u"heightLabel") - self.heightLabel.setGeometry(QRect(1, 55, 67, 22)) - self.heightLabel.setFrameShape(QFrame.NoFrame) - self.heightLabel.setFrameShadow(QFrame.Plain) - self.heightLabel.setTextFormat(Qt.AutoText) - - self.gridLayout.addWidget(self.heightLabel, 2, 0, 1, 1) - - self.colorDepthCombo = QComboBox(dialog) - self.colorDepthCombo.setObjectName(u"colorDepthCombo") - self.colorDepthCombo.setObjectName(u"colorDepthCombo") - self.colorDepthCombo.setGeometry(QRect(74, 83, 227, 22)) - sizePolicy = QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Fixed) - sizePolicy.setHorizontalStretch(0) - sizePolicy.setVerticalStretch(0) - sizePolicy.setHeightForWidth(self.colorDepthCombo.sizePolicy().hasHeightForWidth()) - self.colorDepthCombo.setSizePolicy(sizePolicy) - self.colorDepthCombo.setInsertPolicy(QComboBox.InsertAtBottom) - - self.gridLayout.addWidget(self.colorDepthCombo, 3, 1, 1, 1) - - self.nameLineEdit = QLineEdit(dialog) - self.nameLineEdit.setObjectName(u"nameLineEdit") - self.nameLineEdit.setObjectName(u"nameLineEdit") - self.nameLineEdit.setGeometry(QRect(74, 83, 227, 22)) - sizePolicy1 = QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Fixed) - sizePolicy1.setHorizontalStretch(1) - sizePolicy1.setVerticalStretch(0) - sizePolicy1.setHeightForWidth(self.nameLineEdit.sizePolicy().hasHeightForWidth()) - self.nameLineEdit.setSizePolicy(sizePolicy1) - self.nameLineEdit.setEchoMode(QLineEdit.Normal) - - self.gridLayout.addWidget(self.nameLineEdit, 0, 1, 1, 1) - - self.spinBox = QSpinBox(dialog) - self.spinBox.setObjectName(u"spinBox") - self.spinBox.setObjectName(u"spinBox") - self.spinBox.setGeometry(QRect(74, 1, 227, 20)) - sizePolicy.setHeightForWidth(self.spinBox.sizePolicy().hasHeightForWidth()) - self.spinBox.setSizePolicy(sizePolicy) - self.spinBox.setButtonSymbols(QAbstractSpinBox.UpDownArrows) - self.spinBox.setValue(32) - self.spinBox.setMaximum(1024) - self.spinBox.setMinimum(1) - - self.gridLayout.addWidget(self.spinBox, 1, 1, 1, 1) - - self.spinBox_2 = QSpinBox(dialog) - self.spinBox_2.setObjectName(u"spinBox_2") - self.spinBox_2.setObjectName(u"spinBox_2") - self.spinBox_2.setGeometry(QRect(74, 27, 227, 22)) - sizePolicy.setHeightForWidth(self.spinBox_2.sizePolicy().hasHeightForWidth()) - self.spinBox_2.setSizePolicy(sizePolicy) - self.spinBox_2.setButtonSymbols(QAbstractSpinBox.UpDownArrows) - self.spinBox_2.setValue(32) - self.spinBox_2.setMaximum(1024) - self.spinBox_2.setMinimum(1) - - self.gridLayout.addWidget(self.spinBox_2, 2, 1, 1, 1) - - self.nameLabel = QLabel(dialog) - self.nameLabel.setObjectName(u"nameLabel") - self.nameLabel.setObjectName(u"nameLabel") - self.nameLabel.setGeometry(QRect(1, 1, 67, 20)) - self.nameLabel.setFrameShape(QFrame.NoFrame) - self.nameLabel.setFrameShadow(QFrame.Plain) - self.nameLabel.setTextFormat(Qt.AutoText) - - self.gridLayout.addWidget(self.nameLabel, 0, 0, 1, 1) - - self.colorDepthLabel = QLabel(dialog) - self.colorDepthLabel.setObjectName(u"colorDepthLabel") - self.colorDepthLabel.setObjectName(u"colorDepthLabel") - self.colorDepthLabel.setGeometry(QRect(1, 83, 67, 22)) - self.colorDepthLabel.setFrameShape(QFrame.NoFrame) - self.colorDepthLabel.setFrameShadow(QFrame.Plain) - self.colorDepthLabel.setTextFormat(Qt.AutoText) - - self.gridLayout.addWidget(self.colorDepthLabel, 3, 0, 1, 1) - - - self.vboxLayout.addLayout(self.gridLayout) - - self.spacerItem = QSpacerItem(0, 0, QSizePolicy.Minimum, QSizePolicy.Expanding) - - self.vboxLayout.addItem(self.spacerItem) - - self.hboxLayout = QHBoxLayout() -#ifndef Q_OS_MAC - self.hboxLayout.setSpacing(6) -#endif - self.hboxLayout.setContentsMargins(1, 1, 1, 1) - self.hboxLayout.setObjectName(u"hboxLayout") - self.hboxLayout.setObjectName(u"") - self.spacerItem1 = QSpacerItem(0, 0, QSizePolicy.Expanding, QSizePolicy.Minimum) - - self.hboxLayout.addItem(self.spacerItem1) - - self.okButton = QPushButton(dialog) - self.okButton.setObjectName(u"okButton") - self.okButton.setObjectName(u"okButton") - self.okButton.setGeometry(QRect(135, 1, 80, 24)) - - self.hboxLayout.addWidget(self.okButton) - - self.cancelButton = QPushButton(dialog) - self.cancelButton.setObjectName(u"cancelButton") - self.cancelButton.setObjectName(u"cancelButton") - self.cancelButton.setGeometry(QRect(221, 1, 80, 24)) - - self.hboxLayout.addWidget(self.cancelButton) - - - self.vboxLayout.addLayout(self.hboxLayout) - - QWidget.setTabOrder(self.nameLineEdit, self.spinBox) - QWidget.setTabOrder(self.spinBox, self.spinBox_2) - QWidget.setTabOrder(self.spinBox_2, self.colorDepthCombo) - QWidget.setTabOrder(self.colorDepthCombo, self.okButton) - QWidget.setTabOrder(self.okButton, self.cancelButton) - - self.retranslateUi(dialog) - self.nameLineEdit.returnPressed.connect(self.okButton.animateClick) - - QMetaObject.connectSlotsByName(dialog) - # setupUi - - def retranslateUi(self, dialog): - dialog.setWindowTitle(QCoreApplication.translate("ImageDialog", u"Create Image", None)) - self.widthLabel.setText(QCoreApplication.translate("ImageDialog", u"Width:", None)) - self.heightLabel.setText(QCoreApplication.translate("ImageDialog", u"Height:", None)) - self.nameLineEdit.setText(QCoreApplication.translate("ImageDialog", u"Untitled image", None)) - self.nameLabel.setText(QCoreApplication.translate("ImageDialog", u"Name:", None)) - self.colorDepthLabel.setText(QCoreApplication.translate("ImageDialog", u"Color depth:", None)) - self.okButton.setText(QCoreApplication.translate("ImageDialog", u"OK", None)) - self.cancelButton.setText(QCoreApplication.translate("ImageDialog", u"Cancel", None)) - # retranslateUi - diff --git a/tools/qtpy2cpp_lib/tokenizer.py b/tools/qtpy2cpp_lib/tokenizer.py deleted file mode 100644 index dee63c177..000000000 --- a/tools/qtpy2cpp_lib/tokenizer.py +++ /dev/null @@ -1,91 +0,0 @@ -############################################################################# -## -## Copyright (C) 2019 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -"""Tool to dump Python Tokens""" - - -import sys -import tokenize - - -def format_token(t): - r = repr(t) - if r.startswith('TokenInfo('): - r = r[10:] - pos = r.find("), line='") - if pos < 0: - pos = r.find('), line="') - if pos > 0: - r = r[:pos + 1] - return r - - -def first_non_space(s): - for i, c in enumerate(s): - if c != ' ': - return i - return 0 - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print("Specify file Name") - sys.exit(1) - filename = sys.argv[1] - indent_level = 0 - indent = '' - last_line_number = -1 - with tokenize.open(filename) as f: - generator = tokenize.generate_tokens(f.readline) - for t in generator: - line_number = t.start[0] - if line_number != last_line_number: - code_line = t.line.rstrip() - non_space = first_non_space(code_line) - print('{:04d} {}{}'.format(line_number, '_' * non_space, - code_line[non_space:])) - last_line_number = line_number - if t.type == tokenize.INDENT: - indent_level = indent_level + 1 - indent = ' ' * indent_level - elif t.type == tokenize.DEDENT: - indent_level = indent_level - 1 - indent = ' ' * indent_level - else: - print(' ', indent, format_token(t)) diff --git a/tools/qtpy2cpp_lib/visitor.py b/tools/qtpy2cpp_lib/visitor.py deleted file mode 100644 index d17d5f53c..000000000 --- a/tools/qtpy2cpp_lib/visitor.py +++ /dev/null @@ -1,260 +0,0 @@ -############################################################################# -## -## Copyright (C) 2020 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -"""AST visitor printing out C++""" - -import ast -import sys -import tokenize -import warnings - -from .formatter import (CppFormatter, format_for_loop, - format_function_def_arguments, format_inheritance, - format_literal, format_reference, - format_start_function_call, - write_import, write_import_from) - -from .nodedump import debug_format_node - - -class ConvertVisitor(ast.NodeVisitor, CppFormatter): - """AST visitor printing out C++ - Note on implementation: - - Any visit_XXX() overridden function should call self.generic_visit(node) - to continue visiting - - When controlling the visiting manually (cf visit_Call()), - self.visit(child) needs to be called since that dispatches to - visit_XXX(). This is usually done to prevent undesired output - for example from references of calls, etc. - """ - - debug = False - - def __init__(self, output_file): - ast.NodeVisitor.__init__(self) - CppFormatter.__init__(self, output_file) - self._class_scope = [] # List of class names - self._stack = [] # nodes - self._debug_indent = 0 - - @staticmethod - def create_ast(filename): - """Create an Abstract Syntax Tree on which a visitor can be run""" - node = None - with tokenize.open(filename) as file: - node = ast.parse(file.read(), mode="exec") - return node - - def generic_visit(self, node): - parent = self._stack[-1] if self._stack else None - if self.debug: - self._debug_enter(node, parent) - self._stack.append(node) - try: - super().generic_visit(node) - except Exception as e: - line_no = node.lineno if hasattr(node, 'lineno') else -1 - message = 'Error "{}" at line {}'.format(str(e), line_no) - warnings.warn(message) - self._output_file.write(f'\n// {message}\n') - del self._stack[-1] - if self.debug: - self._debug_leave(node) - - def visit_Add(self, node): - self.generic_visit(node) - self._output_file.write(' + ') - - def visit_Assign(self, node): - self._output_file.write('\n') - self.INDENT() - for target in node.targets: - if isinstance(target, ast.Tuple): - warnings.warn('List assignment not handled (line {}).'. - format(node.lineno)) - elif isinstance(target, ast.Subscript): - warnings.warn('Subscript assignment not handled (line {}).'. - format(node.lineno)) - else: - self._output_file.write(format_reference(target)) - self._output_file.write(' = ') - self.visit(node.value) - self._output_file.write(';\n') - - def visit_Attribute(self, node): - """Format a variable reference (cf visit_Name)""" - self._output_file.write(format_reference(node)) - - def visit_BinOp(self, node): - # Parentheses are not exposed, so, every binary operation needs to - # be enclosed by (). - self._output_file.write('(') - self.generic_visit(node) - self._output_file.write(')') - - def visit_Call(self, node): - self._output_file.write(format_start_function_call(node)) - # Manually do visit(), skip the children of func - for i, arg in enumerate(node.args): - if i > 0: - self._output_file.write(', ') - self.visit(arg) - self._output_file.write(')') - - def visit_ClassDef(self, node): - # Manually do visit() to skip over base classes - # and annotations - self._class_scope.append(node.name) - self.write_class_def(node) - self.indent() - for b in node.body: - self.visit(b) - self.dedent() - self.indent_line('};') - del self._class_scope[-1] - - def visit_Expr(self, node): - self._output_file.write('\n') - self.INDENT() - self.generic_visit(node) - self._output_file.write(';\n') - - def visit_Gt(self, node): - self.generic_visit(node) - self._output_file.write('>') - - def visit_For(self, node): - # Manually do visit() to get the indentation right. - # TODO: what about orelse? - self.indent_line(format_for_loop(node)) - self.indent() - for b in node.body: - self.visit(b) - self.dedent() - self.indent_line('}') - - def visit_FunctionDef(self, node): - class_context = self._class_scope[-1] if self._class_scope else None - self.write_function_def(node, class_context) - self.indent() - self.generic_visit(node) - self.dedent() - self.indent_line('}') - - def visit_If(self, node): - # Manually do visit() to get the indentation right. Note: - # elsif() is modelled as nested if. - self.indent_string('if (') - self.visit(node.test) - self._output_file.write(') {\n') - self.indent() - for b in node.body: - self.visit(b) - self.dedent() - self.indent_string('}') - if node.orelse: - self._output_file.write(' else {\n') - self.indent() - for b in node.orelse: - self.visit(b) - self.dedent() - self.indent_string('}') - self._output_file.write('\n') - - def visit_Import(self, node): - write_import(self._output_file, node) - - def visit_ImportFrom(self, node): - write_import_from(self._output_file, node) - - def visit_List(self, node): - # Manually do visit() to get separators right - self._output_file.write('{') - for i, el in enumerate(node.elts): - if i > 0: - self._output_file.write(', ') - self.visit(el) - self._output_file.write('}') - - def visit_Lt(self, node): - self.generic_visit(node) - self._output_file.write('<') - - def visit_Mult(self, node): - self.generic_visit(node) - self._output_file.write(' * ') - - def visit_Name(self, node): - """Format a variable reference (cf visit_Attribute)""" - self._output_file.write(format_reference(node)) - - def visit_NameConstant(self, node): - self.generic_visit(node) - if node.value is None: - self._output_file.write('nullptr') - elif not node.value: - self._output_file.write('false') - else: - self._output_file.write('true') - - def visit_Num(self, node): - self.generic_visit(node) - self._output_file.write(format_literal(node)) - - def visit_Str(self, node): - self.generic_visit(node) - self._output_file.write(format_literal(node)) - - def visit_UnOp(self, node): - self.generic_visit(node) - - def _debug_enter(self, node, parent=None): - message = '{}>generic_visit({})'.format(' ' * self ._debug_indent, - debug_format_node(node)) - if parent: - message += ', parent={}'.format(debug_format_node(parent)) - message += '\n' - sys.stderr.write(message) - self._debug_indent += 1 - - def _debug_leave(self, node): - self._debug_indent -= 1 - message = '{}<generic_visit({})\n'.format(' ' * self ._debug_indent, - type(node).__name__) - sys.stderr.write(message) diff --git a/tools/regenerate_example_resources.py b/tools/regenerate_example_resources.py new file mode 100644 index 000000000..098c58b1f --- /dev/null +++ b/tools/regenerate_example_resources.py @@ -0,0 +1,60 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +""" +regenerate_example_resources.py +=============================== + +Regenerates the QRC resource files of the PySide examples. +""" + + +import subprocess +import sys +from pathlib import Path + +RCC_COMMAND = "pyside6-rcc" +LRELEASE_COMMAND = "lrelease" + + +def prepare_linguist_example(path): + """Create the .qm files for the Linguist example which are bundled in the QRC file""" + translations_dir = path / "translations" + if not translations_dir.is_dir(): + translations_dir.mkdir(parents=True) + + for ts_file in path.glob("*.ts"): + qm_file = translations_dir / f"{ts_file.stem}.qm" + print("Regenerating ", ts_file, qm_file) + ex = subprocess.call([LRELEASE_COMMAND, ts_file, "-qm", qm_file]) + if ex != 0: + print(f"{LRELEASE_COMMAND} failed for {ts_file}", file=sys.stderr) + sys.exit(ex) + + +def generate_rc_file(qrc_file): + """Regenerate the QRC resource file.""" + dir = qrc_file.parent + if dir.name == "linguist": + prepare_linguist_example(dir) + + target_file = dir / f"{qrc_file.stem}_rc.py" + if not target_file.is_file(): # prefix naming convention + target_file2 = qrc_file.parent / f"rc_{qrc_file.stem}.py" + if target_file2.is_file(): + target_file = target_file2 + if not target_file.is_file(): + print(target_file, " does not exist.", file=sys.stderr) + return + + print("Regenerating ", qrc_file, target_file) + ex = subprocess.call([RCC_COMMAND, qrc_file, "-o", target_file]) + if ex != 0: + print(f"{RCC_COMMAND} failed for {qrc_file}", file=sys.stderr) + sys.exit(ex) + + +if __name__ == '__main__': + examples_path = Path(__file__).resolve().parent.parent / "examples" + for qrc_file in examples_path.glob("**/*.qrc"): + generate_rc_file(qrc_file) diff --git a/tools/regenerate_example_ui.py b/tools/regenerate_example_ui.py new file mode 100644 index 000000000..2e0881c07 --- /dev/null +++ b/tools/regenerate_example_ui.py @@ -0,0 +1,36 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +""" +regenerate_example_ui.py +======================== + +Regenerates the ui files of the PySide examples. +""" + + +import subprocess +import sys +from pathlib import Path + +UIC_COMMAND = "pyside6-uic" + + +def generate_ui_file(ui_file): + """Regenerate the ui file.""" + target_file = ui_file.parent / f"ui_{ui_file.stem}.py" + if not target_file.is_file(): + print(target_file, " does not exist.", file=sys.stderr) + return + + print("Regenerating ", ui_file, target_file) + ex = subprocess.call([UIC_COMMAND, ui_file, "-o", target_file]) + if ex != 0: + print(f"{UIC_COMMAND} failed for {ui_file}", file=sys.stderr) + sys.exit(ex) + + +if __name__ == '__main__': + examples_path = Path(__file__).resolve().parent.parent / "examples" + for ui_file in examples_path.glob("**/*.ui"): + generate_ui_file(ui_file) diff --git a/tools/scanqtclasses.py b/tools/scanqtclasses.py new file mode 100644 index 000000000..0f87d80bd --- /dev/null +++ b/tools/scanqtclasses.py @@ -0,0 +1,122 @@ +# Copyright (C) 2024 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +from pathlib import Path +import os +import re +import subprocess +import sys + +"""Scan the Qt C++ headers per module for classes that should be present + in the matching type system and print the missing classes.""" + + +VALUE_TYPE = re.compile(r'^\s*<value-type name="([^"]+)"') + + +OBJECT_TYPE = re.compile(r'^\s*<object-type name="([^"]+)"') + + +def query_qtpaths(keyword): + """Query qtpaths for a keyword.""" + query_cmd = ["qtpaths", "-query", keyword] + output = subprocess.check_output(query_cmd, stderr=subprocess.STDOUT, + universal_newlines=True) + return output.strip() + + +def is_class_exluded(name): + """Check for excluded classes that do not make sense in a typesystem.""" + if len(name) < 2: + return True + if "Iterator" in name or "iterator" in name: + return True + if name.startswith("If") or name.startswith("Is") or name.startswith("When"): + return True + if name[:1].islower(): + return True + if name.startswith("QOpenGLFunctions") and name.endswith("Backend"): + return True + return False + + +def class_from_header_line(line): + """Extract a class name from a C++ header line.""" + def _is_macro(token): + return "EXPORT" in token or "API" in token + + def _fix_class_name(name): + pos = name.find('<') # Some template specialization "class Name<TemplateParam>" + if pos > 0: + name = name[:pos] + if name.endswith(':'): + name = name[:-1] + return name + + if line.startswith('//') or line.endswith(';'): # comment/forward decl + return None + line = line.strip() + if not line.startswith("class ") and not line.startswith("struct "): + return None + tokens = line.split() + pos = 1 + while pos < len(tokens) and _is_macro(tokens[pos]): + pos += 1 + return _fix_class_name(tokens[pos]) if pos < len(tokens) else None + + +def classes_from_header(header): + """Extract classes from C++ header file.""" + result = [] + for line in header.read_text("utf-8").splitlines(): + name = class_from_header_line(line) + if name and not is_class_exluded(name): + result.append(name) + return sorted(result) + + +def classes_from_typesystem(typesystem): + """Extract classes from typesystem XML file.""" + result = [] + for line in typesystem.read_text("utf-8").splitlines(): + match = VALUE_TYPE.search(line) or OBJECT_TYPE.search(line) + if match: + result.append(match.group(1)) + return sorted(result) + + +def check_classes(qt_module_inc_dir, pyside_dir): + """Check classes of a module.""" + module_name = qt_module_inc_dir.name + sys.stderr.write(f"Checking {module_name} ") + cpp_classes = [] + typesystem_classes = [] + for header in qt_module_inc_dir.glob("q*.h"): + if not header.name.endswith("_p.h"): + cpp_classes.extend(classes_from_header(header)) + for typesystem in pyside_dir.glob("*.xml"): + typesystem_classes.extend(classes_from_typesystem(typesystem)) + + cpp_count = len(cpp_classes) + typesystem_count = len(typesystem_classes) + sys.stderr.write(f"found {cpp_count} C++ / {typesystem_count} typesystem classes") + if cpp_count <= typesystem_count: + sys.stderr.write(" ok\n") + else: + sys.stderr.write(f", {cpp_count-typesystem_count} missing\n") + for cpp_class in cpp_classes: + if cpp_class not in typesystem_classes: + wrapper_name = cpp_class.lower() + "_wrapper.cpp" + print(f"{module_name}:{cpp_class}:{wrapper_name}") + + +if __name__ == '__main__': + qt_version = query_qtpaths("QT_VERSION") + qt_inc_dir = Path(query_qtpaths("QT_INSTALL_HEADERS")) + print(f"Qt {qt_version} at {os.fspath(qt_inc_dir.parent)}", file=sys.stderr) + + dir = Path(__file__).parents[1].resolve() + for module_dir in (dir / "sources" / "pyside6" / "PySide6").glob("Qt*"): + qt_module_inc_dir = qt_inc_dir / module_dir.name + if qt_module_inc_dir.is_dir(): + check_classes(qt_module_inc_dir, module_dir) diff --git a/tools/snippets_translate/README.md b/tools/snippets_translate/README.md index 9e1a5a949..8d9ab86f8 100644 --- a/tools/snippets_translate/README.md +++ b/tools/snippets_translate/README.md @@ -11,7 +11,7 @@ Here's an explanation for each file: * `main.py`, main file that handle the arguments, the general process of copying/writing files into the pyside-setup/ repository. * `converter.py`, main function that translate each line depending - of the decision making process that use different handlers. + on the decision-making process that use different handlers. * `handlers.py`, functions that handle the different translation cases. * `parse_utils.py`, some useful function that help the translation process. * `tests/test_converter.py`, tests cases for the converter function. @@ -20,20 +20,26 @@ Here's an explanation for each file: ``` % python main.py -h -usage: sync_snippets [-h] --qt QT_DIR --pyside PYSIDE_DIR [-w] [-v] +usage: sync_snippets [-h] --qt QT_DIR --target PYSIDE_DIR [-f DIRECTORY] [-w] [-v] [-d] [-s SINGLE_SNIPPET] [--filter FILTER_SNIPPET] optional arguments: -h, --help show this help message and exit --qt QT_DIR Path to the Qt directory (QT_SRC_DIR) - --pyside PYSIDE_DIR Path to the pyside-setup directory + --target TARGET_DIR Directory into which to generate the snippets -w, --write Actually copy over the files to the pyside-setup directory -v, --verbose Generate more output + -d, --debug Generate even more output + -s SINGLE_SNIPPET, --single SINGLE_SNIPPET + Path to a single file to be translated + -f, --directory DIRECTORY Path to a directory containing the snippets to be translated + --filter FILTER_SNIPPET + String to filter the snippets to be translated ``` For example: ``` -python main.py --qt /home/cmaureir/dev/qt6/ --pyside /home/cmaureir/dev/pyside-setup -w +python main.py --qt /home/cmaureir/dev/qt6/ --target /home/cmaureir/dev/pyside-setup -w ``` which will create all the snippet files in the pyside repository. The `-w` @@ -79,7 +85,7 @@ goes to: ### Examples -Everything that has .../examples/*/*, for example: +Everything that has .../examples/*, for example: ``` ./qtbase/examples/widgets/dialogs/licensewizard @@ -175,5 +181,3 @@ for m in modules: _out[m] = m_classes pprint(_out) ``` - -PySide2 was used to cover more classes that are not available for Qt 6.0. diff --git a/tools/snippets_translate/converter.py b/tools/snippets_translate/converter.py index 8eeaee551..d45bf277f 100644 --- a/tools/snippets_translate/converter.py +++ b/tools/snippets_translate/converter.py @@ -1,58 +1,59 @@ -############################################################################# -## -## Copyright (C) 2021 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import re - -from handlers import (handle_casts, handle_class, handle_condition, +from handlers import (handle_array_declarations, handle_casts, handle_class, handle_conditions, handle_constructor_default_values, handle_constructors, handle_cout_endl, handle_emit, - handle_for, handle_foreach, handle_inc_dec, - handle_include, handle_keywords, handle_negate, - handle_type_var_declaration, handle_void_functions, - handle_methods_return_type, handle_functions, - handle_array_declarations, handle_useless_qt_classes,) - -from parse_utils import get_indent, dstrip, remove_ref + handle_for, handle_foreach, handle_functions, + handle_inc_dec, handle_include, handle_keywords, + handle_methods_return_type, handle_negate, + handle_type_var_declaration, handle_useless_qt_classes, + handle_new, + handle_void_functions, handle_qt_connects) +from parse_utils import dstrip, get_indent, remove_ref + + +VOID_METHOD_PATTERN = re.compile(r"^ *void *[\w\_]+(::)?[\w\d\_]+\(") +QT_QUALIFIER_PATTERN = re.compile(r"Q[\w]+::") +TERNARY_OPERATOR_PATTERN = re.compile(r"^.* \? .+ : .+$") +COUT_PATTERN = re.compile("^ *(std::)?cout") +FOR_PATTERN = re.compile(r"^ *for *\(") +FOREACH_PATTERN = re.compile(r"^ *foreach *\(") +ELSE_PATTERN = re.compile(r"^ *}? *else *{?") +ELSE_REPLACEMENT_PATTERN = re.compile(r"}? *else *{?") +CLASS_PATTERN = re.compile(r"^ *class ") +STRUCT_PATTERN = re.compile(r"^ *struct ") +DELETE_PATTERN = re.compile(r"^ *delete ") +VAR1_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*\&]+(\(.*?\))? ?(?!.*=|:).*$") +VAR2_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w]+::[\w\*\&]+\(.*\)$") +VAR3_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*]+ *= *[\w\.\"\']*(\(.*?\))?") +VAR4_PATTERN = re.compile(r"\w+ = [A-Z]{1}\w+") +CONSTRUCTOR_PATTERN = re.compile(r"^ *\w+::\w+\(.*?\)") +ARRAY_VAR_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*]+\[?\]? * =? *\{") +RETURN_TYPE_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w]+::[\w\*\&]+\(.*\)$") +FUNCTION_PATTERN = re.compile(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*\&]+\(.*\)$") +ITERATOR_PATTERN = re.compile(r"(std::)?[\w]+<[\w]+>::(const_)?iterator") +SCOPE_PATTERN = re.compile(r"[\w]+::") +SWITCH_PATTERN = re.compile(r"^\s*switch\s*\(([a-zA-Z0-9_\.]+)\)\s*{.*$") +CASE_PATTERN = re.compile(r"^(\s*)case\s+([a-zA-Z0-9_:\.]+):.*$") +DEFAULT_PATTERN = re.compile(r"^(\s*)default:.*$") + + +QUALIFIERS = {"public:", "protected:", "private:", "public slots:", + "protected slots:", "private slots:", "signals:"} + + +FUNCTION_QUALIFIERS = ["virtual ", " override", "inline ", " noexcept"] + + +switch_var = None +switch_branch = 0 def snippet_translate(x): + global switch_var, switch_branch ## Cases which are not C++ ## TODO: Maybe expand this with lines that doesn't need to be translated @@ -62,22 +63,28 @@ def snippet_translate(x): ## General Rules # Remove ';' at the end of the lines - if x.endswith(";"): + has_semicolon = x.endswith(";") + if has_semicolon: x = x[:-1] # Remove lines with only '{' or '}' - if x.strip() == "{" or x.strip() == "}": + xs = x.strip() + if xs == "{" or xs == "}": return "" # Skip lines with the snippet related identifier '//!' - if x.strip().startswith("//!"): + if xs.startswith("//!"): return x # handle lines with only comments using '//' - if x.lstrip().startswith("//"): + if xs.startswith("//"): x = x.replace("//", "#", 1) return x + qt_connects = handle_qt_connects(x) + if qt_connects: + return qt_connects + # Handle "->" if "->" in x: x = x.replace("->", ".") @@ -99,7 +106,7 @@ def snippet_translate(x): # This contains an extra whitespace because of some variables # that include the string 'new' if "new " in x: - x = x.replace("new ", "") + x = handle_new(x) # Handle 'const' # Some variables/functions have the word 'const' so we explicitly @@ -141,13 +148,31 @@ def snippet_translate(x): if "throw" in x: x = handle_keywords(x, "throw", "raise") + switch_match = SWITCH_PATTERN.match(x) + if switch_match: + switch_var = switch_match.group(1) + switch_branch = 0 + return "" + + switch_match = CASE_PATTERN.match(x) + if switch_match: + indent = switch_match.group(1) + value = switch_match.group(2).replace("::", ".") + cond = "if" if switch_branch == 0 else "elif" + switch_branch += 1 + return f"{indent}{cond} {switch_var} == {value}:" + + switch_match = DEFAULT_PATTERN.match(x) + if switch_match: + indent = switch_match.group(1) + return f"{indent}else:" + # handle 'void Class::method(...)' and 'void method(...)' - if re.search(r"^ *void *[\w\_]+(::)?[\w\d\_]+\(", x): + if VOID_METHOD_PATTERN.search(x): x = handle_void_functions(x) # 'Q*::' -> 'Q*.' - # FIXME: This will break iterators, but it's a small price. - if re.search(r"Q[\w]+::", x): + if QT_QUALIFIER_PATTERN.search(x): x = x.replace("::", ".") # handle 'nullptr' @@ -155,78 +180,76 @@ def snippet_translate(x): x = x.replace("nullptr", "None") ## Special Cases Rules - + xs = x.strip() # Special case for 'main' - if x.strip().startswith("int main("): + if xs.startswith("int main("): return f'{get_indent(x)}if __name__ == "__main__":' - if x.strip().startswith("QApplication app(argc, argv)"): + if xs.startswith("QApplication app(argc, argv)"): return f"{get_indent(x)}app = QApplication([])" # Special case for 'return app.exec()' - if x.strip().startswith("return app.exec"): + if xs.startswith("return app.exec"): return x.replace("return app.exec()", "sys.exit(app.exec())") # Handle includes -> import - if x.strip().startswith("#include"): + if xs.startswith("#include"): x = handle_include(x) return dstrip(x) - if x.strip().startswith("emit "): + if xs.startswith("emit "): x = handle_emit(x) return dstrip(x) # *_cast if "_cast<" in x: x = handle_casts(x) + xs = x.strip() # Handle Qt classes that needs to be removed x = handle_useless_qt_classes(x) # Handling ternary operator - if re.search(r"^.* \? .+ : .+$", x.strip()): + if TERNARY_OPERATOR_PATTERN.search(xs): x = x.replace(" ? ", " if ") x = x.replace(" : ", " else ") + xs = x.strip() # Handle 'while', 'if', and 'else if' # line might end in ')' or ") {" - if x.strip().startswith(("while", "if", "else if", "} else if")): + if xs.startswith(("while", "if", "else if", "} else if")): x = handle_conditions(x) return dstrip(x) - elif re.search("^ *}? *else *{?", x): - x = re.sub(r"}? *else *{?", "else:", x) + elif ELSE_PATTERN.search(x): + x = ELSE_REPLACEMENT_PATTERN.sub("else:", x) return dstrip(x) # 'cout' and 'endl' - if re.search("^ *(std::)?cout", x) or ("endl" in x) or x.lstrip().startswith("qDebug()"): + if COUT_PATTERN.search(x) or ("endl" in x) or xs.startswith("qDebug()"): x = handle_cout_endl(x) return dstrip(x) # 'for' loops - if re.search(r"^ *for *\(", x.strip()): + if FOR_PATTERN.search(xs): return dstrip(handle_for(x)) # 'foreach' loops - if re.search(r"^ *foreach *\(", x.strip()): + if FOREACH_PATTERN.search(xs): return dstrip(handle_foreach(x)) # 'class' and 'structs' - if re.search(r"^ *class ", x) or re.search(r"^ *struct ", x): + if CLASS_PATTERN.search(x) or STRUCT_PATTERN.search(x): if "struct " in x: x = x.replace("struct ", "class ") return handle_class(x) # 'delete' - if re.search(r"^ *delete ", x): + if DELETE_PATTERN.search(x): return x.replace("delete", "del") - # 'public:' - if re.search(r"^public:$", x.strip()): - return x.replace("public:", "# public") - - # 'private:' - if re.search(r"^private:$", x.strip()): - return x.replace("private:", "# private") + # 'public:', etc + if xs in QUALIFIERS: + return f"# {x}".replace(":", "") # For expressions like: `Type var` # which does not contain a `= something` on the right side @@ -241,9 +264,10 @@ def snippet_translate(x): # At the end we skip methods with the form: # QStringView Message::body() # to threat them as methods. - if (re.search(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*\&]+(\(.*?\))? ?(?!.*=|:).*$", x.strip()) - and x.strip().split()[0] not in ("def", "return", "and", "or") - and not re.search(r"^[a-zA-Z0-9]+(<.*?>)? [\w]+::[\w\*\&]+\(.*\)$", x.strip()) + if (has_semicolon and VAR1_PATTERN.search(xs) + and not ([f for f in FUNCTION_QUALIFIERS if f in x]) + and xs.split()[0] not in ("def", "return", "and", "or") + and not VAR2_PATTERN.search(xs) and ("{" not in x and "}" not in x)): # FIXME: this 'if' is a hack for a function declaration with this form: @@ -260,8 +284,8 @@ def snippet_translate(x): # QSome thing = b(...) # float v = 0.1 # QSome *thing = ... - if (re.search(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*]+ *= *[\w\.\"\']*(\(.*?\))?", x.strip()) and - ("{" not in x and "}" not in x)): + if (VAR3_PATTERN.search(xs) + and ("{" not in x and "}" not in x)): left, right = x.split("=", 1) var_name = " ".join(left.strip().split()[1:]) x = f"{get_indent(x)}{remove_ref(var_name)} = {right.strip()}" @@ -271,23 +295,26 @@ def snippet_translate(x): # layout = QVBoxLayout # so we need to add '()' at the end if it's just a word # with only alpha numeric content - if re.search(r"\w+ = [A-Z]{1}\w+", x.strip()) and not x.strip().endswith(")"): - x = f"{x.rstrip()}()" + if VAR4_PATTERN.search(xs) and not xs.endswith(")"): + v = x.rstrip() + if (not v.endswith(" True") and not v.endswith(" False") + and not v.endswith(" None")): + x = f"{v}()" return dstrip(x) # For constructors, that we now the shape is: # ClassName::ClassName(...) - if re.search(r"^ *\w+::\w+\(.*?\)", x.strip()): + if CONSTRUCTOR_PATTERN.search(xs): x = handle_constructors(x) return dstrip(x) # For base object constructor: # : QWidget(parent) if ( - x.strip().startswith(": ") + xs.startswith(": ") and ("<<" not in x) and ("::" not in x) - and not x.strip().endswith(";") + and not xs.endswith(";") ): return handle_constructor_default_values(x) @@ -295,22 +322,40 @@ def snippet_translate(x): # Arrays declarations with the form: # type var_name[] = {... # type var_name {... - #if re.search(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*]+\[\] * = *\{", x.strip()): - if re.search(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*]+\[?\]? * =? *\{", x.strip()): + # if re.search(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*]+\[\] * = *\{", x.strip()): + if ARRAY_VAR_PATTERN.search(xs): x = handle_array_declarations(x) + xs = x.strip() # Methods with return type # int Class::method(...) # QStringView Message::body() - if re.search(r"^[a-zA-Z0-9]+(<.*?>)? [\w]+::[\w\*\&]+\(.*\)$", x.strip()): + if RETURN_TYPE_PATTERN.search(xs): # We just need to capture the 'method name' and 'arguments' x = handle_methods_return_type(x) + xs = x.strip() # Handling functions # By this section of the function, we cover all the other cases # So we can safely assume it's not a variable declaration - if re.search(r"^[a-zA-Z0-9]+(<.*?>)? [\w\*\&]+\(.*\)$", x.strip()): + if FUNCTION_PATTERN.search(xs): x = handle_functions(x) + xs = x.strip() + + # if it is a C++ iterator declaration, then ignore it due to dynamic typing in Python + # eg: std::vector<int> it; + # the case of iterator being used inside a for loop is already handed in handle_for(..) + # TODO: handle iterator initialization statement like it = container.begin(); + if ITERATOR_PATTERN.search(x): + x = "" + return x + + # By now all the typical special considerations of scope resolution operator should be handled + # 'Namespace*::' -> 'Namespace*.' + # TODO: In the case where a C++ class function is defined outside the class, this would be wrong + # but we do not have such a code snippet yet + if SCOPE_PATTERN.search(x): + x = x.replace("::", ".") # General return for no special cases return dstrip(x) diff --git a/tools/snippets_translate/handlers.py b/tools/snippets_translate/handlers.py index 510498a30..34e969a62 100644 --- a/tools/snippets_translate/handlers.py +++ b/tools/snippets_translate/handlers.py @@ -1,46 +1,51 @@ -############################################################################# -## -## Copyright (C) 2021 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import re import sys -from parse_utils import get_indent, dstrip, remove_ref, parse_arguments, replace_main_commas, get_qt_module_class +from parse_utils import (dstrip, get_indent, get_qt_module_class, + parse_arguments, remove_ref, replace_main_commas) + +IF_PATTERN = re.compile(r'^\s*if\s*\(') +PARENTHESES_NONEMPTY_CONTENT_PATTERN = re.compile(r"\((.+)\)") +LOCAL_INCLUDE_PATTERN = re.compile(r'"(.*)"') +GLOBAL_INCLUDE_PATTERN = re.compile(r"<(.*)>") +IF_CONDITION_PATTERN = PARENTHESES_NONEMPTY_CONTENT_PATTERN +ELSE_IF_PATTERN = re.compile(r'^\s*}?\s*else if\s*\(') +WHILE_PATTERN = re.compile(r'^\s*while\s*\(') +CAST_PATTERN = re.compile(r"[a-z]+_cast<(.*?)>\((.*?)\)") # Non greedy match of <> +ITERATOR_LOOP_PATTERN = re.compile(r"= *(.*)egin\(") +REMOVE_TEMPLATE_PARAMETER_PATTERN = re.compile("<.*>") +PARENTHESES_CONTENT_PATTERN = re.compile(r"\((.*)\)") +CONSTRUCTOR_BODY_PATTERN = re.compile(".*{ *}.*") +CONSTRUCTOR_BODY_REPLACEMENT_PATTERN = re.compile("{ *}") +CONSTRUCTOR_BASE_PATTERN = re.compile("^ *: *") +NEGATE_PATTERN = re.compile(r"!(.)") +CLASS_TEMPLATE_PATTERN = re.compile(r".*<.*>") +EMPTY_CLASS_PATTERN = re.compile(r".*{.*}") +EMPTY_CLASS_REPLACEMENT_PATTERN = re.compile(r"{.*}") +FUNCTION_BODY_PATTERN = re.compile(r"\{(.*)\}") +ARRAY_DECLARATION_PATTERN = re.compile(r"^[a-zA-Z0-9\<\>]+ ([\w\*]+) *\[?\]?") +RETURN_TYPE_PATTERN = re.compile(r"^ *[a-zA-Z0-9]+ [\w]+::([\w\*\&]+\(.*\)$)") +CAPTURE_PATTERN = re.compile(r"^ *([a-zA-Z0-9]+) ([\w\*\&]+\(.*\)$)") +USELESS_QT_CLASSES_PATTERNS = [ + re.compile(r'QLatin1StringView\(("[^"]*")\)'), + re.compile(r'QLatin1String\(("[^"]*")\)'), + re.compile(r'QString\.fromLatin1\(("[^"]*")\)'), + re.compile(r"QLatin1Char\(('[^']*')\)"), + re.compile(r'QStringLiteral\(("[^"]*")\)'), + re.compile(r'QString\.fromUtf8\(("[^"]*")\)'), + re.compile(r'u("[^"]*")_s') +] +COMMENT1_PATTERN = re.compile(r" *# *[\w\ ]+$") +COMMENT2_PATTERN = re.compile(r" *# *(.*)$") +COUT_ENDL_PATTERN = re.compile(r"cout *<<(.*)<< *.*endl") +COUT1_PATTERN = re.compile(r" *<< *") +COUT2_PATTERN = re.compile(r".*cout *<<") +COUT_ENDL2_PATTERN = re.compile(r"<< +endl") +NEW_PATTERN = re.compile(r"new +([a-zA-Z][a-zA-Z0-9_]*)") + def handle_condition(x, name): # Make sure it's not a multi line condition @@ -56,9 +61,13 @@ def handle_condition(x, name): comment = f" #{comment_content[-1]}" x = x.replace(f"//{comment_content[-1]}", "") - re_par = re.compile(r"\((.+)\)") - condition = re_par.search(x).group(1) - return f"{get_indent(x)}{name} {condition.strip()}:{comment}" + match = IF_CONDITION_PATTERN.search(x) + if match: + condition = match.group(1) + return f"{get_indent(x)}{name} {condition.strip()}:{comment}" + else: + print(f'snippets_translate: Warning "{x}" does not match condition pattern', + file=sys.stderr) return x @@ -82,35 +91,23 @@ def handle_inc_dec(x, operator): def handle_casts(x): - cast = None - re_type = re.compile(r"<(.*)>") - re_data = re.compile(r"_cast<.*>\((.*)\)") - type_name = re_type.search(x) - data_name = re_data.search(x) - - if type_name and data_name: - type_name = type_name.group(1).replace("*", "") - data_name = data_name.group(1) - new_value = f"{type_name}({data_name})" - - if "static_cast" in x: - x = re.sub(r"static_cast<.*>\(.*\)", new_value, x) - elif "dynamic_cast" in x: - x = re.sub(r"dynamic_cast<.*>\(.*\)", new_value, x) - elif "const_cast" in x: - x = re.sub(r"const_cast<.*>\(.*\)", new_value, x) - elif "reinterpret_cast" in x: - x = re.sub(r"reinterpret_cast<.*>\(.*\)", new_value, x) - elif "qobject_cast" in x: - x = re.sub(r"qobject_cast<.*>\(.*\)", new_value, x) + while True: + match = CAST_PATTERN.search(x) + if not match: + break + type_name = match.group(1).strip() + while type_name.endswith("*") or type_name.endswith("&") or type_name.endswith(" "): + type_name = type_name[:-1] + data_name = match.group(2).strip() + python_cast = f"{type_name}({data_name})" + x = x[0:match.start(0)] + python_cast + x[match.end(0):] return x def handle_include(x): if '"' in x: - re_par = re.compile(r'"(.*)"') - header = re_par.search(x) + header = LOCAL_INCLUDE_PATTERN.search(x) if header: header_name = header.group(1).replace(".h", "") module_name = header_name.replace('/', '.') @@ -120,8 +117,7 @@ def handle_include(x): # besides '"something.h"' x = "" elif "<" in x and ">" in x: - re_par = re.compile(r"<(.*)>") - name = re_par.search(x).group(1) + name = GLOBAL_INCLUDE_PATTERN.search(x).group(1) t = get_qt_module_class(name) # if it's not a Qt module or class, we discard it. if t is None: @@ -137,12 +133,11 @@ def handle_include(x): def handle_conditions(x): - x_strip = x.strip() - if x_strip.startswith("while") and "(" in x: + if WHILE_PATTERN.match(x): x = handle_condition(x, "while") - elif x_strip.startswith("if") and "(" in x: + elif IF_PATTERN.match(x): x = handle_condition(x, "if") - elif x_strip.startswith(("else if", "} else if")): + elif ELSE_IF_PATTERN.match(x): x = handle_condition(x, "else if") x = x.replace("else if", "elif") x = x.replace("::", ".") @@ -150,8 +145,7 @@ def handle_conditions(x): def handle_for(x): - re_content = re.compile(r"\((.*)\)") - content = re_content.search(x) + content = PARENTHESES_CONTENT_PATTERN.search(x) new_x = x if content: @@ -166,7 +160,7 @@ def handle_for(x): # iterators if "begin(" in x.lower() and "end(" in x.lower(): - name = re.search(r"= *(.*)egin\(", start) + name = ITERATOR_LOOP_PATTERN.search(start) iterable = None iterator = None if name: @@ -187,7 +181,7 @@ def handle_for(x): # Malformed for-loop: # for (; pixel1 > start; pixel1 -= stride) # We return the same line - if not start.strip(): + if not start.strip() or "=" not in start: return f"{get_indent(x)}{dstrip(x)}" raw_var, value = start.split("=") raw_var = raw_var.strip() @@ -242,28 +236,30 @@ def handle_for(x): elif x.count(":") > 0: iterator, iterable = content.split(":", 1) var = iterator.split()[-1].replace("&", "").strip() - new_x = f"for {remove_ref(var)} in {iterable.strip()}:" + iterable = iterable.strip() + if iterable.startswith("qAsConst(") or iterable.startswith("std::as_const("): + iterable = iterable[iterable.find("(") + 1: -1] + new_x = f"for {remove_ref(var)} in {iterable}:" return f"{get_indent(x)}{dstrip(new_x)}" def handle_foreach(x): - re_content = re.compile(r"\((.*)\)") - content = re_content.search(x) + content = PARENTHESES_CONTENT_PATTERN.search(x) if content: parenthesis = content.group(1) iterator, iterable = parenthesis.split(",", 1) # remove iterator type it = dstrip(iterator.split()[-1]) # remove <...> from iterable - value = re.sub("<.*>", "", iterable) + value = REMOVE_TEMPLATE_PARAMETER_PATTERN.sub("", iterable) return f"{get_indent(x)}for {it} in {value}:" def handle_type_var_declaration(x): # remove content between <...> if "<" in x and ">" in x: - x = " ".join(re.sub("<.*>", "", i) for i in x.split()) - content = re.search(r"\((.*)\)", x) + x = " ".join(REMOVE_TEMPLATE_PARAMETER_PATTERN.sub("", i) for i in x.split()) + content = PARENTHESES_CONTENT_PATTERN.search(x) if content: # this means we have something like: # QSome thing(...) @@ -279,8 +275,7 @@ def handle_type_var_declaration(x): def handle_constructors(x): - re_content = re.compile(r"\((.*)\)") - arguments = re_content.search(x).group(1) + arguments = PARENTHESES_CONTENT_PATTERN.search(x).group(1) class_method = x.split("(")[0].split("::") if len(class_method) == 2: # Equal 'class name' and 'method name' @@ -298,8 +293,8 @@ def handle_constructor_default_values(x): # we discard that section completely, since even with a single # value, we don't need to take care of it, for example: # ' : a(1) { } -> self.a = 1 - if re.search(".*{ *}.*", x): - x = re.sub("{ *}", "", x) + if CONSTRUCTOR_BODY_PATTERN.search(x): + x = CONSTRUCTOR_BODY_REPLACEMENT_PATTERN.sub("", x) values = "".join(x.split(":", 1)) # Check the commas that are not inside round parenthesis @@ -314,55 +309,58 @@ def handle_constructor_default_values(x): if "@" in values: return_values = "" for arg in values.split("@"): - arg = re.sub("^ *: *", "", arg).strip() + arg = CONSTRUCTOR_BASE_PATTERN.sub("", arg).strip() if arg.startswith("Q"): class_name = arg.split("(")[0] content = arg.replace(class_name, "")[1:-1] - return_values += f" {class_name}.__init__(self, {content})\n" + return_values += f" super().__init__({content})\n" elif arg: var_name = arg.split("(")[0] - re_par = re.compile(r"\((.+)\)") - content = re_par.search(arg).group(1) + content = PARENTHESES_NONEMPTY_CONTENT_PATTERN.search(arg).group(1) return_values += f" self.{var_name} = {content}\n" else: - arg = re.sub("^ *: *", "", values).strip() + arg = CONSTRUCTOR_BASE_PATTERN.sub("", values).strip() if arg.startswith("Q"): class_name = arg.split("(")[0] content = arg.replace(class_name, "")[1:-1] - return f" {class_name}.__init__(self, {content})" + return f" super().__init__({content})" elif arg: var_name = arg.split("(")[0] - re_par = re.compile(r"\((.+)\)") - content = re_par.search(arg).group(1) - return f" self.{var_name} = {content}" - + match = PARENTHESES_NONEMPTY_CONTENT_PATTERN.search(arg) + if match: + content = match.group(1) + return f" self.{var_name} = {content}" + else: + print(f'snippets_translate: Warning "{arg}" does not match pattern', + file=sys.stderr) + return "" return return_values.rstrip() def handle_cout_endl(x): # if comment at the end comment = "" - if re.search(r" *# *[\w\ ]+$", x): - comment = f' # {re.search(" *# *(.*)$", x).group(1)}' + if COMMENT1_PATTERN.search(x): + match = COMMENT2_PATTERN.search(x).group(1) + comment = f' # {match}' x = x.split("#")[0] if "qDebug()" in x: x = x.replace("qDebug()", "cout") if "cout" in x and "endl" in x: - re_cout_endl = re.compile(r"cout *<<(.*)<< *.*endl") - data = re_cout_endl.search(x) + data = COUT_ENDL_PATTERN.search(x) if data: data = data.group(1) - data = re.sub(" *<< *", ", ", data) + data = COUT1_PATTERN.sub(", ", data) x = f"{get_indent(x)}print({data}){comment}" elif "cout" in x: - data = re.sub(".*cout *<<", "", x) - data = re.sub(" *<< *", ", ", data) + data = COUT2_PATTERN.sub("", x) + data = COUT1_PATTERN.sub(", ", data) x = f"{get_indent(x)}print({data}){comment}" elif "endl" in x: - data = re.sub("<< +endl", "", x) - data = re.sub(" *<< *", ", ", data) + data = COUT_ENDL2_PATTERN.sub("", x) + data = COUT1_PATTERN.sub(", ", data) x = f"{get_indent(x)}print({data}){comment}" x = x.replace("( ", "(").replace(" )", ")").replace(" ,", ",").replace("(, ", "(") @@ -378,8 +376,7 @@ def handle_negate(x): elif "/*" in x: if x.index("/*") < x.index("!"): return x - re_negate = re.compile(r"!(.)") - next_char = re_negate.search(x).group(1) + next_char = NEGATE_PATTERN.search(x).group(1) if next_char not in ("=", '"'): x = x.replace("!", "not ") return x @@ -387,8 +384,7 @@ def handle_negate(x): def handle_emit(x): function_call = x.replace("emit ", "").strip() - re_content = re.compile(r"\((.*)\)") - match = re_content.search(function_call) + match = PARENTHESES_CONTENT_PATTERN.search(function_call) if not match: stmt = x.strip() print(f'snippets_translate: Warning "{stmt}" does not match function call', @@ -409,16 +405,16 @@ def handle_void_functions(x): method_name = class_method.strip() # if the arguments are in the same line: + arguments = None if ")" in x: - re_content = re.compile(r"\((.*)\)") - parenthesis = re_content.search(x).group(1) + parenthesis = PARENTHESES_CONTENT_PATTERN.search(x).group(1) arguments = dstrip(parse_arguments(parenthesis)) elif "," in x: arguments = dstrip(parse_arguments(x.split("(")[-1])) # check if includes a '{ ... }' after the method signature after_signature = x.split(")")[-1] - re_decl = re.compile(r"\{(.*)\}").search(after_signature) + re_decl = FUNCTION_BODY_PATTERN.search(after_signature) extra = "" if re_decl: extra = re_decl.group(1) @@ -454,13 +450,13 @@ def handle_class(x): bases_name = "" # Check if the class_name is templated, then remove it - if re.search(r".*<.*>", class_name): + if CLASS_TEMPLATE_PATTERN.search(class_name): class_name = class_name.split("<")[0] # Special case: invalid notation for an example: # class B() {...} -> clas B(): pass - if re.search(r".*{.*}", class_name): - class_name = re.sub(r"{.*}", "", class_name).rstrip() + if EMPTY_CLASS_PATTERN.search(class_name): + class_name = EMPTY_CLASS_REPLACEMENT_PATTERN.sub("", class_name).rstrip() return f"{class_name}(): pass" # Special case: check if the line ends in ',' @@ -474,23 +470,22 @@ def handle_class(x): else: return x + def handle_array_declarations(x): - re_varname = re.compile(r"^[a-zA-Z0-9\<\>]+ ([\w\*]+) *\[?\]?") - content = re_varname.search(x.strip()) + content = ARRAY_DECLARATION_PATTERN.search(x.strip()) if content: var_name = content.group(1) rest_line = "".join(x.split("{")[1:]) x = f"{get_indent(x)}{var_name} = {{{rest_line}" return x + def handle_methods_return_type(x): - re_capture = re.compile(r"^ *[a-zA-Z0-9]+ [\w]+::([\w\*\&]+\(.*\)$)") - capture = re_capture.search(x) + capture = RETURN_TYPE_PATTERN.search(x) if capture: content = capture.group(1) method_name = content.split("(")[0] - re_par = re.compile(r"\((.+)\)") - par_capture = re_par.search(x) + par_capture = PARENTHESES_NONEMPTY_CONTENT_PATTERN.search(x) arguments = "(self)" if par_capture: arguments = f"(self, {par_capture.group(1)})" @@ -499,13 +494,14 @@ def handle_methods_return_type(x): def handle_functions(x): - re_capture = re.compile(r"^ *[a-zA-Z0-9]+ ([\w\*\&]+\(.*\)$)") - capture = re_capture.search(x) + capture = CAPTURE_PATTERN.search(x) if capture: - content = capture.group(1) + return_type = capture.group(1) + if return_type == "return": # "return QModelIndex();" + return x + content = capture.group(2) function_name = content.split("(")[0] - re_par = re.compile(r"\((.+)\)") - par_capture = re_par.search(x) + par_capture = PARENTHESES_NONEMPTY_CONTENT_PATTERN.search(x) arguments = "" if par_capture: for arg in par_capture.group(1).split(","): @@ -516,11 +512,85 @@ def handle_functions(x): x = f"{get_indent(x)}def {function_name}({dstrip(arguments)}):" return x + def handle_useless_qt_classes(x): - _classes = ("QLatin1String", "QLatin1Char") - for i in _classes: - re_content = re.compile(fr"{i}\((.*)\)") - content = re_content.search(x) - if content: - x = x.replace(content.group(0), content.group(1)) - return x + for c in USELESS_QT_CLASSES_PATTERNS: + while True: + match = c.search(x) + if match: + x = x[0:match.start()] + match.group(1) + x[match.end():] + else: + break + return x.replace('"_L1', '"').replace("u'", "'") + + +def handle_new(x): + """Parse operator new() and add parentheses were needed: + func(new Foo, new Bar(x))" -> "func(Foo(), Bar(x))""" + result = "" + last_pos = 0 + for match in NEW_PATTERN.finditer(x): + end = match.end(0) + parentheses_needed = end >= len(x) or x[end] != "(" + type_name = match.group(1) + result += x[last_pos:match.start(0)] + type_name + if parentheses_needed: + result += "()" + last_pos = end + result += x[last_pos:] + return result + + +# The code below handles pairs of instance/pointer to member functions (PMF) +# which appear in Qt in connect statements like: +# "connect(fontButton, &QAbstractButton::clicked, this, &Dialog::setFont)". +# In a first pass, these pairs are replaced by: +# "connect(fontButton.clicked, self.setFont)" to be able to handle statements +# spanning lines. A 2nd pass then checks for the presence of a connect +# statement and replaces it by: +# "fontButton.clicked.connect(self.setFont)". +# To be called right after checking for comments. + + +INSTANCE_PMF_RE = re.compile(r"&?(\w+),\s*&\w+::(\w+)") + + +CONNECT_RE = re.compile(r"^(\s*)(QObject::)?connect\(([A-Za-z0-9_\.]+),\s*") + + +def handle_qt_connects(line_in): + if not INSTANCE_PMF_RE.search(line_in): + return None + # 1st pass, "fontButton, &QAbstractButton::clicked" -> "fontButton.clicked" + + is_connect = "connect(" in line_in + line = line_in + # Remove any smart pointer access, etc in connect statements + if is_connect: + line = line.replace(".get()", "").replace(".data()", "").replace("->", ".") + last_pos = 0 + result = "" + for match in INSTANCE_PMF_RE.finditer(line): + instance = match.group(1) + if instance == "this": + instance = "self" + member_fun = match.group(2) + next_pos = match.start() + result += line[last_pos:next_pos] + last_pos = match.end() + result += f"{instance}.{member_fun}" + result += line[last_pos:] + + if not is_connect: + return result + + # 2nd pass, reorder connect. + connect_match = CONNECT_RE.match(result) + if not connect_match: + return result + + space = connect_match.group(1) + signal_ = connect_match.group(3) + connect_stmt = f"{space}{signal_}.connect(" + connect_stmt += result[connect_match.end():] + return connect_stmt diff --git a/tools/snippets_translate/main.py b/tools/snippets_translate/main.py index c5f4b9690..01ea06c5e 100644 --- a/tools/snippets_translate/main.py +++ b/tools/snippets_translate/main.py @@ -1,54 +1,32 @@ -############################################################################# -## -## Copyright (C) 2021 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -import argparse +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + import logging import os import re -import shutil import sys +from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter from enum import Enum from pathlib import Path from textwrap import dedent +from typing import Dict, List +from override import python_example_snippet_mapping from converter import snippet_translate +HELP = """Converts Qt C++ code snippets to Python snippets. + +Ways to override Snippets: + +1) Complete snippets from local files: + To replace snippet "[1]" of "foo/bar.cpp", create a file + "sources/pyside6/doc/snippets/foo/bar_1.cpp.py" . +2) Snippets extracted from Python examples: + To use snippets from Python examples, add markers ("#! [id]") to it + and an entry to _PYTHON_EXAMPLE_SNIPPET_MAPPING. +""" + + # Logger configuration try: from rich.logging import RichHandler @@ -73,9 +51,14 @@ log = logging.getLogger("snippets_translate") # Filter and paths configuration SKIP_END = (".pro", ".pri", ".cmake", ".qdoc", ".yaml", ".frag", ".qsb", ".vert", "CMakeLists.txt") SKIP_BEGIN = ("changes-", ".") -OUT_MAIN = Path("sources/pyside6/doc/codesnippets/") -OUT_SNIPPETS = OUT_MAIN / "doc/src/snippets/" -OUT_EXAMPLES = OUT_MAIN / "doc/codesnippets/examples/" +CPP_SNIPPET_PATTERN = re.compile(r"//! ?\[([^]]+)\]") +PYTHON_SNIPPET_PATTERN = re.compile(r"#! ?\[([^]]+)\]") + +ROOT_PATH = Path(__file__).parents[2] +SOURCE_PATH = ROOT_PATH / "sources" / "pyside6" / "doc" / "snippets" + + +OVERRIDDEN_SNIPPET = "# OVERRIDDEN_SNIPPET" class FileStatus(Enum): @@ -83,9 +66,14 @@ class FileStatus(Enum): New = 1 -def get_parser(): - parser = argparse.ArgumentParser(prog="snippets_translate") - # List pyproject files +def get_parser() -> ArgumentParser: + """ + Returns a parser for the command line arguments of the script. + See README.md for more information. + """ + parser = ArgumentParser(prog="snippets_translate", + description=HELP, + formatter_class=RawDescriptionHelpFormatter) parser.add_argument( "--qt", action="store", @@ -95,11 +83,11 @@ def get_parser(): ) parser.add_argument( - "--pyside", + "--target", action="store", - dest="pyside_dir", + dest="target_dir", required=True, - help="Path to the pyside-setup directory", + help="Directory into which to generate the snippets", ) parser.add_argument( @@ -135,6 +123,14 @@ def get_parser(): ) parser.add_argument( + "-f", + "--directory", + action="store", + dest="single_directory", + help="Path to a single directory to be translated", + ) + + parser.add_argument( "--filter", action="store", dest="filter_snippet", @@ -144,7 +140,7 @@ def get_parser(): def is_directory(directory): - if not os.path.isdir(directory): + if not directory.is_dir(): log.error(f"Path '{directory}' is not a directory") return False return True @@ -156,7 +152,7 @@ def check_arguments(options): if options.write_files: if not opt_quiet: log.warning( - f"Files will be copied from '{options.qt_dir}':\n" f"\tto '{options.pyside_dir}'" + f"Files will be copied from '{options.qt_dir}':\n" f"\tto '{options.target_dir}'" ) else: msg = "This is a listing only, files are not being copied" @@ -165,11 +161,8 @@ def check_arguments(options): if not opt_quiet: log.info(msg, extra=extra) - # Check 'qt_dir' and 'pyside_dir' - if is_directory(options.qt_dir) and is_directory(options.pyside_dir): - return True - - return False + # Check 'qt_dir' + return is_directory(Path(options.qt_dir)) def is_valid_file(x): @@ -191,58 +184,154 @@ def is_valid_file(x): return True -def get_snippets(data): - snippet_lines = "" - is_snippet = False - snippets = [] - for line in data: - if not is_snippet and line.startswith("//! ["): - snippet_lines = line - is_snippet = True - elif is_snippet: - snippet_lines = f"{snippet_lines}\n{line}" - if line.startswith("//! ["): - is_snippet = False - snippets.append(snippet_lines) - # Special case when a snippet line is: - # //! [1] //! [2] - if line.count("//!") > 1: - snippet_lines = "" - is_snippet = True - return snippets +def get_snippet_ids(line: str, pattern: re.Pattern) -> List[str]: + # Extract the snippet ids for a line '//! [1] //! [2]' + result = [] + for m in pattern.finditer(line): + result.append(m.group(1)) + return result + + +def overriden_snippet_lines(lines: List[str], start_id: str) -> List[str]: + """Wrap an overridden snippet with marker and id lines.""" + id_string = f"//! [{start_id}]" + result = [OVERRIDDEN_SNIPPET, id_string] + result.extend(lines) + result.append(id_string) + return result + + +def get_snippet_override(start_id: str, rel_path: str) -> List[str]: + """Check if the snippet is overridden by a local file under + sources/pyside6/doc/snippets.""" + file_start_id = start_id.replace(' ', '_') + override_name = f"{rel_path.stem}_{file_start_id}{rel_path.suffix}.py" + override_path = SOURCE_PATH / rel_path.parent / override_name + if not override_path.is_file(): + return [] + lines = override_path.read_text().splitlines() + return overriden_snippet_lines(lines, start_id) + + +def _get_snippets(lines: List[str], + comment: str, + pattern: re.Pattern) -> Dict[str, List[str]]: + """Helper to extract (potentially overlapping) snippets from a C++ file + indicated by pattern ("//! [1]") and return them as a dict by <id>.""" + snippets: Dict[str, List[str]] = {} + snippet: List[str] + done_snippets : List[str] = [] + + i = 0 + while i < len(lines): + line = lines[i] + i += 1 + + start_ids = get_snippet_ids(line, pattern) + while start_ids: + # Start of a snippet + start_id = start_ids.pop(0) + if start_id in done_snippets: + continue + # Reconstruct a single ID line to avoid repetitive ID lines + # by consecutive snippets with multi-ID lines like "//! [1] [2]" + id_line = f"{comment}! [{start_id}]" + done_snippets.append(start_id) + snippet = [id_line] # The snippet starts with this id + + # Find the end of the snippet + j = i + while j < len(lines): + l = lines[j] + j += 1 + + # Add the line to the snippet + snippet.append(l) + + # Check if the snippet is complete + if start_id in get_snippet_ids(l, pattern): + # End of snippet + snippet[len(snippet) - 1] = id_line + snippets[start_id] = snippet + break + + return snippets -def get_license_from_file(filename): - lines = [] - with open(filename, "r") as f: - line = True - while line: - line = f.readline().rstrip() +def get_python_example_snippet_override(start_id: str, rel_path: str) -> List[str]: + """Check if the snippet is overridden by a python example snippet.""" + key = (os.fspath(rel_path), start_id) + value = python_example_snippet_mapping().get(key) + if not value: + return [] + path, id = value + file_lines = path.read_text().splitlines() + snippet_dict = _get_snippets(file_lines, '#', PYTHON_SNIPPET_PATTERN) + lines = snippet_dict.get(id) + if not lines: + raise RuntimeError(f'Snippet "{id}" not found in "{os.fspath(path)}"') + lines = lines[1:-1] # Strip Python snippet markers + return overriden_snippet_lines(lines, start_id) + + +def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]: + """Extract (potentially overlapping) snippets from a C++ file indicated + by '//! [1]'.""" + result = _get_snippets(lines, '//', CPP_SNIPPET_PATTERN) + id_list = result.keys() + for snippet_id in id_list: + # Check file overrides and example overrides + snippet = get_snippet_override(snippet_id, rel_path) + if not snippet: + snippet = get_python_example_snippet_override(snippet_id, rel_path) + if snippet: + result[snippet_id] = snippet + + return result.values() + + +def get_license_from_file(lines): + result = [] + spdx = len(lines) >= 2 and lines[0].startswith("//") and "SPDX" in lines[1] + if spdx: # SPDX, 6.4 + for line in lines: + if line.startswith("//"): + result.append("# " + line[3:]) + else: + break + else: # Old style, C-Header, 6.2 + for line in lines: if line.startswith("/*") or line.startswith("**"): - lines.append(line) + result.append(line) # End of the comment if line.endswith("*/"): break - if lines: - # We know we have the whole block, so we can - # perform replacements to translate the comment - lines[0] = lines[0].replace("/*", "**").replace("*", "#") - lines[-1] = lines[-1].replace("*/", "**").replace("*", "#") + if result: + # We know we have the whole block, so we can + # perform replacements to translate the comment + result[0] = result[0].replace("/*", "**").replace("*", "#") + result[-1] = result[-1].replace("*/", "**").replace("*", "#") - for i in range(1, len(lines) - 1): - lines[i] = re.sub(r"^\*\*", "##", lines[i]) + for i in range(1, len(result) - 1): + result[i] = re.sub(r"^\*\*", "##", result[i]) + return "\n".join(result) - return "\n".join(lines) - else: - return "" -def translate_file(file_path, final_path, debug, write): - with open(str(file_path)) as f: - snippets = get_snippets(f.read().splitlines()) +def translate_file(file_path, final_path, qt_path, debug, write): + lines = [] + snippets = [] + try: + with file_path.open("r", encoding="utf-8") as f: + lines = f.read().splitlines() + rel_path = file_path.relative_to(qt_path) + snippets = get_snippets(lines, rel_path) + except Exception as e: + log.error(f"Error reading {file_path}: {e}") + raise if snippets: # TODO: Get license header first - license_header = get_license_from_file(str(file_path)) + license_header = get_license_from_file(lines) if debug: if have_rich: console = Console() @@ -250,11 +339,13 @@ def translate_file(file_path, final_path, debug, write): table.add_column("C++") table.add_column("Python") - file_snippets = [] + translated_lines = [] for snippet in snippets: - lines = snippet.split("\n") - translated_lines = [] - for line in lines: + if snippet and snippet[0] == OVERRIDDEN_SNIPPET: + translated_lines.extend(snippet[1:]) + continue + + for line in snippet: if not line: continue translated_line = snippet_translate(line) @@ -268,43 +359,45 @@ def translate_file(file_path, final_path, debug, write): if not opt_quiet: print(line, translated_line) - if debug and have_rich: - if not opt_quiet: - console.print(table) - - file_snippets.append("\n".join(translated_lines)) + if debug and have_rich: + if not opt_quiet: + console.print(table) if write: # Open the final file - with open(str(final_path), "w") as out_f: + new_suffix = ".h.py" if final_path.name.endswith(".h") else ".py" + target_file = final_path.with_suffix(new_suffix) + + # Directory where the file will be placed, if it does not exists + # we create it. The option 'parents=True' will create the parents + # directories if they don't exist, and if some of them exists, + # the option 'exist_ok=True' will ignore them. + if not target_file.parent.is_dir(): + if not opt_quiet: + log.info(f"Creating directories for {target_file.parent}") + target_file.parent.mkdir(parents=True, exist_ok=True) + + with target_file.open("w", encoding="utf-8") as out_f: + out_f.write("//! [AUTO]\n\n") out_f.write(license_header) - out_f.write("\n") + out_f.write("\n\n") - for s in file_snippets: + for s in translated_lines: out_f.write(s) - out_f.write("\n\n") + out_f.write("\n") - # Rename to .py - written_file = shutil.move(str(final_path), str(final_path.with_suffix(".py"))) if not opt_quiet: - log.info(f"Written: {written_file}") + log.info(f"Written: {target_file}") else: if not opt_quiet: log.warning("No snippets were found") +def copy_file(file_path, qt_path, out_path, write=False, debug=False): -def copy_file(file_path, py_path, category, category_path, write=False, debug=False): - - if not category: - translate_file(file_path, Path("_translated.py"), debug, write) - return - # Get path after the directory "snippets" or "examples" - # and we add +1 to avoid the same directory - idx = file_path.parts.index(category) + 1 - rel_path = Path().joinpath(*file_path.parts[idx:]) - - final_path = py_path / category_path / rel_path + # Replicate the Qt path including module under the PySide snippets directory + qt_path_count = len(qt_path.parts) + final_path = out_path.joinpath(*file_path.parts[qt_path_count:]) # Check if file exists. if final_path.exists(): @@ -328,140 +421,102 @@ def copy_file(file_path, py_path, category, category_path, write=False, debug=Fa else: log.info(f"{status_msg:10s} {final_path}") - # Directory where the file will be placed, if it does not exists - # we create it. The option 'parents=True' will create the parents - # directories if they don't exist, and if some of them exists, - # the option 'exist_ok=True' will ignore them. - if write and not final_path.parent.is_dir(): - if not opt_quiet: - log.info(f"Creating directories for {final_path.parent}") - final_path.parent.mkdir(parents=True, exist_ok=True) - - # Change .cpp to .py - # TODO: - # - What do we do with .h in case both .cpp and .h exists with - # the same name? - + # Change .cpp to .py, .h to .h.py # Translate C++ code into Python code - if final_path.name.endswith(".cpp"): - translate_file(file_path, final_path, debug, write) + if final_path.name.endswith(".cpp") or final_path.name.endswith(".h"): + translate_file(file_path, final_path, qt_path, debug, write) return status -def process(options): - qt_path = Path(options.qt_dir) - py_path = Path(options.pyside_dir) +def single_directory(options, qt_path, out_path): + # Process all files in the directory + directory_path = Path(options.single_directory) + for file_path in directory_path.glob("**/*"): + if file_path.is_dir() or not is_valid_file(file_path): + continue + copy_file(file_path, qt_path, out_path, write=options.write_files, debug=options.debug) + - # (new, exists) +def single_snippet(options, qt_path, out_path): + # Process a single file + file = Path(options.single_snippet) + if is_valid_file(file): + copy_file(file, qt_path, out_path, write=options.write_files, debug=options.debug) + + +def all_modules_in_directory(options, qt_path, out_path): + """ + Process all Qt modules in the directory. Logs how many files were processed. + """ + # New files, already existing files valid_new, valid_exists = 0, 0 - # Creating directories in case they don't exist - if not OUT_SNIPPETS.is_dir(): - OUT_SNIPPETS.mkdir(parents=True) - - if not OUT_EXAMPLES.is_dir(): - OUT_EXAMPLES.mkdir(parents=True) - - if options.single_snippet: - f = Path(options.single_snippet) - if is_valid_file(f): - if "snippets" in f.parts: - status = copy_file( - f, - py_path, - "snippets", - OUT_SNIPPETS, - write=options.write_files, - debug=options.debug, - ) - elif "examples" in f.parts: - status = copy_file( - f, - py_path, - "examples", - OUT_EXAMPLES, - write=options.write_files, - debug=options.debug, - ) - else: - log.warning("Path did not contain 'snippets' nor 'examples'." - "File will not be copied over, just generated locally.") - status = copy_file( - f, - py_path, - None, - None, - write=options.write_files, - debug=options.debug, - ) + for module in qt_path.iterdir(): + module_name = module.name - else: - for i in qt_path.iterdir(): - module_name = i.name - # FIXME: remove this, since it's just for testing. - if i.name != "qtbase": + # Filter only Qt modules + if not module_name.startswith("qt"): + continue + + if not opt_quiet: + log.info(f"Module {module_name}") + + # Iterating everything + for f in module.glob("**/*.*"): + # Proceed only if the full path contain the filter string + if not is_valid_file(f): continue - # Filter only Qt modules - if not module_name.startswith("qt"): + if options.filter_snippet and options.filter_snippet not in str(f.absolute()): continue - if not opt_quiet: - log.info(f"Module {module_name}") - - # Iterating everything - for f in i.glob("**/*.*"): - if is_valid_file(f): - if options.filter_snippet: - # Proceed only if the full path contain the filter string - if options.filter_snippet not in str(f.absolute()): - continue - if "snippets" in f.parts: - status = copy_file( - f, - py_path, - "snippets", - OUT_SNIPPETS, - write=options.write_files, - debug=options.debug, - ) - elif "examples" in f.parts: - status = copy_file( - f, - py_path, - "examples", - OUT_EXAMPLES, - write=options.write_files, - debug=options.debug, - ) - - # Stats - if status == FileStatus.New: - valid_new += 1 - elif status == FileStatus.Exists: - valid_exists += 1 - if not opt_quiet: - log.info( - dedent( - f"""\ - Summary: - Total valid files: {valid_new + valid_exists} - New files: {valid_new} - Existing files: {valid_exists} - """ - ) + status = copy_file(f, qt_path, out_path, write=options.write_files, debug=options.debug) + + # Stats + if status == FileStatus.New: + valid_new += 1 + elif status == FileStatus.Exists: + valid_exists += 1 + + if not opt_quiet: + log.info( + dedent( + f"""\ + Summary: + Total valid files: {valid_new + valid_exists} + New files: {valid_new} + Existing files: {valid_exists} + """ ) + ) + + +def process_files(options: Namespace) -> None: + qt_path = Path(options.qt_dir) + out_path = Path(options.target_dir) + + # Creating directories in case they don't exist + if not out_path.is_dir(): + out_path.mkdir(parents=True) + + if options.single_directory: + single_directory(options, qt_path, out_path) + elif options.single_snippet: + single_snippet(options, qt_path, out_path) + else: + # General case: process all Qt modules in the directory + all_modules_in_directory(options, qt_path, out_path) if __name__ == "__main__": parser = get_parser() - options = parser.parse_args() - opt_quiet = False if options.verbose else True - opt_quiet = False if options.debug else opt_quiet + opt: Namespace = parser.parse_args() + opt_quiet = not (opt.verbose or opt.debug) - if not check_arguments(options): + if not check_arguments(opt): + # Error, invalid arguments parser.print_help() - sys.exit(0) + sys.exit(-1) - process(options) + process_files(opt) diff --git a/tools/snippets_translate/module_classes.py b/tools/snippets_translate/module_classes.py index 364550f26..df4c7557c 100644 --- a/tools/snippets_translate/module_classes.py +++ b/tools/snippets_translate/module_classes.py @@ -1,41 +1,5 @@ -############################################################################# -## -## Copyright (C) 2021 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only module_classes = { @@ -568,6 +532,7 @@ module_classes = { "QAccessibleEvent", "QAccessibleInterface", "QAccessibleObject", + "QAccessibleSelectionInterface", "QAccessibleStateChangeEvent", "QAccessibleTableCellInterface", "QAccessibleTableModelChangeEvent", diff --git a/tools/snippets_translate/override.py b/tools/snippets_translate/override.py new file mode 100644 index 000000000..e7623d8a5 --- /dev/null +++ b/tools/snippets_translate/override.py @@ -0,0 +1,112 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +from pathlib import Path + +ROOT_PATH = Path(__file__).parents[2] +EXAMPLES_PATH = ROOT_PATH / "examples" +TUTORIAL_EXAMPLES_PATH = ROOT_PATH / "sources" / "pyside6" / "doc" / "tutorials" + + +_PYTHON_EXAMPLE_SNIPPET_MAPPING = { + ("qtbase/examples/widgets/tutorials/modelview/1_readonly/mymodel.cpp", + "Quoting ModelView Tutorial"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "1_readonly.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/2_formatting/mymodel.cpp", + "Quoting ModelView Tutorial"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "2_formatting.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp", + "quoting mymodel_QVariant"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "2"), + ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp", + "quoting mymodel_a"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp", + "quoting mymodel_b"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "3"), + ("qtbase/examples/widgets/tutorials/modelview/4_headers/mymodel.cpp", + "quoting mymodel_c"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "4_headers.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/5_edit/mymodel.cpp", + "quoting mymodel_e"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "5_edit.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/5_edit/mymodel.cpp", + "quoting mymodel_f"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "5_edit.py", "2"), + ("qtbase/examples/widgets/tutorials/modelview/6_treeview/mainwindow.cpp", + "Quoting ModelView Tutorial"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "6_treeview.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/7_selections/mainwindow.cpp", + "quoting modelview_a"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "7_selections.py", "1"), + ("qtbase/examples/widgets/tutorials/modelview/7_selections/mainwindow.cpp", + "quoting modelview_b"): + (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "7_selections.py", "2"), + ("qtbase/src/widgets/doc/snippets/qlistview-dnd/mainwindow.cpp.cpp", "0"): + (TUTORIAL_EXAMPLES_PATH / "modelviewprogramming" / "qlistview-dnd.py", "mainwindow0") +} + + +_python_example_snippet_mapping = {} + + +def python_example_snippet_mapping(): + global _python_example_snippet_mapping + if not _python_example_snippet_mapping: + result = _PYTHON_EXAMPLE_SNIPPET_MAPPING + + qt_path = "qtbase/src/widgets/doc/snippets/simplemodel-use/main.cpp" + pyside_path = TUTORIAL_EXAMPLES_PATH / "modelviewprogramming" / "stringlistmodel.py" + for i in range(3): + snippet_id = str(i) + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtbase/src/widgets/doc/snippets/stringlistmodel/main.cpp" + pyside_path = TUTORIAL_EXAMPLES_PATH / "modelviewprogramming" / "stringlistmodel.py" + for i in range(6): + snippet_id = str(i) + result[(qt_path, snippet_id)] = pyside_path, f"main{snippet_id}" + + qt_path = "qtbase/examples/widgets/itemviews/spinboxdelegate/delegate.cpp" + pyside_path = (EXAMPLES_PATH / "widgets" / "itemviews" / "spinboxdelegate" + / "spinboxdelegate.py") + for i in range(5): + snippet_id = str(i) + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtbase/src/widgets/doc/snippets/stringlistmodel/model.cpp" + pyside_path = (TUTORIAL_EXAMPLES_PATH / "modelviewprogramming" + / "stringlistmodel.py") + for i in range(10): + snippet_id = str(i) + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtbase/src/widgets/doc/snippets/qlistview-dnd/model.cpp" + pyside_path = (TUTORIAL_EXAMPLES_PATH / "modelviewprogramming" + / "qlistview-dnd.py") + for i in range(11): + snippet_id = str(i) + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtconnectivity/examples/bluetooth/heartrate_game/devicefinder.cpp" + pyside_path = EXAMPLES_PATH / "bluetooth" / "heartrate_game" / "devicefinder.py" + for i in range(5): + snippet_id = f"devicediscovery-{i}" + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtconnectivity/examples/bluetooth/heartrate_game/devicehandler.cpp" + pyside_path = EXAMPLES_PATH / "bluetooth" / "heartrate_game" / "devicehandler.py" + for snippet_id in ["Connect-Signals-1", "Connect-Signals-2", + "Filter HeartRate service 2", "Find HRM characteristic", + "Reading value"]: + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + qt_path = "qtconnectivity/examples/bluetooth/heartrate_server/main.cpp" + pyside_path = EXAMPLES_PATH / "bluetooth" / "heartrate_server" / "heartrate_server.py" + for snippet_id in ["Advertising Data", "Start Advertising", "Service Data", + "Provide Heartbeat"]: + result[(qt_path, snippet_id)] = pyside_path, snippet_id + + _python_example_snippet_mapping = result + + return _python_example_snippet_mapping diff --git a/tools/snippets_translate/parse_utils.py b/tools/snippets_translate/parse_utils.py index d82108355..234d1b669 100644 --- a/tools/snippets_translate/parse_utils.py +++ b/tools/snippets_translate/parse_utils.py @@ -1,43 +1,8 @@ -############################################################################# -## -## Copyright (C) 2021 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import re + from module_classes import module_classes @@ -68,6 +33,7 @@ def get_qt_module_class(x): def get_indent(x): return " " * (len(x) - len(x.lstrip())) + # Remove more than one whitespace from the code, but not considering # the indentation. Also removes '&', '*', and ';' from arguments. def dstrip(x): @@ -141,4 +107,3 @@ def replace_main_commas(v): new_v += c return new_v - diff --git a/tools/snippets_translate/snippets_translate.pyproject b/tools/snippets_translate/snippets_translate.pyproject index 8016eb637..f660033c1 100644 --- a/tools/snippets_translate/snippets_translate.pyproject +++ b/tools/snippets_translate/snippets_translate.pyproject @@ -1,3 +1,4 @@ { - "files": ["main.py", "converter.py", "handlers.py", "tests/test_converter.py"] + "files": ["main.py", "converter.py", "handlers.py", "override.py", + "tests/test_converter.py", "tests/test_snippets.py"] } diff --git a/tools/snippets_translate/tests/test_converter.py b/tools/snippets_translate/tests/test_converter.py index 5656ff5e8..084cc8a6d 100644 --- a/tools/snippets_translate/tests/test_converter.py +++ b/tools/snippets_translate/tests/test_converter.py @@ -1,45 +1,14 @@ -############################################################################# -## -## Copyright (C) 2021 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of Qt for Python. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only from converter import snippet_translate as st +def multi_st(lines): + result = [st(l) for l in lines.split("\n")] + return "\n".join(result) + + def test_comments(): assert st("// This is a comment") == "# This is a comment" assert st("// double slash // inside") == "# double slash // inside" @@ -78,6 +47,7 @@ def test_and_or(): def test_while_if_elseif(): assert st("while(a)") == "while a:" assert st("if (condition){") == "if condition:" + assert st(" if (condition){") == " if condition:" assert st("} else if (a) {") == " elif a:" assert ( st("if (!m_vbo.isCreated()) // init() failed,") @@ -99,7 +69,11 @@ def test_else(): def test_new(): assert st("a = new Something(...);") == "a = Something(...)" - assert st("a = new Something") == "a = Something" + assert st("a = new Something") == "a = Something()" + assert st("foo(new X, new Y(b), new Z)") == "foo(X(), Y(b), Z())" + # Class member initialization list + assert st("m_mem(new Something(p)),") == "m_mem(Something(p))," + assert st("m_mem(new Something),") == "m_mem(Something())," def test_semicolon(): @@ -126,13 +100,36 @@ def test_cast(): st("elapsed = (elapsed + qobject_cast<QTimer*>(sender())->interval()) % 1000;") == "elapsed = (elapsed + QTimer(sender()).interval()) % 1000" ) + assert ( + st("a = qobject_cast<type*>(data) * 9 + static_cast<int>(42)") + == "a = type(data) * 9 + int(42)" + ) def test_double_colon(): assert st("Qt::Align") == "Qt.Align" assert st('QSound::play("mysounds/bells.wav");') == 'QSound.play("mysounds/bells.wav")' - # FIXME - assert st("Widget::method") == "Widget::method" + assert st("Widget::method") == "Widget.method" + + # multiline statement connect statement + # eg: connect(reply, &QNetworkReply::errorOccurred, + # this, &MyClass::slotError); + assert st("this, &MyClass::slotError);") == "self.slotError)" + + +def test_connects(): + assert ( + st("connect(button, &QPushButton::clicked, this, &MyClass::slotClicked);") + == "button.clicked.connect(self.slotClicked)" + ) + assert ( + st("connect(m_ui->button, &QPushButton::clicked, this, &MyClass::slotClicked);") + == "m_ui.button.clicked.connect(self.slotClicked)" + ) + assert ( + st("connect(button.get(), &QPushButton::clicked, this, &MyClass::slotClicked);") + == "button.clicked.connect(self.slotClicked)" + ) def test_cout_endl(): @@ -169,19 +166,25 @@ def test_cout_endl(): def test_variable_declaration(): assert st("QLabel label;") == "label = QLabel()" - assert st('QLabel label("Hello")') == 'label = QLabel("Hello")' + assert st('QLabel label("Hello");') == 'label = QLabel("Hello")' assert st("Widget w;") == "w = Widget()" assert st('QLabel *label = new QLabel("Hello");') == 'label = QLabel("Hello")' assert st('QLabel label = a_function("Hello");') == 'label = a_function("Hello")' assert st('QString a = "something";') == 'a = "something"' assert st("int var;") == "var = int()" assert st("float v = 0.1;") == "v = 0.1" - assert st("QSome<thing> var") == "var = QSome()" + assert st("QSome<thing> var;") == "var = QSome()" assert st("QQueue<int> queue;") == "queue = QQueue()" assert st("QVBoxLayout *layout = new QVBoxLayout;") == "layout = QVBoxLayout()" assert st("QPointer<QLabel> label = new QLabel;") == "label = QLabel()" assert st("QMatrix4x4 matrix;") == "matrix = QMatrix4x4()" assert st("QList<QImage> collage =") == "collage =" + assert st("bool b = true;") == "b = True" + assert st("Q3DBars *m_graph = nullptr;") == "m_graph = None" + # Do not fall for member function definitions + assert st("Q3DBars *Graph::bars() const") == "Q3DBars Graph.bars()" + # Do not fall for member function declarations + assert st("virtual Q3DBars *bars();") == "virtual Q3DBars bars()" def test_for(): @@ -223,7 +226,7 @@ def test_for(): assert st("for (QChar ch : s)") == "for ch in s:" assert ( st("for (const QByteArray &ext : " "qAsConst(extensionList))") - == "for ext in qAsConst(extensionList):" + == "for ext in extensionList:" ) assert st("for (QTreeWidgetItem *item : found) {") == "for item in found:" @@ -317,24 +320,24 @@ def test_constuctors(): def test_inheritance_init(): assert ( st(": QClass(fun(re, 1, 2), parent), a(1)") - == " QClass.__init__(self, fun(re, 1, 2), parent)\n self.a = 1" + == " super().__init__(fun(re, 1, 2), parent)\n self.a = 1" ) assert ( st(": QQmlNdefRecord(copyFooRecord(record), parent)") - == " QQmlNdefRecord.__init__(self, copyFooRecord(record), parent)" + == " super().__init__(copyFooRecord(record), parent)" ) assert ( st(" : QWidget(parent), helper(helper)") - == " QWidget.__init__(self, parent)\n self.helper = helper" + == " super().__init__(parent)\n self.helper = helper" ) - assert st(" : QWidget(parent)") == " QWidget.__init__(self, parent)" + assert st(" : QWidget(parent)") == " super().__init__(parent)" assert ( st(": a(0), bB(99), cC2(1), p_S(10),") == " self.a = 0\n self.bB = 99\n self.cC2 = 1\n self.p_S = 10" ) assert ( st(": QAbstractFileEngineIterator(nameFilters, filters), index(0) ") - == " QAbstractFileEngineIterator.__init__(self, nameFilters, filters)\n self.index = 0" + == " super().__init__(nameFilters, filters)\n self.index = 0" ) assert ( st(": m_document(doc), m_text(text)") == " self.m_document = doc\n self.m_text = text" @@ -344,7 +347,7 @@ def test_inheritance_init(): st(": option->palette.color(QPalette::Mid);") == " self.option.palette.color = QPalette.Mid" ) - assert st(": QSqlResult(driver) {}") == " QSqlResult.__init__(self, driver)" + assert st(": QSqlResult(driver) {}") == " super().__init__(driver)" def test_arrays(): @@ -362,6 +365,7 @@ def test_functions(): st("QString myDecoderFunc(const QByteArray &localFileName);") == "def myDecoderFunc(localFileName):" ) + assert st("return QModelIndex();") == "return QModelIndex()" def test_foreach(): @@ -387,9 +391,16 @@ def test_ternary_operator(): == "if not game.saveGame(json if Game.Json else Game.Binary):" ) + def test_useless_qt_classes(): assert st('result += QLatin1String("; ");') == 'result += "; "' + assert st('result += QString::fromLatin1("; ");') == 'result += "; "' + assert ( + st('result = QStringLiteral("A") + QStringLiteral("B");') + == 'result = "A" + "B"') assert st("<< QLatin1Char('\0') << endl;") == "print('\0')" + assert st('result = u"A"_s;') == 'result = "A"' + def test_special_cases(): assert ( @@ -403,7 +414,7 @@ def test_special_cases(): ) assert ( st("QObject::connect(&window1, &Window::messageSent,") - == "QObject.connect(window1, Window.messageSent," + == "window1.messageSent.connect(" ) assert st("double num;") == "num = float()" @@ -411,6 +422,8 @@ def test_special_cases(): assert st("public:") == "# public" assert st("private:") == "# private" + #iterator declaration + assert st("std::vector<int>::iterator i;") == "" # TODO: Handle the existing ones with Python equivalents # assert st("std::...") @@ -434,6 +447,35 @@ def test_lambdas(): pass +def test_switch_case(): + source = """switch (v) { +case 1: + f1(); + break; +case ClassName::EnumValue: + f2(); + break; +default: + f3(); + break; +} +""" + expected = """ +if v == 1: + f1() + break +elif v == ClassName.EnumValue: + f2() + break +else: + f3() + break + +""" + + assert multi_st(source) == expected + + def test_std_function(): # std::function<QImage(const QImage &)> scale = [](const QImage &img) { pass diff --git a/tools/snippets_translate/tests/test_snippets.py b/tools/snippets_translate/tests/test_snippets.py new file mode 100644 index 000000000..84897d815 --- /dev/null +++ b/tools/snippets_translate/tests/test_snippets.py @@ -0,0 +1,134 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +from main import _get_snippets, get_snippet_ids, CPP_SNIPPET_PATTERN + + +C_COMMENT = "//" + + +def test_stacking(): + lines = [ + "//! [A] //! [B] ", + "//! [C] //! [D] //! [E]", + "// Content", + "//! [C] //! [A] ", + "//! [B] //! [D] //! [E]", + ] + snippets = _get_snippets(lines, C_COMMENT, CPP_SNIPPET_PATTERN) + assert len(snippets) == 5 + + snippet_a = snippets["A"] + assert len(snippet_a) == 4 # A starts at line 0 and ends at line 3 + + snippet_b = snippets["B"] + assert len(snippet_b) == 5 # B starts at line 0 and ends at line 4 + + snippet_c = snippets["C"] + assert len(snippet_c) == 3 # C starts at line 1 and ends at line 3 + + snippet_d = snippets["D"] + assert len(snippet_d) == 4 # D starts at line 1 and ends at line 4 + + snippet_e = snippets["E"] + assert len(snippet_e) == 4 # E starts at line 1 and ends at line 4 + + +def test_nesting(): + lines = [ + "//! [A]", + "//! [B]", + "//! [C]", + "// Content", + "//! [A]", + "//! [C]", + "//! [B]", + ] + snippets = _get_snippets(lines, C_COMMENT, CPP_SNIPPET_PATTERN) + assert len(snippets) == 3 + + snippet_a = snippets["A"] + assert len(snippet_a) == 5 + assert snippet_a == lines[:5] + + snippet_b = snippets["B"] + assert len(snippet_b) == 6 + assert snippet_b == lines[1:] + + snippet_c = snippets["C"] + assert len(snippet_c) == 4 + assert snippet_c == lines[2:6] + + +def test_overlapping(): + a_id = "//! [A]" + b_id = "//! [B]" + lines = [ + "pretext", + a_id, + "l1", + "//! [C]", + "//! [A] //! [B]", + "l2", + "l3 // Comment", + b_id, + "posttext", + "//! [C]", + ] + snippets = _get_snippets(lines, C_COMMENT, CPP_SNIPPET_PATTERN) + assert len(snippets) == 3 + + # Simple snippet ID lines are generated + snippet_a = snippets["A"] + assert len(snippet_a) == 4 + assert snippet_a == lines[1:4] + [a_id] + + snippet_c = snippets["C"] + assert len(snippet_c) == 7 + assert snippet_c == lines[3:] + + snippet_b = snippets["B"] + assert len(snippet_b) == 4 + assert snippet_b == [b_id] + lines[5:8] + + +def test_snippets(): + a_id = "//! [A]" + b_id = "//! [B]" + + lines = [ + "pretext", + a_id, + "l1", + "//! [A] //! [B]", + "l2", + "l3 // Comment", + b_id, + "posttext" + ] + + snippets = _get_snippets(lines, C_COMMENT, CPP_SNIPPET_PATTERN) + assert len(snippets) == 2 + + snippet_a = snippets["A"] + + assert len(snippet_a) == 3 + assert snippet_a == lines[1:3] + [a_id] + + snippet_b = snippets["B"] + assert len(snippet_b) == 4 + assert snippet_b == [b_id] + lines[4:7] + + +def test_snippet_ids(): + assert get_snippet_ids("", CPP_SNIPPET_PATTERN) == [] + assert get_snippet_ids("//! ", + CPP_SNIPPET_PATTERN) == [] # Invalid id + assert get_snippet_ids("//! [some name]", + CPP_SNIPPET_PATTERN) == ["some name"] + assert get_snippet_ids("//! [some name] [some other name]", + CPP_SNIPPET_PATTERN) == ["some name"] + assert get_snippet_ids("//! [some name] //! ", + CPP_SNIPPET_PATTERN) == ["some name"] # Invalid id + assert get_snippet_ids("//! [some name] //! [some other name]", + CPP_SNIPPET_PATTERN) == ["some name", "some other name"] diff --git a/tools/uic_test.py b/tools/uic_test.py index 5f8a786a9..208536963 100644 --- a/tools/uic_test.py +++ b/tools/uic_test.py @@ -1,41 +1,5 @@ -############################################################################# -## -## Copyright (C) 2021 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the Qt for Python project. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 3 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL3 included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 3 requirements -## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 2.0 or (at your option) the GNU General -## Public license version 3 or any later version approved by the KDE Free -## Qt Foundation. The licenses are as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-2.0.html and -## https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only import os import re @@ -47,7 +11,6 @@ from pathlib import Path from textwrap import dedent from typing import Optional, Tuple - VERSION = 6 @@ -59,15 +22,15 @@ TEMP_DIR = Path(tempfile.gettempdir()) def get_class_name(file: Path) -> Tuple[Optional[str], Optional[str]]: """Return class name and widget name of UI file.""" - pattern = re.compile('^\s*<widget class="(\w+)" name="(\w+)"\s*>.*$') - for l in Path(file).read_text().splitlines(): - match = pattern.match(l) + pattern = re.compile(r'^\s*<widget class="(\w+)" name="(\w+)"\s*>.*$') + for line in Path(file).read_text().splitlines(): + match = pattern.match(line) if match: return (match.group(1), match.group(2)) return (None, None) -def test_file(file: str, uic: bool=False) -> bool: +def test_file(file: str, uic: bool = False) -> bool: """Run uic on a UI file and show the resulting UI.""" path = Path(file) (klass, name) = get_class_name(path) |