diff options
author | Christian Tismer <tismer@stackless.com> | 2021-03-20 19:24:38 +0100 |
---|---|---|
committer | Qt Cherry-pick Bot <cherrypick_bot@qt-project.org> | 2021-05-17 10:31:36 +0000 |
commit | cf6b7be1328025f2c8be9ad0fcf52cfde041d1a0 (patch) | |
tree | ba227abc5ecab509bb0cd0bc6a7382d980c73560 /sources | |
parent | f4890761767f911c4a8fb33dae088d44436a6032 (diff) |
Turn generate_pyi into a general pyi_generator tool, preparation
generate_pyi is now split into a pyi_generator in signature.lib
and the remaining stub in generate_pyi.py .
This is the first step in the transformation.
Task-number: PYSIDE-1415
Change-Id: I7b9075d3f6d8d2b8f8efa4591234f89d4bbe3e46
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
(cherry picked from commit 71a0e10ba935a2788da4962e0f06880e3d3d0d5d)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
Diffstat (limited to 'sources')
5 files changed, 317 insertions, 202 deletions
diff --git a/sources/pyside6/PySide6/support/generate_pyi.py b/sources/pyside6/PySide6/support/generate_pyi.py index f99911fa0..29754f600 100644 --- a/sources/pyside6/PySide6/support/generate_pyi.py +++ b/sources/pyside6/PySide6/support/generate_pyi.py @@ -46,25 +46,16 @@ This script generates the .pyi files for all PySide modules. import argparse import inspect -import io import logging import os -import re import sys import typing -from contextlib import contextmanager from pathlib import Path -from textwrap import dedent - -# Make sure not to get .pyc in Python2. -sourcepath = Path(__file__).parent.resolve() / (Path(__file__).stem + ".py") # Can we use forward references? USE_PEP563 = sys.version_info[:2] >= (3, 7) -indent = " " * 4 -is_py3 = sys.version_info[0] == 3 is_ci = os.environ.get("QTEST_ENVIRONMENT", "") == "ci" is_debug = is_ci or os.environ.get("QTEST_ENVIRONMENT") @@ -72,197 +63,6 @@ logging.basicConfig(level=logging.DEBUG if is_debug else logging.INFO) logger = logging.getLogger("generate_pyi") -class Writer(object): - def __init__(self, outfile): - self.outfile = outfile - self.history = [True, True] - - def print(self, *args, **kw): - # controlling too much blank lines - if self.outfile: - if args == () or args == ("",): - # Python 2.7 glitch: Empty tuples have wrong encoding. - # But we use that to skip too many blank lines: - if self.history[-2:] == [True, True]: - return - print("", file=self.outfile, **kw) - self.history.append(True) - else: - print(*args, file=self.outfile, **kw) - self.history.append(False) - - -class Formatter(Writer): - """ - Formatter is formatting the signature listing of an enumerator. - - It is written as context managers in order to avoid many callbacks. - The separation in formatter and enumerator is done to keep the - unrelated tasks of enumeration and formatting apart. - """ - def __init__(self, *args): - Writer.__init__(self, *args) - # patching __repr__ to disable the __repr__ of typing.TypeVar: - """ - def __repr__(self): - if self.__covariant__: - prefix = '+' - elif self.__contravariant__: - prefix = '-' - else: - prefix = '~' - return prefix + self.__name__ - """ - def _typevar__repr__(self): - return "typing." + self.__name__ - typing.TypeVar.__repr__ = _typevar__repr__ - - # Adding a pattern to substitute "Union[T, NoneType]" by "Optional[T]" - # I tried hard to replace typing.Optional by a simple override, but - # this became _way_ too much. - # See also the comment in layout.py . - brace_pat = build_brace_pattern(3, ",") - pattern = fr"\b Union \s* \[ \s* {brace_pat} \s*, \s* NoneType \s* \]" - replace = r"Optional[\1]" - optional_searcher = re.compile(pattern, flags=re.VERBOSE) - def optional_replacer(source): - return optional_searcher.sub(replace, str(source)) - self.optional_replacer = optional_replacer - # self.level is maintained by enum_sig.py - # self.after_enum() is a one-shot set by enum_sig.py . - # self.is_method() is true for non-plain functions. - - @contextmanager - def module(self, mod_name): - self.mod_name = mod_name - self.print("# Module", mod_name) - self.print("import PySide6") - self.print("import typing") - self.print("from PySide6.support.signature.mapping import (") - self.print(" Virtual, Missing, Invalid, Default, Instance)") - self.print() - self.print("class Object(object): pass") - self.print() - self.print("from shiboken6 import Shiboken") - self.print("Shiboken.Object = Object") - self.print() - # This line will be replaced by the missing imports postprocess. - self.print("IMPORTS") - yield - - @contextmanager - def klass(self, class_name, class_str): - spaces = indent * self.level - while "." in class_name: - class_name = class_name.split(".", 1)[-1] - class_str = class_str.split(".", 1)[-1] - self.print() - if self.level == 0: - self.print() - here = self.outfile.tell() - if self.have_body: - self.print(f"{spaces}class {class_str}:") - else: - self.print(f"{spaces}class {class_str}: ...") - yield - - @contextmanager - def function(self, func_name, signature): - if self.after_enum() or func_name == "__init__": - self.print() - key = func_name - spaces = indent * self.level - if type(signature) == type([]): - for sig in signature: - self.print(f'{spaces}@typing.overload') - self._function(func_name, sig, spaces) - else: - self._function(func_name, signature, spaces) - if func_name == "__init__": - self.print() - yield key - - def _function(self, func_name, signature, spaces): - if self.is_method() and "self" not in tuple(signature.parameters.keys()): - self.print(f'{spaces}@staticmethod') - signature = self.optional_replacer(signature) - self.print(f'{spaces}def {func_name}{signature}: ...') - - @contextmanager - def enum(self, class_name, enum_name, value): - spaces = indent * self.level - hexval = hex(value) - self.print(f"{spaces}{enum_name:25}: {class_name} = ... # {hexval}") - yield - - -def get_license_text(): - with io.open(sourcepath) as f: - lines = f.readlines() - license_line = next((lno for lno, line in enumerate(lines) - if "$QT_END_LICENSE$" in line)) - return "".join(lines[:license_line + 3]) - - -def find_imports(text): - return [imp for imp in PySide6.__all__ if imp + "." in text] - - -def generate_pyi(import_name, outpath, options): - """ - Generates a .pyi file. - """ - plainname = import_name.split(".")[-1] - outfilepath = Path(outpath) / (plainname + ".pyi") - top = __import__(import_name) - obj = getattr(top, plainname) - if not getattr(obj, "__file__", None) or Path(obj.__file__).is_dir(): - raise ModuleNotFoundError(f"We do not accept a namespace as module {plainname}") - module = sys.modules[import_name] - - outfile = io.StringIO() - fmt = Formatter(outfile) - fmt.print(get_license_text()) # which has encoding, already - need_imports = not USE_PEP563 - if USE_PEP563: - fmt.print("from __future__ import annotations") - fmt.print() - fmt.print(dedent(f'''\ - """ - This file contains the exact signatures for all functions in module - {import_name}, except for defaults which are replaced by "...". - """ - ''')) - HintingEnumerator(fmt).module(import_name) - fmt.print() - fmt.print("# eof") - # Postprocess: resolve the imports - with open(outfilepath, "w") as realfile: - wr = Writer(realfile) - outfile.seek(0) - while True: - line = outfile.readline() - if not line: - break - line = line.rstrip() - # we remove the IMPORTS marker and insert imports if needed - if line == "IMPORTS": - if need_imports: - for mod_name in find_imports(outfile.getvalue()): - imp = "PySide6." + mod_name - if imp != import_name: - wr.print("import " + imp) - wr.print("import " + import_name) - wr.print() - wr.print() - else: - wr.print(line) - logger.info(f"Generated: {outfilepath}") - if is_py3 and (options.check or is_ci): - # Python 3: We can check the file directly if the syntax is ok. - subprocess.check_output([sys.executable, outfilepath]) - - def generate_all_pyi(outpath, options): ps = os.pathsep if options.sys_path: @@ -278,6 +78,7 @@ def generate_all_pyi(outpath, options): import PySide6 from PySide6.support.signature.lib.enum_sig import HintingEnumerator from PySide6.support.signature.lib.tool import build_brace_pattern + from PySide6.support.signature.lib.pyi_generator import generate_pyi # propagate USE_PEP563 to the mapping module. # Perhaps this can be automated? @@ -317,5 +118,6 @@ if __name__ == "__main__": if outpath and not Path(outpath).exists(): os.makedirs(outpath) logger.info(f"+++ Created path {outpath}") + options._pyside_call = True generate_all_pyi(outpath, options=options) # eof diff --git a/sources/shiboken6/CMakeLists.txt b/sources/shiboken6/CMakeLists.txt index 3b7418e88..07eabd635 100644 --- a/sources/shiboken6/CMakeLists.txt +++ b/sources/shiboken6/CMakeLists.txt @@ -195,6 +195,7 @@ endif() set(shiboken_python_files "signature/lib/__init__.py" "signature/lib/enum_sig.py" + "signature/lib/pyi_generator.py" "signature/lib/tool.py" "signature/__init__.py" "signature/errorhandler.py" diff --git a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/layout.py b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/layout.py index 3ab916326..bd5bc4194 100644 --- a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/layout.py +++ b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/layout.py @@ -175,10 +175,10 @@ That information would be lost when we use the "..." convention. Note that the typing module has the remarkable expansion - Optional[T] is Variant[T, NoneType] + Optional[T] is Union[T, NoneType] We want to avoid that when generating the .pyi file. -This is done by a regex in generate_pyi.py . +This is done by a regex in pyi_generator.py . The following would work in Python 3, but this is a version-dependent hack that also won't work in Python 2 and would be _very_ complex. """ diff --git a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py new file mode 100644 index 000000000..78428282a --- /dev/null +++ b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py @@ -0,0 +1,310 @@ +# This Python file uses the following encoding: utf-8 +############################################################################# +## +## Copyright (C) 2021 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of Qt for Python. +## +## $QT_BEGIN_LICENSE:LGPL$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU Lesser General Public License Usage +## Alternatively, this file may be used under the terms of the GNU Lesser +## General Public License version 3 as published by the Free Software +## Foundation and appearing in the file LICENSE.LGPL3 included in the +## packaging of this file. Please review the following information to +## ensure the GNU Lesser General Public License version 3 requirements +## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 2.0 or (at your option) the GNU General +## Public license version 3 or any later version approved by the KDE Free +## Qt Foundation. The licenses are as published by the Free Software +## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-2.0.html and +## https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +""" +pyi_generator.py + +This script generates .pyi files for arbitrary modules. +""" + +import argparse +import io +import logging +import os +import re +import subprocess +import sys +import typing +from pathlib import Path +from contextlib import contextmanager +from textwrap import dedent + +sourcepath = Path(__file__).resolve() + +from shiboken6 import Shiboken +from shibokensupport.signature.lib.enum_sig import HintingEnumerator +from shibokensupport.signature.lib.tool import build_brace_pattern + +# Can we use forward references? +USE_PEP563 = sys.version_info[:2] >= (3, 7) + +indent = " " * 4 +is_ci = os.environ.get("QTEST_ENVIRONMENT", "") == "ci" +is_debug = is_ci or os.environ.get("QTEST_ENVIRONMENT") + +logging.basicConfig(level=logging.DEBUG if is_debug else logging.INFO) +logger = logging.getLogger("generate_pyi") + + +class Writer(object): + def __init__(self, outfile): + self.outfile = outfile + self.history = [True, True] + + def print(self, *args, **kw): + # controlling too much blank lines + if self.outfile: + if args == () or args == ("",): + # Python 2.7 glitch: Empty tuples have wrong encoding. + # But we use that to skip too many blank lines: + if self.history[-2:] == [True, True]: + return + print("", file=self.outfile, **kw) + self.history.append(True) + else: + print(*args, file=self.outfile, **kw) + self.history.append(False) + + +class Formatter(Writer): + """ + Formatter is formatting the signature listing of an enumerator. + + It is written as context managers in order to avoid many callbacks. + The separation in formatter and enumerator is done to keep the + unrelated tasks of enumeration and formatting apart. + """ + def __init__(self, *args): + Writer.__init__(self, *args) + # patching __repr__ to disable the __repr__ of typing.TypeVar: + """ + def __repr__(self): + if self.__covariant__: + prefix = '+' + elif self.__contravariant__: + prefix = '-' + else: + prefix = '~' + return prefix + self.__name__ + """ + def _typevar__repr__(self): + return "typing." + self.__name__ + typing.TypeVar.__repr__ = _typevar__repr__ + + # Adding a pattern to substitute "Union[T, NoneType]" by "Optional[T]" + # I tried hard to replace typing.Optional by a simple override, but + # this became _way_ too much. + # See also the comment in layout.py . + brace_pat = build_brace_pattern(3, ",") + pattern = fr"\b Union \s* \[ \s* {brace_pat} \s*, \s* NoneType \s* \]" + replace = r"Optional[\1]" + optional_searcher = re.compile(pattern, flags=re.VERBOSE) + def optional_replacer(source): + return optional_searcher.sub(replace, str(source)) + self.optional_replacer = optional_replacer + # self.level is maintained by enum_sig.py + # self.after_enum() is a one-shot set by enum_sig.py . + # self.is_method() is true for non-plain functions. + + @contextmanager + def module(self, mod_name): + self.mod_name = mod_name + self.print("# Module", mod_name) + self.print("import PySide6") + self.print("import typing") + self.print("from typing import Any, Callable, Dict, List, Optional, Tuple, Union") + self.print("from PySide6.support.signature.mapping import (") + self.print(" Virtual, Missing, Invalid, Default, Instance)") + self.print() + self.print("class Object(object): pass") + self.print() + self.print("from shiboken6 import Shiboken") + self.print("Shiboken.Object = Object") + self.print() + # This line will be replaced by the missing imports postprocess. + self.print("IMPORTS") + yield + + @contextmanager + def klass(self, class_name, class_str): + spaces = indent * self.level + while "." in class_name: + class_name = class_name.split(".", 1)[-1] + class_str = class_str.split(".", 1)[-1] + self.print() + if self.level == 0: + self.print() + here = self.outfile.tell() + if self.have_body: + self.print(f"{spaces}class {class_str}:") + else: + self.print(f"{spaces}class {class_str}: ...") + yield + + @contextmanager + def function(self, func_name, signature): + if self.after_enum() or func_name == "__init__": + self.print() + key = func_name + spaces = indent * self.level + if type(signature) == type([]): + for sig in signature: + self.print(f'{spaces}@typing.overload') + self._function(func_name, sig, spaces) + else: + self._function(func_name, signature, spaces) + if func_name == "__init__": + self.print() + yield key + + def _function(self, func_name, signature, spaces): + if self.is_method() and "self" not in tuple(signature.parameters.keys()): + self.print(f'{spaces}@staticmethod') + signature = self.optional_replacer(signature) + self.print(f'{spaces}def {func_name}{signature}: ...') + + @contextmanager + def enum(self, class_name, enum_name, value): + spaces = indent * self.level + hexval = hex(value) + self.print(f"{spaces}{enum_name:25}: {class_name} = ... # {hexval}") + yield + + +def get_license_text(): + with sourcepath.open() as f: + lines = f.readlines() + license_line = next((lno for lno, line in enumerate(lines) + if "$QT_END_LICENSE$" in line)) + return "".join(lines[:license_line + 3]) + + +def find_imports(text): + return [imp for imp in PySide6.__all__ if imp + "." in text] + + +def find_module(import_name, outpath): + """ + Find a module either directly by import, or use the full path, + add the path to sys.path and import then. + """ + if import_name.startswith("PySide6."): + # internal mode for generate_pyi.py + plainname = import_name.split(".")[-1] + outfilepath = Path(outpath) / (plainname + ".pyi") + return import_name, plainname, outfilepath + # we are alone in external module mode + p = Path(import_name).resolve() + if not p.exists(): + raise ValueError(f"File {import_name} does not exist.") + if not outpath: + outpath = p.parent + # temporarily add the path and do the import + sys.path.insert(0, os.fspath(p.parent)) + plainname = p.name.split(".")[0] + return plainname, plainname, outpath / (plainname + ".pyi") + + +def generate_pyi(import_name, outpath, options): + """ + Generates a .pyi file. + """ + import_name, plainname, outfilepath = find_module(import_name, outpath) + top = __import__(import_name) + obj = getattr(top, plainname) if import_name != plainname else top + if not getattr(obj, "__file__", None) or Path(obj.__file__).is_dir(): + raise ModuleNotFoundError(f"We do not accept a namespace as module {plainname}") + module = sys.modules[import_name] + + outfile = io.StringIO() + fmt = Formatter(outfile) + fmt.print(get_license_text()) # which has encoding, already + need_imports = options._pyside_call and not USE_PEP563 + if USE_PEP563: + fmt.print("from __future__ import annotations") + fmt.print() + fmt.print(dedent(f'''\ + """ + This file contains the exact signatures for all functions in module + {import_name}, except for defaults which are replaced by "...". + """ + ''')) + HintingEnumerator(fmt).module(import_name) + fmt.print() + fmt.print("# eof") + # Postprocess: resolve the imports + if options._pyside_call: + global PySide6 + import PySide6 + with open(outfilepath, "w") as realfile: + wr = Writer(realfile) + outfile.seek(0) + while True: + line = outfile.readline() + if not line: + break + line = line.rstrip() + # we remove the IMPORTS marker and insert imports if needed + if line == "IMPORTS": + if need_imports: + for mod_name in find_imports(outfile.getvalue()): + imp = "PySide6." + mod_name + if imp != import_name: + wr.print("import " + imp) + wr.print("import " + import_name) + wr.print() + wr.print() + else: + wr.print(line) + logger.info(f"Generated: {outfilepath}") + if options and options.check or is_ci: + # Python 3.7 and up: We can check the file directly if the syntax is ok. + if USE_PEP563: + subprocess.check_output([sys.executable, outfilepath]) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="This script generates the .pyi file for an arbitrary module.") + parser.add_argument("module", + help="The name of an importable module, or the full path to the binary") + parser.add_argument("--check", action="store_true", help="Test the output") + parser.add_argument("--outpath", + help="the output directory (default = binary location)") + options = parser.parse_args() + module = options.module + # XXX find a path that ends in a module and use that + outpath = options.outpath + if outpath and not Path(outpath).exists(): + os.makedirs(outpath) + logger.info(f"+++ Created path {outpath}") + options._pyside_call = False + generate_pyi(module, outpath, options=options) + +# eof diff --git a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/loader.py b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/loader.py index 84fa7fc49..e0fd9dbb0 100644 --- a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/loader.py +++ b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/loader.py @@ -151,6 +151,7 @@ def move_into_pyside_package(): put_into_package(PySide6.support.signature, parser) put_into_package(PySide6.support.signature, importhandler) put_into_package(PySide6.support.signature.lib, enum_sig) + put_into_package(PySide6.support.signature.lib, pyi_generator) from shibokensupport.signature import mapping from shibokensupport.signature import errorhandler @@ -159,6 +160,7 @@ from shibokensupport.signature import lib from shibokensupport.signature import parser from shibokensupport.signature import importhandler from shibokensupport.signature.lib import enum_sig +from shibokensupport.signature.lib import pyi_generator if "PySide6" in sys.modules: # We publish everything under "PySide6.support", again. |