Turn generate_pyi into a general pyi_generator tool, main

generate_pyi is now split into a pyi_generator in signature.lib
and the remaining stub in generate_pyi.py .

pyi_generator can create .pyi files from arbitrary modules
created with shiboken.

Fixes: PYSIDE-1415
Change-Id: I966cf9f48859185d7ecb72140b533319226e511d
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
(cherry picked from commit e9dbf86de7741a59c776c29a5821cad06a177804)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

5 files changed, 77 insertions, 46 deletions

diff --git a/sources/pyside6/PySide6/support/generate_pyi.py b/sources/pyside6/PySide6/support/generate_pyi.py
index 29754f600..dd09f72d9 100644
--- a/sources/pyside6/PySide6/support/generate_pyi.py
+++ b/sources/pyside6/PySide6/support/generate_pyi.py
@@ -1,7 +1,7 @@
 # This Python file uses the following encoding: utf-8
 #############################################################################
 ##
-## Copyright (C) 2020 The Qt Company Ltd.
+## Copyright (C) 2021 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of Qt for Python.
diff --git a/sources/shiboken6/libshiboken/signature/signature_doc.rst b/sources/shiboken6/libshiboken/signature/signature_doc.rst
index 872ed024a..70fbda714 100644
--- a/sources/shiboken6/libshiboken/signature/signature_doc.rst
+++ b/sources/shiboken6/libshiboken/signature/signature_doc.rst
@@ -327,6 +327,14 @@ This serves as an extra challenge that has a very positive effect on
 the completeness and correctness of signatures.
 
 
+pyi_generator.py
+----------------
+
+``shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py``
+has been extracted from ``generate_pyi.py``. It allows the generation of ``.pyi``
+files from arbitrary extension modules created with shiboken.
+
+
 Current Extensions
 ------------------
 
diff --git a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
index dce11249d..8249d6adf 100644
--- a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
+++ b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
@@ -74,8 +74,6 @@ class ExactEnumerator(object):
         self.fmt = formatter
         self.result_type = result_type
         self.fmt.level = 0
-        self.fmt.after_enum = self.after_enum
-        self._after_enum = False
         self.fmt.is_method = self.is_method
 
     def is_method(self):
@@ -86,9 +84,9 @@ class ExactEnumerator(object):
         tp = type(self.func)
         return tp not in (types.BuiltinFunctionType, types.FunctionType)
 
-    def after_enum(self):
-        ret = self._after_enum
-        self._after_enum = False
+    def section(self):
+        if hasattr(self.fmt, "section"):
+            self.fmt.section()
 
     def module(self, mod_name):
         __import__(mod_name)
@@ -101,8 +99,12 @@ class ExactEnumerator(object):
             self.fmt.class_name = None
             for class_name, klass in members:
                 ret.update(self.klass(class_name, klass))
+            if len(members):
+                self.section()
             for func_name, func in functions:
                 ret.update(self.function(func_name, func))
+            if len(functions):
+                self.section()
             return ret
 
     def klass(self, class_name, klass):
@@ -151,6 +153,8 @@ class ExactEnumerator(object):
                 for enum_name, enum_class_name, value in enums:
                     with self.fmt.enum(enum_class_name, enum_name, int(value)):
                         pass
+                if len(enums):
+                    self.section()
             for subclass_name, subclass in subclasses:
                 if klass == subclass:
                     # this is a side effect of the typing module for Python 2.7
@@ -159,11 +163,15 @@ class ExactEnumerator(object):
                     continue
                 ret.update(self.klass(subclass_name, subclass))
                 self.fmt.class_name = class_name
+            if len(subclasses):
+                self.section()
             ret.update(self.function("__init__", klass))
             for func_name, func in functions:
                 if func_name != "__init__":
                     ret.update(self.function(func_name, func))
             self.fmt.level -= 1
+            if len(functions):
+                self.section()
         return ret
 
     @staticmethod
diff --git a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py
index 78428282a..2d7b5102d 100644
--- a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py
+++ b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py
@@ -52,16 +52,17 @@ import re
 import subprocess
 import sys
 import typing
+
 from pathlib import Path
 from contextlib import contextmanager
 from textwrap import dedent
 
-sourcepath = Path(__file__).resolve()
-
 from shiboken6 import Shiboken
 from shibokensupport.signature.lib.enum_sig import HintingEnumerator
 from shibokensupport.signature.lib.tool import build_brace_pattern
 
+sourcepath = Path(__file__).resolve()
+
 # Can we use forward references?
 USE_PEP563 = sys.version_info[:2] >= (3, 7)
 
@@ -74,7 +75,7 @@ logger = logging.getLogger("generate_pyi")
 
 
 class Writer(object):
-    def __init__(self, outfile):
+    def __init__(self, outfile, *args):
         self.outfile = outfile
         self.history = [True, True]
 
@@ -101,8 +102,9 @@ class Formatter(Writer):
     The separation in formatter and enumerator is done to keep the
     unrelated tasks of enumeration and formatting apart.
     """
-    def __init__(self, *args):
-        Writer.__init__(self, *args)
+    def __init__(self, outfile, options, *args):
+        self.options = options
+        Writer.__init__(self, outfile, *args)
         # patching __repr__ to disable the __repr__ of typing.TypeVar:
         """
             def __repr__(self):
@@ -115,7 +117,7 @@ class Formatter(Writer):
                 return prefix + self.__name__
         """
         def _typevar__repr__(self):
-            return "typing." + self.__name__
+            return f"typing.{self.__name__}"
         typing.TypeVar.__repr__ = _typevar__repr__
 
         # Adding a pattern to substitute "Union[T, NoneType]" by "Optional[T]"
@@ -130,24 +132,31 @@ class Formatter(Writer):
             return optional_searcher.sub(replace, str(source))
         self.optional_replacer = optional_replacer
         # self.level is maintained by enum_sig.py
-        # self.after_enum() is a one-shot set by enum_sig.py .
         # self.is_method() is true for non-plain functions.
 
+    def section(self):
+        if self.level == 0:
+            self.print()
+        self.print()
+
     @contextmanager
     def module(self, mod_name):
         self.mod_name = mod_name
-        self.print("# Module", mod_name)
-        self.print("import PySide6")
-        self.print("import typing")
-        self.print("from typing import Any, Callable, Dict, List, Optional, Tuple, Union")
-        self.print("from PySide6.support.signature.mapping import (")
-        self.print("    Virtual, Missing, Invalid, Default, Instance)")
-        self.print()
-        self.print("class Object(object): pass")
-        self.print()
-        self.print("from shiboken6 import Shiboken")
-        self.print("Shiboken.Object = Object")
-        self.print()
+        support = "PySide6.support" if self.options._pyside_call else "shibokensupport"
+        txt = f"""\
+            # Module `{mod_name}`
+
+            import typing
+            from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+            from shiboken6 import Shiboken
+            from {support}.signature.mapping import (
+                Virtual, Missing, Invalid, Default, Instance)
+
+            class Object(object): pass
+
+            Shiboken.Object = Object
+            """
+        self.print(dedent(txt))
         # This line will be replaced by the missing imports postprocess.
         self.print("IMPORTS")
         yield
@@ -158,9 +167,6 @@ class Formatter(Writer):
         while "." in class_name:
             class_name = class_name.split(".", 1)[-1]
             class_str = class_str.split(".", 1)[-1]
-        self.print()
-        if self.level == 0:
-            self.print()
         here = self.outfile.tell()
         if self.have_body:
             self.print(f"{spaces}class {class_str}:")
@@ -170,11 +176,11 @@ class Formatter(Writer):
 
     @contextmanager
     def function(self, func_name, signature):
-        if self.after_enum() or func_name == "__init__":
+        if func_name == "__init__":
             self.print()
         key = func_name
         spaces = indent * self.level
-        if type(signature) == type([]):
+        if isinstance(signature, list):
             for sig in signature:
                 self.print(f'{spaces}@typing.overload')
                 self._function(func_name, sig, spaces)
@@ -185,7 +191,7 @@ class Formatter(Writer):
         yield key
 
     def _function(self, func_name, signature, spaces):
-        if self.is_method() and "self" not in tuple(signature.parameters.keys()):
+        if self.is_method() and "self" not in signature.parameters:
             self.print(f'{spaces}@staticmethod')
         signature = self.optional_replacer(signature)
         self.print(f'{spaces}def {func_name}{signature}: ...')
@@ -201,8 +207,8 @@ class Formatter(Writer):
 def get_license_text():
     with sourcepath.open() as f:
         lines = f.readlines()
-        license_line = next((lno for lno, line in enumerate(lines)
-                             if "$QT_END_LICENSE$" in line))
+    license_line = next((lno for lno, line in enumerate(lines)
+                         if "$QT_END_LICENSE$" in line))
     return "".join(lines[:license_line + 3])
 
 
@@ -210,15 +216,15 @@ def find_imports(text):
     return [imp for imp in PySide6.__all__ if imp + "." in text]
 
 
-def find_module(import_name, outpath):
+def find_module(import_name, outpath, from_pyside):
     """
     Find a module either directly by import, or use the full path,
     add the path to sys.path and import then.
     """
-    if import_name.startswith("PySide6."):
+    if from_pyside:
         # internal mode for generate_pyi.py
         plainname = import_name.split(".")[-1]
-        outfilepath = Path(outpath) / (plainname + ".pyi")
+        outfilepath = Path(outpath) / f"{plainname}.pyi"
         return import_name, plainname, outfilepath
     # we are alone in external module mode
     p = Path(import_name).resolve()
@@ -229,22 +235,24 @@ def find_module(import_name, outpath):
     # temporarily add the path and do the import
     sys.path.insert(0, os.fspath(p.parent))
     plainname = p.name.split(".")[0]
-    return plainname, plainname, outpath / (plainname + ".pyi")
+    __import__(plainname)
+    sys.path.pop(0)
+    return plainname, plainname, Path(outpath) / (plainname + ".pyi")
 
 
 def generate_pyi(import_name, outpath, options):
     """
     Generates a .pyi file.
     """
-    import_name, plainname, outfilepath = find_module(import_name, outpath)
+    import_name, plainname, outfilepath = find_module(import_name, outpath, options._pyside_call)
     top = __import__(import_name)
     obj = getattr(top, plainname) if import_name != plainname else top
     if not getattr(obj, "__file__", None) or Path(obj.__file__).is_dir():
-        raise ModuleNotFoundError(f"We do not accept a namespace as module {plainname}")
+        raise ModuleNotFoundError(f"We do not accept a namespace as module `{plainname}`")
     module = sys.modules[import_name]
 
     outfile = io.StringIO()
-    fmt = Formatter(outfile)
+    fmt = Formatter(outfile, options)
     fmt.print(get_license_text())  # which has encoding, already
     need_imports = options._pyside_call and not USE_PEP563
     if USE_PEP563:
@@ -257,13 +265,12 @@ def generate_pyi(import_name, outpath, options):
         """
         '''))
     HintingEnumerator(fmt).module(import_name)
-    fmt.print()
     fmt.print("# eof")
     # Postprocess: resolve the imports
     if options._pyside_call:
         global PySide6
         import PySide6
-    with open(outfilepath, "w") as realfile:
+    with outfilepath.open("w") as realfile:
         wr = Writer(realfile)
         outfile.seek(0)
         while True:
@@ -289,17 +296,25 @@ def generate_pyi(import_name, outpath, options):
         if USE_PEP563:
             subprocess.check_output([sys.executable, outfilepath])
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        description="This script generates the .pyi file for an arbitrary module.")
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=dedent("""\
+            pyi_generator.py
+            ----------------
+
+            This script generates the .pyi file for an arbitrary module.
+            You pass in the full path of a compiled, importable module.
+            pyi_generator will try to generate an interface "<module>.pyi".
+            """))
     parser.add_argument("module",
-        help="The name of an importable module, or the full path to the binary")
+        help="The full path name of an importable module binary (.pyd, .so)")
     parser.add_argument("--check", action="store_true", help="Test the output")
     parser.add_argument("--outpath",
         help="the output directory (default = binary location)")
     options = parser.parse_args()
     module = options.module
-    # XXX find a path that ends in a module and use that
     outpath = options.outpath
     if outpath and not Path(outpath).exists():
         os.makedirs(outpath)
diff --git a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/mapping.py b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
index 21a127087..9ba40679e 100644
--- a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
+++ b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
@@ -1,6 +1,6 @@
 #############################################################################
 ##
-## Copyright (C) 2019 The Qt Company Ltd.
+## Copyright (C) 2021 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of Qt for Python.