signature: Support typing.Optional[T] and refine a bit

The signature was missing "typing.Optional[T]" which has to be wrapped
around any argument with a default value of "None".

This is the only case where the repr of a type looks different than
it was written, because it renders as "typing.Union[T, NoneType]".
Solving that by redefining a few typing structures was way too
hard and too error prone. It was finally solved by a regex replacemet
that is run as a post process in generate_pyi.py .

The enumerations are now even more complete, since toplevel enums
are also included. This had the effect that enums with Python
keywords were revealed, and so the function "createEnumItem" had
to be modified.

The order of creation was also changed to avoid name clashes.

The overall structure was improved, and instead of parsing the
generated signatures to find out if something is a class method,
this is now very cleanly implemented as an inquiry to get_signature().

I tried to make sense of the flags structure that comes with many
enums. PyQt5 has a standard set of "__...__" methods without useful
signature information. I could mimick that as well, but that would
create a whole lot of pointless extra information. We should decide
later if it makes sense to include that. Right now the flags
structures show the class name, only.

This patch will be merged with the 5.14 branch. The additions of this
patch could fortunately be placed into areas which do almost not
overlap with the 5.14 signature additions.

Change-Id: Ie513e15917b04d746ab597fb7a9eb1fd766f7c73
Fixes: PYSIDE-1079
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>

8 files changed, 161 insertions, 78 deletions

diff --git a/sources/pyside2/PySide2/support/generate_pyi.py b/sources/pyside2/PySide2/support/generate_pyi.py
index 8aa69c983..d5bbe5d7c 100644
--- a/sources/pyside2/PySide2/support/generate_pyi.py
+++ b/sources/pyside2/PySide2/support/generate_pyi.py
@@ -100,6 +100,38 @@ class Formatter(Writer):
     The separation in formatter and enumerator is done to keep the
     unrelated tasks of enumeration and formatting apart.
     """
+    def __init__(self, *args):
+        Writer.__init__(self, *args)
+        # patching __repr__ to disable the __repr__ of typing.TypeVar:
+        """
+            def __repr__(self):
+                if self.__covariant__:
+                    prefix = '+'
+                elif self.__contravariant__:
+                    prefix = '-'
+                else:
+                    prefix = '~'
+                return prefix + self.__name__
+        """
+        def _typevar__repr__(self):
+            return "typing." + self.__name__
+        typing.TypeVar.__repr__ = _typevar__repr__
+
+        # Adding a pattern to substitute "Union[T, NoneType]" by "Optional[T]"
+        # I tried hard to replace typing.Optional by a simple override, but
+        # this became _way_ too much.
+        # See also the comment in layout.py .
+        brace_pat = build_brace_pattern(3)
+        pattern = (r"\b Union \s* \[ \s* {brace_pat} \s*, \s* NoneType \s* \]"
+                   .format(**locals()))
+        replace = r"Optional[\1]"
+        optional_searcher = re.compile(pattern, flags=re.VERBOSE)
+        def optional_replacer(source):
+            return optional_searcher.sub(replace, str(source))
+        self.optional_replacer = optional_replacer
+        # self.level is maintained by enum_sig.py
+        # self.after_enum() is a one-shot set by enum_sig.py .
+
     @contextmanager
     def module(self, mod_name):
         self.mod_name = mod_name
@@ -121,26 +153,21 @@ class Formatter(Writer):
 
     @contextmanager
     def klass(self, class_name, class_str):
-        self.class_name = class_name
-        spaces = ""
+        spaces = indent * self.level
         while "." in class_name:
-            spaces += indent
             class_name = class_name.split(".", 1)[-1]
             class_str = class_str.split(".", 1)[-1]
         self.print()
-        if not spaces:
+        if self.level == 0:
             self.print()
         here = self.outfile.tell()
         self.print("{spaces}class {class_str}:".format(**locals()))
-        self.print()
         pos = self.outfile.tell()
-        self.spaces = spaces
         yield
         if pos == self.outfile.tell():
             # we have not written any function
             self.outfile.seek(here)
             self.outfile.truncate()
-            # Note: we cannot use class_str when we have no body.
             self.print("{spaces}class {class_str}: ...".format(**locals()))
         if "<" in class_name:
             # This is happening in QtQuick for some reason:
@@ -150,29 +177,32 @@ class Formatter(Writer):
             self.outfile.truncate()
 
     @contextmanager
-    def function(self, func_name, signature):
+    def function(self, func_name, signature, modifier=None):
+        if self.after_enum() or func_name == "__init__":
+            self.print()
         key = func_name
-        spaces = indent + self.spaces if self.class_name else ""
+        spaces = indent * self.level
         if type(signature) == type([]):
             for sig in signature:
                 self.print('{spaces}@typing.overload'.format(**locals()))
-                self._function(func_name, sig, spaces)
+                self._function(func_name, sig, modifier, spaces)
         else:
-            self._function(func_name, signature, spaces)
+            self._function(func_name, signature, modifier, spaces)
+        if func_name == "__init__":
+            self.print()
         yield key
 
-    def _function(self, func_name, signature, spaces):
-        # this would be nicer to get somehow together with the signature
-        is_meth = re.match(r"\((\w*)", str(signature)).group(1) == "self"
-        if self.class_name and not is_meth:
-            self.print('{spaces}@staticmethod'.format(**locals()))
+    def _function(self, func_name, signature, modifier, spaces):
+        if modifier:
+            self.print('{spaces}@{modifier}'.format(**locals()))
+        signature = self.optional_replacer(signature)
         self.print('{spaces}def {func_name}{signature}: ...'.format(**locals()))
 
     @contextmanager
     def enum(self, class_name, enum_name, value):
-        spaces = self.spaces
+        spaces = indent * self.level
         hexval = hex(value)
-        self.print("{spaces}{enum_name:20}: {class_name} = ... # {hexval}".format(**locals()))
+        self.print("{spaces}{enum_name:25}: {class_name} = ... # {hexval}".format(**locals()))
         yield
 
 
@@ -254,10 +284,11 @@ def generate_all_pyi(outpath, options):
         os.environ["PYTHONPATH"] = pypath
 
     # now we can import
-    global PySide2, inspect, HintingEnumerator, EnumType
+    global PySide2, inspect, typing, HintingEnumerator, build_brace_pattern
     import PySide2
-    from PySide2.support.signature import inspect
-    from PySide2.support.signature.lib.enum_sig import HintingEnumerator, EnumType
+    from PySide2.support.signature import inspect, typing
+    from PySide2.support.signature.lib.enum_sig import HintingEnumerator
+    from PySide2.support.signature.lib.tool import build_brace_pattern
 
     # propagate USE_PEP563 to the mapping module.
     # Perhaps this can be automated?
diff --git a/sources/shiboken2/libshiboken/sbkenum.cpp b/sources/shiboken2/libshiboken/sbkenum.cpp
index 71fcf5f64..2dc785884 100644
--- a/sources/shiboken2/libshiboken/sbkenum.cpp
+++ b/sources/shiboken2/libshiboken/sbkenum.cpp
@@ -412,6 +412,13 @@ PyTypeObject *createScopedEnum(SbkObjectType *scope, const char *name, const cha
 
 static PyObject *createEnumItem(PyTypeObject *enumType, const char *itemName, long itemValue)
 {
+    char mangled[20];
+    if (strcmp(itemName, "None") == 0
+            || strcmp(itemName, "False") == 0 || strcmp(itemName, "True") == 0) {
+        strcpy(mangled, itemName);
+        strcat(mangled, "_");
+        itemName = mangled;
+    }
     PyObject *enumItem = newItem(enumType, itemValue, itemName);
     if (PyDict_SetItemString(enumType->tp_dict, itemName, enumItem) < 0)
         return nullptr;
diff --git a/sources/shiboken2/libshiboken/signature.cpp b/sources/shiboken2/libshiboken/signature.cpp
index 2cade65e0..0afdbcdc9 100644
--- a/sources/shiboken2/libshiboken/signature.cpp
+++ b/sources/shiboken2/libshiboken/signature.cpp
@@ -99,7 +99,7 @@ CreateSignature(PyObject *props, PyObject *key)
 {
     /*
      * Here is the new function to create all signatures. It simply calls
-     * into Python and creates a signature object for a dummy-function.
+     * into Python and creates a signature object directly.
      * This is so much simpler than using all the attributes explicitly
      * to support '_signature_is_functionlike()'.
      */
@@ -313,7 +313,7 @@ pyside_tp_get___signature__(PyObject *obtype_mod, const char *modifier)
 
 // forward
 static PyObject *
-GetSignature_Cached(PyObject *props, const char *sig_kind, const char *modifier);
+GetSignature_Cached(PyObject *props, const char *func_kind, const char *modifier);
 
 static PyObject *
 GetTypeKey(PyObject *ob)
@@ -381,16 +381,16 @@ GetSignature_Function(PyObject *obfunc, const char *modifier)
         Py_RETURN_NONE;
 
     int flags = PyCFunction_GET_FLAGS(obfunc);
-    const char *sig_kind;
+    const char *func_kind;
     if (PyModule_Check(obtype_mod))
-        sig_kind = "function";
+        func_kind = "function";
     else if (flags & METH_CLASS)
-        sig_kind = "classmethod";
+        func_kind = "classmethod";
     else if (flags & METH_STATIC)
-        sig_kind = "staticmethod";
+        func_kind = "staticmethod";
     else
-        sig_kind = "method";
-    return GetSignature_Cached(props, sig_kind, modifier);
+        func_kind = "method";
+    return GetSignature_Cached(props, func_kind, modifier);
 }
 
 static PyObject *
@@ -427,11 +427,15 @@ GetSignature_TypeMod(PyObject *ob, const char *modifier)
 }
 
 static PyObject *
-GetSignature_Cached(PyObject *props, const char *sig_kind, const char *modifier)
+GetSignature_Cached(PyObject *props, const char *func_kind, const char *modifier)
 {
+    // Special case: We want to know the func_kind.
+    if (modifier && strcmp(modifier, "__func_kind__") == 0)
+        return Py_BuildValue("s", func_kind);
+
     Shiboken::AutoDecRef key(modifier == nullptr
-                             ? Py_BuildValue("s", sig_kind)
-                             : Py_BuildValue("(ss)", sig_kind, modifier));
+                             ? Py_BuildValue("s", func_kind)
+                             : Py_BuildValue("(ss)", func_kind, modifier));
     PyObject *value = PyDict_GetItem(props, key);
     if (value == nullptr) {
         // we need to compute a signature object
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/layout.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/layout.py
index af8ada065..384273d92 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/layout.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/layout.py
@@ -56,7 +56,7 @@ used literally as strings like "signature", "existence", etc.
 """
 
 from textwrap import dedent
-from shibokensupport.signature import inspect
+from shibokensupport.signature import inspect, typing
 from shibokensupport.signature.mapping import ellipsis
 from shibokensupport.signature.lib.tool import SimpleNamespace
 
@@ -162,6 +162,35 @@ def define_nameless_parameter():
 
 NamelessParameter = define_nameless_parameter()
 
+"""
+Note on the "Optional" feature:
+
+When an annotation has a default value that is None, then the
+type has to be wrapped into "typing.Optional".
+
+Note that only the None value creates an Optional expression,
+because the None leaves the domain of the variable.
+Defaults like integer values are ignored: They stay in the domain.
+
+That information would be lost when we use the "..." convention.
+
+Note that the typing module has the remarkable expansion
+
+    Optional[T]    is    Variant[T, NoneType]
+
+We want to avoid that when generating the .pyi file.
+This is done by a regex in generate_pyi.py .
+The following would work in Python 3, but this is a version-dependent
+hack that also won't work in Python 2 and would be _very_ complex.
+"""
+# import sys
+# if sys.version_info[0] == 3:
+#     class hugo(list):pass
+#     typing._normalize_alias["hugo"] = "Optional"
+#     Optional = typing._alias(hugo, typing.T, inst=False)
+# else:
+#     Optional = typing.Optional
+
 
 def make_signature_nameless(signature):
     """
@@ -217,8 +246,6 @@ def create_signature(props, key):
     defaults = props["defaults"][:]
     if not layout.defaults:
         defaults = ()
-    if layout.ellipsis:
-        defaults = (ellipsis,) * len(defaults)
     annotations = props["annotations"].copy()
     if not layout.return_annotation and "return" in annotations:
         del annotations["return"]
@@ -235,6 +262,10 @@ def create_signature(props, key):
         name = name.lstrip("*")
         defpos = idx - len(varnames) + len(defaults)
         default = defaults[defpos] if defpos >= 0 else _empty
+        if default is None:
+            ann = typing.Optional[ann]
+        if default is not _empty and layout.ellipsis:
+            default = ellipsis
         param = inspect.Parameter(name, kind, annotation=ann, default=default)
         params.append(param)
         if kind == _VAR_POSITIONAL:
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
index 42b123046..b026a5d20 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
@@ -71,6 +71,13 @@ class ExactEnumerator(object):
     def __init__(self, formatter, result_type=dict):
         self.fmt = formatter
         self.result_type = result_type
+        self.fmt.level = 0
+        self.fmt.after_enum = self.after_enum
+        self._after_enum = False
+
+    def after_enum(self):
+        ret = self._after_enum
+        self._after_enum = False
 
     def module(self, mod_name):
         __import__(mod_name)
@@ -80,10 +87,12 @@ class ExactEnumerator(object):
             functions = inspect.getmembers(module, inspect.isroutine)
             ret = self.result_type()
             self.fmt.class_name = None
-            for func_name, func in functions:
-                ret.update(self.function(func_name, func))
             for class_name, klass in members:
                 ret.update(self.klass(class_name, klass))
+            if isinstance(klass, EnumType):
+                self.enum(klass)
+            for func_name, func in functions:
+                ret.update(self.function(func_name, func))
             return ret
 
     def klass(self, class_name, klass):
@@ -95,7 +104,7 @@ class ExactEnumerator(object):
         bases_list = []
         for base in klass.__bases__:
             name = base.__name__
-            if name == "object":
+            if name in ("object", "type"):
                 pass
             else:
                 modname = base.__module__
@@ -103,30 +112,41 @@ class ExactEnumerator(object):
             bases_list.append(name)
         class_str = "{}({})".format(class_name, ", ".join(bases_list))
         with self.fmt.klass(class_name, class_str):
-            ret = self.function("__init__", klass)
+            ret = self.result_type()
             # class_members = inspect.getmembers(klass)
             # gives us also the inherited things.
             class_members = sorted(list(klass.__dict__.items()))
             subclasses = []
+            functions = []
             for thing_name, thing in class_members:
                 if inspect.isclass(thing):
                     subclass_name = ".".join((class_name, thing_name))
                     subclasses.append((subclass_name, thing))
-                else:
+                elif inspect.isroutine(thing):
                     func_name = thing_name.split(".")[0]   # remove ".overload"
-                    ret.update(self.function(func_name, thing))
+                    functions.append((func_name, thing))
+            self.fmt.level += 1
             for subclass_name, subclass in subclasses:
                 ret.update(self.klass(subclass_name, subclass))
                 if isinstance(subclass, EnumType):
                     self.enum(subclass)
-            return ret
+            ret = self.function("__init__", klass)
+            for func_name, func in functions:
+                func_kind = get_signature(func, "__func_kind__")
+                modifier = func_kind if func_kind in (
+                    "staticmethod", "classmethod") else None
+                ret.update(self.function(func_name, func, modifier))
+            self.fmt.level -= 1
+        return ret
 
-    def function(self, func_name, func):
+    def function(self, func_name, func, modifier=None):
+        self.fmt.level += 1
         ret = self.result_type()
         signature = getattr(func, '__signature__', None)
         if signature is not None:
-            with self.fmt.function(func_name, signature) as key:
+            with self.fmt.function(func_name, signature, modifier) as key:
                 ret[key] = signature
+        self.fmt.level -= 1
         return ret
 
     def enum(self, subclass):
@@ -138,6 +158,7 @@ class ExactEnumerator(object):
             if type(type(value)) is EnumType:
                 with self.fmt.enum(class_name, enum_name, int(value)):
                     pass
+        self._after_enum = True
 
 
 def stringify(signature):
@@ -160,7 +181,7 @@ class SimplifyingEnumerator(ExactEnumerator):
     is desired.
     """
 
-    def function(self, func_name, func):
+    def function(self, func_name, func, modifier=None):
         ret = self.result_type()
         signature = get_signature(func, 'existence')
         sig = stringify(signature) if signature is not None else None
@@ -177,11 +198,11 @@ class HintingEnumerator(ExactEnumerator):
     hinting stubs. Only default values are replaced by "...".
     """
 
-    def function(self, func_name, func):
+    def function(self, func_name, func, modifier=None):
         ret = self.result_type()
         signature = get_signature(func, 'hintingstub')
         if signature is not None:
-            with self.fmt.function(func_name, signature) as key:
+            with self.fmt.function(func_name, signature, modifier) as key:
                 ret[key] = signature
         return ret
 
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/tool.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/tool.py
index b34bfb404..3b0825049 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/tool.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/tool.py
@@ -43,6 +43,8 @@ from __future__ import print_function, absolute_import
 tool.py
 
 Some useful stuff, see below.
+On the function with_metaclass see the answer from Martijn Pieters on
+https://stackoverflow.com/questions/18513821/python-metaclass-understanding-the-with-metaclass
 """
 
 from textwrap import dedent
@@ -132,4 +134,21 @@ def build_brace_pattern(level, separators=""):
                                  indent = idx * "    ", **locals())
     return pattern.replace("C", "{").replace("D", "}")
 
+
+# Copied from the six module:
+def with_metaclass(meta, *bases):
+    """Create a base class with a metaclass."""
+    # This requires a bit of explanation: the basic idea is to make a dummy
+    # metaclass for one level of class instantiation that replaces itself with
+    # the actual metaclass.
+    class metaclass(type):
+
+        def __new__(cls, name, this_bases, d):
+            return meta(name, bases, d)
+
+        @classmethod
+        def __prepare__(cls, name, this_bases):
+            return meta.__prepare__(name, bases)
+    return type.__new__(metaclass, 'temporary_class', (), {})
+
 # eof
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
index 8192f9bca..8eff19d77 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
@@ -85,20 +85,6 @@ def formatannotation(annotation, base_module=None):
         return annotation.__module__ + '.' + annotation.__qualname__
     return repr(annotation)
 
-# patching __repr__ to disable the __repr__ of typing.TypeVar:
-"""
-    def __repr__(self):
-        if self.__covariant__:
-            prefix = '+'
-        elif self.__contravariant__:
-            prefix = '-'
-        else:
-            prefix = '~'
-        return prefix + self.__name__
-"""
-def _typevar__repr__(self):
-    return "typing." + self.__name__
-
 # Note also that during the tests we have a different encoding that would
 # break the Python license decorated files without an encoding line.
 
@@ -171,7 +157,6 @@ else:
     inspect.__doc__ += _doc
     # force inspect to find all attributes. See "heuristic" in pydoc.py!
     inspect.__all__ = list(x for x in dir(inspect) if not x.startswith("_"))
-typing.TypeVar.__repr__ = _typevar__repr__
 
 # Fix the module names in typing if possible. This is important since
 # the typing names should be I/O compatible, so that typing.Dict
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
index 10d0f16ad..163aac851 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
@@ -55,6 +55,7 @@ import os
 
 from shibokensupport.signature import typing
 from shibokensupport.signature.typing import TypeVar, Generic
+from shibokensupport.signature.lib.tool import with_metaclass
 
 class ellipsis(object):
     def __repr__(self):
@@ -76,22 +77,6 @@ _S = TypeVar("_S")
 # Building our own Char type, which is much nicer than
 # Char = typing.Union[str, int]     # how do I model the limitation to 1 char?
 
-# Copied from the six module:
-def with_metaclass(meta, *bases):
-    """Create a base class with a metaclass."""
-    # This requires a bit of explanation: the basic idea is to make a dummy
-    # metaclass for one level of class instantiation that replaces itself with
-    # the actual metaclass.
-    class metaclass(type):
-
-        def __new__(cls, name, this_bases, d):
-            return meta(name, bases, d)
-
-        @classmethod
-        def __prepare__(cls, name, this_bases):
-            return meta.__prepare__(name, bases)
-    return type.__new__(metaclass, 'temporary_class', (), {})
-
 class _CharMeta(type):
     def __repr__(self):
         return '%s.%s' % (self.__module__, self.__name__)