Signature: Implement Nested Classes and Functions for Shiboken

This patch again contains a complete overhaul of the signature
module. The code was re-implemented to properly support nested
classes. Also, the code was reduced by AutoDecRef and by
adopting a concise C++ style.

Note.. We will add a shiboken signature test and complete
mapping.py after the split into three projects is done. The split
changes a lot and is needed right now!

Signatures were quite complete for PySide, but the support for Shiboken
was under-developed.

Since we are planning to generally enhance error messages by using
the Signature module, we should be able to rely on them to always
produce a signature. Therefore, a general overhaul was needed
to resolve all cornes cases for Python 2 and 3.

Nested classes are supported, as well as plain module functions.

The usage of the typing module might improve over time, but the
Signature implementation is now considered complete.

The loader will respect now the path settings which might not be
the package dir but the build dir. This is more consistens with COIN
testing.

Task-number: PYSIDE-795
Change-Id: I246449d4df895dadf2bcb4d997eaa13d78463d9b
Reviewed-by: Simo Fält <simo.falt@qt.io>

6 files changed, 118 insertions, 16 deletions

diff --git a/sources/pyside2/PySide2/__init__.py.in b/sources/pyside2/PySide2/__init__.py.in
index 631f5f13a..ac75f52b6 100644
--- a/sources/pyside2/PySide2/__init__.py.in
+++ b/sources/pyside2/PySide2/__init__.py.in
@@ -21,8 +21,6 @@ def _setupQtDirectories():
     import shiboken2
 
     pyside_package_dir =  os.path.abspath(os.path.dirname(__file__))
-    # Used by signature module.
-    os.environ["PYSIDE_PACKAGE_DIR"] = pyside_package_dir
 
     if sys.platform == 'win32':
         # PATH has to contain the package directory, otherwise plugins
diff --git a/sources/pyside2/PySide2/support/signature/__init__.py b/sources/pyside2/PySide2/support/signature/__init__.py
index 14e63a5fb..49224bf92 100644
--- a/sources/pyside2/PySide2/support/signature/__init__.py
+++ b/sources/pyside2/PySide2/support/signature/__init__.py
@@ -42,5 +42,5 @@ from __future__ import print_function, absolute_import
 from .loader import inspect
 from PySide2 import QtCore
 if QtCore.QProcess.__signature__:
-    pass   # trigger initialization
+    pass   # trigger initialization phase 2, so we can import:
 from signature_loader import get_signature
diff --git a/sources/pyside2/PySide2/support/signature/layout.py b/sources/pyside2/PySide2/support/signature/layout.py
index ac7833f03..e18cb2172 100644
--- a/sources/pyside2/PySide2/support/signature/layout.py
+++ b/sources/pyside2/PySide2/support/signature/layout.py
@@ -58,6 +58,7 @@ used literally as strings like "signature", "existence", etc.
 from textwrap import dedent
 from .loader import inspect
 
+
 class SimpleNamespace(object):
     # From types.rst, because the builtin is implemented in Python 3, only.
     def __init__(self, **kwargs):
@@ -71,6 +72,7 @@ class SimpleNamespace(object):
     def __eq__(self, other):
         return self.__dict__ == other.__dict__
 
+
 class SignatureLayout(SimpleNamespace):
     """
     Configure a signature.
@@ -140,6 +142,7 @@ typeerror = SignatureLayout(definition=False,
                             return_annotation=False,
                             parameter_names=False)
 
+
 def define_nameless_parameter():
     """
     Create Nameless Parameters
@@ -168,8 +171,10 @@ def define_nameless_parameter():
     body["__str__"] = __str__
     return type(newname, bases, body)
 
+
 NamelessParameter = define_nameless_parameter()
 
+
 def make_signature_nameless(signature):
     """
     Make a Signature Nameless
@@ -178,7 +183,8 @@ def make_signature_nameless(signature):
     The signature looks different, but is totally intact.
     """
     for key in signature.parameters.keys():
-        Signature.parameters[key].__class__ = NamelessParameter
+        signature.parameters[key].__class__ = NamelessParameter
+
 
 def create_signature(props, key):
     if not props:
@@ -193,7 +199,7 @@ def create_signature(props, key):
     else:
         sig_kind, modifier = key, "signature"
 
-    layout = globals()[modifier]  # lookup of the modifier, here
+    layout = globals()[modifier]  # lookup of the modifier in this module
     if not isinstance(layout, SignatureLayout):
         raise SystemError("Modifiers must be names of a SignatureLayout "
                           "instance")
@@ -201,14 +207,16 @@ def create_signature(props, key):
     # this is the basic layout of a signature
     varnames = props["varnames"]
     if layout.definition:
-        if sig_kind == "method":
+        if sig_kind == "function":
+            pass
+        elif sig_kind == "method":
             varnames = ("self",) + varnames
         elif sig_kind == "staticmethod":
             pass
         elif sig_kind == "classmethod":
             varnames = ("klass",) + varnames
         else:
-            raise SystemError("Methods must be normal, staticmethod or "
+            raise SystemError("Methods must be function, method, staticmethod or "
                               "classmethod")
     # calculate the modifications
     defaults = props["defaults"][:]
diff --git a/sources/pyside2/PySide2/support/signature/lib/enum_sig.py b/sources/pyside2/PySide2/support/signature/lib/enum_sig.py
index 702ee7ebd..c043f04f8 100644
--- a/sources/pyside2/PySide2/support/signature/lib/enum_sig.py
+++ b/sources/pyside2/PySide2/support/signature/lib/enum_sig.py
@@ -37,6 +37,8 @@
 ##
 #############################################################################
 
+from __future__ import print_function, absolute_import
+
 import sys
 from PySide2.support.signature import inspect, get_signature
 
@@ -58,7 +60,11 @@ class ExactEnumerator(object):
         with self.fmt.module(mod_name):
             module = sys.modules[mod_name]
             members = inspect.getmembers(module, inspect.isclass)
+            functions = inspect.getmembers(module, inspect.isroutine)
             ret = self.result_type()
+            self.fmt.class_name = None
+            for func_name, func in functions:
+                ret.update(self.function(func_name, func))
             for class_name, klass in members:
                 ret.update(self.klass(class_name, klass))
             return ret
@@ -79,8 +85,15 @@ class ExactEnumerator(object):
             # class_members = inspect.getmembers(klass)
             # gives us also the inherited things.
             class_members = sorted(list(klass.__dict__.items()))
-            for func_name, func in class_members:
-                ret.update(self.function(func_name, func))
+            subclasses = []
+            for thing_name, thing in class_members:
+                if inspect.isclass(thing):
+                    subclass_name = ".".join((class_name, thing_name))
+                    subclasses.append((subclass_name, thing))
+                else:
+                    ret.update(self.function(thing_name, thing))
+            for subclass_name, subclass in subclasses:
+                ret.update(self.klass(subclass_name, subclass))
             return ret
 
     def function(self, func_name, func):
@@ -92,6 +105,27 @@ class ExactEnumerator(object):
         return ret
 
 
+def simplify(signature):
+    if isinstance(signature, list):
+        # remove duplicates which still sometimes occour:
+        ret = set(simplify(sig) for sig in signature)
+        return sorted(ret) if len(ret) > 1 else list(ret)[0]
+    ret = []
+    for pv in signature.parameters.values():
+        txt = str(pv)
+        if ":" not in txt:   # 'self' or '*args'
+            continue
+        txt = txt[txt.index(":") + 1:]
+        if "=" in txt:
+            txt = txt[:txt.index("=")]
+        quote = txt[0]
+        if quote in ("'", '"') and txt[-1] == quote:
+            txt = txt[1:-1]
+        ret.append(txt.strip())
+    return tuple(ret)
+
+
+### disabled for now:
 class SimplifyingEnumerator(ExactEnumerator):
     """
     SimplifyingEnumerator enumerates all signatures in a module filtered.
@@ -109,5 +143,27 @@ class SimplifyingEnumerator(ExactEnumerator):
         signature = get_signature(func, 'existence')
         if signature is not None and func_name not in ("next", "__next__"):
             with self.fmt.function(func_name, signature) as key:
-                ret[key] = signature
+                ret[key] = str(signature)
+        return ret
+
+
+class SimplifyingEnumerator(ExactEnumerator):
+    """
+    SimplifyingEnumerator enumerates all signatures in a module filtered.
+
+    There are no default values, no variable
+    names and no self parameter. Only types are present after simplification.
+    The functions 'next' resp. '__next__' are removed
+    to make the output identical for Python 2 and 3.
+    An appropriate formatter should be supplied, if printable output
+    is desired.
+    """
+
+    def function(self, func_name, func):
+        ret = self.result_type()
+        signature = getattr(func, '__signature__', None)
+        sig = simplify(signature) if signature is not None else None
+        if sig is not None and func_name not in ("next", "__next__", "__div__"):
+            with self.fmt.function(func_name, sig) as key:
+                ret[key] = sig
         return ret
diff --git a/sources/pyside2/PySide2/support/signature/mapping.py b/sources/pyside2/PySide2/support/signature/mapping.py
index 6b7d1ad01..23ba6a7f1 100644
--- a/sources/pyside2/PySide2/support/signature/mapping.py
+++ b/sources/pyside2/PySide2/support/signature/mapping.py
@@ -56,6 +56,11 @@ import sys
 import struct
 import PySide2
 try:
+    import sample
+except ImportError:
+    pass
+
+try:
     from . import typing
 except ImportError:
     import typing
@@ -64,10 +69,12 @@ ellipsis = "..."
 Char = typing.Union[str, int]     # how do I model the limitation to 1 char?
 StringList = typing.List[str]
 IntList = typing.List[int]
+IntMatrix = typing.List[IntList]
 Variant = typing.Any
 ModelIndexList = typing.List[int]
 QImageCleanupFunction = typing.Callable
-FloatMatrix = typing.List[typing.List[float]]
+FloatList = typing.List[float]
+FloatMatrix = typing.List[FloatList]
 # Pair could be more specific, but we loose the info in the generator.
 Pair = typing.Tuple[typing.Any, typing.Any]
 MultiMap = typing.DefaultDict[str, typing.List[str]]
@@ -132,7 +139,7 @@ class Instance(_NotCalled):
 class Reloader(object):
     def __init__(self):
         self.sys_module_count = 0
-        self.uninitialized = PySide2.__all__[:]
+        self.uninitialized = PySide2.__all__[:] + ["sample"]
 
     def update(self):
         if self.sys_module_count == len(sys.modules):
@@ -140,7 +147,7 @@ class Reloader(object):
         self.sys_module_count = len(sys.modules)
         g = globals()
         for mod_name in self.uninitialized[:]:
-            if "PySide2." + mod_name in sys.modules:
+            if "PySide2." + mod_name in sys.modules or mod_name == "sample":
                 self.uninitialized.remove(mod_name)
                 proc_name = "init_" + mod_name
                 if proc_name in g:
@@ -289,6 +296,7 @@ def init_QtCore():
             "PySide2.QtCore.QAbstractItemModel.CheckIndexOptions.NoOption"), # 5.11
         "QVariantMap": dict,
         "PySide2.QtCore.QCborStreamReader.StringResult": typing.AnyStr,
+        "PySide2.QtCore.double": float,
     })
     try:
         type_map.update({
@@ -299,6 +307,7 @@ def init_QtCore():
         pass
     return locals()
 
+
 def init_QtGui():
     import PySide2.QtGui
     type_map.update({
@@ -328,6 +337,7 @@ def init_QtGui():
     })
     return locals()
 
+
 def init_QtWidgets():
     import PySide2.QtWidgets
     from PySide2.QtWidgets import QWidget, QMessageBox, QStyleOption, QStyleHintReturn, QStyleOptionComplex
@@ -364,6 +374,7 @@ def init_QtWidgets():
     })
     return locals()
 
+
 def init_QtSql():
     import PySide2.QtSql
     from PySide2.QtSql import QSqlDatabase
@@ -373,6 +384,7 @@ def init_QtSql():
     })
     return locals()
 
+
 def init_QtNetwork():
     import PySide2.QtNetwork
     type_map.update({
@@ -383,6 +395,7 @@ def init_QtNetwork():
     })
     return locals()
 
+
 def init_QtXmlPatterns():
     import PySide2.QtXmlPatterns
     from PySide2.QtXmlPatterns import QXmlName
@@ -392,6 +405,7 @@ def init_QtXmlPatterns():
     })
     return locals()
 
+
 def init_QtMultimedia():
     import PySide2.QtMultimedia
     import PySide2.QtMultimediaWidgets
@@ -401,6 +415,7 @@ def init_QtMultimedia():
     })
     return locals()
 
+
 def init_QtOpenGL():
     import PySide2.QtOpenGL
     type_map.update({
@@ -417,6 +432,7 @@ def init_QtOpenGL():
     })
     return locals()
 
+
 def init_QtQml():
     import PySide2.QtQml
     type_map.update({
@@ -429,6 +445,7 @@ def init_QtQml():
     })
     return locals()
 
+
 def init_QtQuick():
     import PySide2.QtQuick
     type_map.update({
@@ -440,6 +457,7 @@ def init_QtQuick():
     })
     return locals()
 
+
 def init_QtScript():
     import PySide2.QtScript
     type_map.update({
@@ -447,6 +465,7 @@ def init_QtScript():
     })
     return locals()
 
+
 def init_QtTest():
     import PySide2.QtTest
     type_map.update({
@@ -471,6 +490,23 @@ def init_QtWinExtras():
     })
     return locals()
 
+def init_sample():
+    type_map.update({
+        "sample.int": int,
+        "Complex": complex,
+        "sample.OddBool": bool,
+        "sample.bool": bool,
+        "sample.PStr": str,
+        "double[]": FloatList,
+        "OddBool": bool,
+        "PStr": str,
+        "sample.char": Char,
+        "double[][]": FloatMatrix,
+        "int[]": IntList,
+        "int[][]": IntMatrix,
+    })
+    return locals()
+
 # Here was testbinding, actually the source of all evil.
 
 # end of file
diff --git a/sources/pyside2/tests/registry/init_platform.py b/sources/pyside2/tests/registry/init_platform.py
index 22875a63e..66ec6f566 100644
--- a/sources/pyside2/tests/registry/init_platform.py
+++ b/sources/pyside2/tests/registry/init_platform.py
@@ -143,7 +143,10 @@ class Formatter(object):
 
     @contextmanager
     def function(self, func_name, signature):
-        key = viskey = "{}.{}".format(self.class_name, func_name)
+        if self.class_name is None:
+            key = viskey = "{}".format(func_name)
+        else:
+            key = viskey = "{}.{}".format(self.class_name, func_name)
         if key.endswith("lY"):
             # Some classes like PySide2.QtGui.QContextMenuEvent have functions
             # globalX and the same with Y. The gerrit robot thinks that this
@@ -176,8 +179,9 @@ def generate_all():
             This file contains the simplified signatures for all functions in PySide
             for module '{}'. There are no default values, no variable
             names and no self parameter. Only types are present after simplification.
-            The functions 'next' resp. '__next__' are removed
-            to make the output identical for Python 2 and 3.
+            The functions 'next' resp. '__next__' are removed to make the output
+            identical for Python 2 and 3. '__div__' is also removed,
+            since it exists in Python 2, only.
             """
             '''.format(module)))
         fmt.print("import sys")