Prepare the Signature Module For More Applications

This is the preparation for a number of planned applications
and extensions using the signature module.
This general overhaul contains:

- Extraction of signature enumerations into enum_sigs.py,

- a list of current keyword errors in arguments which are unsolved
  in shiboken, but temporarily fixed in parser.py (too many for XML),

- fix spurious duplications in multiple signatures

- corrections for keyword errors in function names which cannot be
  fixed by Python (quite few),

- fixing "..." arguments into "*args",

- supporting the "slot wrapper" type. This is necessary for
  methods like "__add__", "__mul__" etc.

- Create an extra function "get_signature" that has a parameter to
  modify the appearance, i.e. without self, without returntype, etc.

Task-number: PYSIDE-510
Change-Id: If16f7bf02c6e7cbbdc970058bb630ea4db2b854a
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>

9 files changed, 253 insertions, 28 deletions

diff --git a/sources/pyside2/PySide2/CMakeLists.txt b/sources/pyside2/PySide2/CMakeLists.txt
index 0263f7441..0a76b6344 100644
--- a/sources/pyside2/PySide2/CMakeLists.txt
+++ b/sources/pyside2/PySide2/CMakeLists.txt
@@ -46,6 +46,10 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/support/signature/mapping.py"
                "${CMAKE_CURRENT_BINARY_DIR}/support/signature/mapping.py" COPYONLY)
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/support/signature/parser.py"
                "${CMAKE_CURRENT_BINARY_DIR}/support/signature/parser.py" COPYONLY)
+configure_file("${CMAKE_CURRENT_SOURCE_DIR}/support/signature/lib/__init__.py"
+               "${CMAKE_CURRENT_BINARY_DIR}/support/signature/lib/__init__.py" COPYONLY)
+configure_file("${CMAKE_CURRENT_SOURCE_DIR}/support/signature/lib/enum_sig.py"
+               "${CMAKE_CURRENT_BINARY_DIR}/support/signature/lib/enum_sig.py" COPYONLY)
 if (PYTHON_VERSION_MAJOR EQUAL 3)
 else()
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/support/signature/backport_inspect.py"
diff --git a/sources/pyside2/PySide2/QtGui/typesystem_gui_common.xml b/sources/pyside2/PySide2/QtGui/typesystem_gui_common.xml
index f67b5a306..6e80230d7 100644
--- a/sources/pyside2/PySide2/QtGui/typesystem_gui_common.xml
+++ b/sources/pyside2/PySide2/QtGui/typesystem_gui_common.xml
@@ -2694,10 +2694,10 @@
 
         The signature "QList<qreal>" is needed by the __reduce__ methods,
         but created by some other object used elsewhere.
-        After the matrix type was changed, "QList<float>" was nowhere created.
+        After the matrix type was changed, "QList<float>" was created nowhere.
 
         I don't know an explicit way to produce the right conversion function, so what I did
-        was to create a dummy function and immediately dele it again.
+        was to create a dummy function and immediately delete it again.
         This has the desired effect of creating the implicitly needed "QList<float>"
         conversion, although the dummy function goes away.
 
@@ -2986,7 +2986,9 @@
   <object-type name="QWindow">
     <enum-type name="AncestorMode"/>
     <enum-type name="Visibility"/>
+    <modify-function signature="raise()" rename="raise_" />
   </object-type>
+
   <!-- Qt5: not sure if this needs support, skipped for now -->
   <rejection class="QWindow" function-name="nativeEvent"/>"
 
@@ -3013,7 +3015,6 @@
             }
         </inject-code>
     </add-function>
-    <modify-function signature="exec()" rename="exec_" allow-thread="yes"/>
     <inject-code class="native" file="glue/qguiapplication_init.cpp" position="beginning" />
 
   </object-type>
diff --git a/sources/pyside2/PySide2/support/signature/__init__.py b/sources/pyside2/PySide2/support/signature/__init__.py
index 0ff9ec7e9..14e63a5fb 100644
--- a/sources/pyside2/PySide2/support/signature/__init__.py
+++ b/sources/pyside2/PySide2/support/signature/__init__.py
@@ -1,6 +1,6 @@
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of Qt for Python.
@@ -40,3 +40,7 @@
 from __future__ import print_function, absolute_import
 
 from .loader import inspect
+from PySide2 import QtCore
+if QtCore.QProcess.__signature__:
+    pass   # trigger initialization
+from signature_loader import get_signature
diff --git a/sources/pyside2/PySide2/support/signature/fix-complaints.py b/sources/pyside2/PySide2/support/signature/fix-complaints.py
index fa2b44420..e078ef1ab 100644
--- a/sources/pyside2/PySide2/support/signature/fix-complaints.py
+++ b/sources/pyside2/PySide2/support/signature/fix-complaints.py
@@ -1,6 +1,6 @@
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of Qt for Python.
diff --git a/sources/pyside2/PySide2/support/signature/lib/__init__.py b/sources/pyside2/PySide2/support/signature/lib/__init__.py
new file mode 100644
index 000000000..2d640cb89
--- /dev/null
+++ b/sources/pyside2/PySide2/support/signature/lib/__init__.py
@@ -0,0 +1,40 @@
+#############################################################################
+##
+## Copyright (C) 2018 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of Qt for Python.
+##
+## $QT_BEGIN_LICENSE:LGPL$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 3 as published by the Free Software
+## Foundation and appearing in the file LICENSE.LGPL3 included in the
+## packaging of this file. Please review the following information to
+## ensure the GNU Lesser General Public License version 3 requirements
+## will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 2.0 or (at your option) the GNU General
+## Public license version 3 or any later version approved by the KDE Free
+## Qt Foundation. The licenses are as published by the Free Software
+## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-2.0.html and
+## https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+# this file intentionally left blank
diff --git a/sources/pyside2/PySide2/support/signature/lib/enum_sig.py b/sources/pyside2/PySide2/support/signature/lib/enum_sig.py
new file mode 100644
index 000000000..ddc03c097
--- /dev/null
+++ b/sources/pyside2/PySide2/support/signature/lib/enum_sig.py
@@ -0,0 +1,134 @@
+#############################################################################
+##
+## Copyright (C) 2018 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of Qt for Python.
+##
+## $QT_BEGIN_LICENSE:LGPL$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU Lesser General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU Lesser
+## General Public License version 3 as published by the Free Software
+## Foundation and appearing in the file LICENSE.LGPL3 included in the
+## packaging of this file. Please review the following information to
+## ensure the GNU Lesser General Public License version 3 requirements
+## will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 2.0 or (at your option) the GNU General
+## Public license version 3 or any later version approved by the KDE Free
+## Qt Foundation. The licenses are as published by the Free Software
+## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-2.0.html and
+## https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+import sys
+from PySide2.support.signature import inspect
+
+
+class ExactEnumerator(object):
+    """
+    ExactEnumerator enumerates all signatures in a module as they are.
+
+    This class is used for generating complete listings of all signatures.
+    An appropriate formatter should be supplied, if printable output
+    is desired.
+    """
+    def __init__(self, formatter, result_type=dict):
+        self.fmt = formatter
+        self.result_type = result_type
+
+    def module(self, mod_name):
+        __import__(mod_name)
+        with self.fmt.module(mod_name):
+            module = sys.modules[mod_name]
+            members = inspect.getmembers(module, inspect.isclass)
+            ret = self.result_type()
+            for class_name, klass in members:
+                ret.update(self.klass(class_name, klass))
+            return ret
+
+    def klass(self, class_name, klass):
+        bases_list = []
+        for base in klass.__bases__:
+            name = base.__name__
+            if name == "object":
+                pass
+            else:
+                modname = base.__module__
+                name = modname + "." + base.__name__
+            bases_list.append(name)
+        class_str = "{}({})".format(class_name, ", ".join(bases_list))
+        with self.fmt.klass(class_name, class_str):
+            ret = self.function("__init__", klass)
+            # class_members = inspect.getmembers(klass)
+            # gives us also the inherited things.
+            class_members = sorted(list(klass.__dict__.items()))
+            for func_name, func in class_members:
+                ret.update(self.function(func_name, func))
+            return ret
+
+    def function(self, func_name, func):
+        ret = self.result_type()
+        signature = getattr(func, '__signature__', None)
+        if signature is not None:
+            with self.fmt.function(func_name, signature) as key:
+                ret[key] = signature
+        return ret
+
+
+def simplify(signature):
+    if isinstance(signature, list):
+        # remove duplicates which still sometimes occour:
+        ret = set(simplify(sig) for sig in signature)
+        return sorted(ret) if len(ret) > 1 else list(ret)[0]
+    ret = []
+    for pv in signature.parameters.values():
+        txt = str(pv)
+        if ":" not in txt:   # 'self' or '*args'
+            continue
+        txt = txt[txt.index(":") + 1:]
+        if "=" in txt:
+            txt = txt[:txt.index("=")]
+        quote = txt[0]
+        if quote in ("'", '"') and txt[-1] == quote:
+            txt = txt[1:-1]
+        ret.append(txt.strip())
+    return tuple(ret)
+
+
+class SimplifyingEnumerator(ExactEnumerator):
+    """
+    SimplifyingEnumerator enumerates all signatures in a module filtered.
+
+    There are no default values, no variable
+    names and no self parameter. Only types are present after simplification.
+    The functions 'next' resp. '__next__' are removed
+    to make the output identical for Python 2 and 3.
+    An appropriate formatter should be supplied, if printable output
+    is desired.
+    """
+
+    def function(self, func_name, func):
+        ret = self.result_type()
+        signature = getattr(func, '__signature__', None)
+        sig = simplify(signature) if signature is not None else None
+        if sig is not None and func_name not in ("next", "__next__"):
+            with self.fmt.function(func_name, sig) as key:
+                ret[key] = sig
+        return ret
diff --git a/sources/pyside2/PySide2/support/signature/loader.py b/sources/pyside2/PySide2/support/signature/loader.py
index f51bafe79..a055337bf 100644
--- a/sources/pyside2/PySide2/support/signature/loader.py
+++ b/sources/pyside2/PySide2/support/signature/loader.py
@@ -1,6 +1,6 @@
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of Qt for Python.
@@ -82,13 +82,17 @@ sys.path.pop(0)
 # break the Python license decorated files without an encoding line.
 
 # name used in signature.cpp
-def create_signature(props, sig_kind):
+def create_signature(props, key):
     if not props:
         # empty signatures string
         return
     if isinstance(props["multi"], list):
-        return list(create_signature(elem, sig_kind)
+        return list(create_signature(elem, key)
                     for elem in props["multi"])
+    if type(key) is tuple:
+        sig_kind, modifier = key
+    else:
+        sig_kind, modifier = key, None
     varnames = props["varnames"]
     if sig_kind == "method":
         varnames = ("self",) + varnames
diff --git a/sources/pyside2/PySide2/support/signature/mapping.py b/sources/pyside2/PySide2/support/signature/mapping.py
index dd3df0988..6b7d1ad01 100644
--- a/sources/pyside2/PySide2/support/signature/mapping.py
+++ b/sources/pyside2/PySide2/support/signature/mapping.py
@@ -1,6 +1,6 @@
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of Qt for Python.
@@ -46,7 +46,7 @@ This module has the mapping from the pyside C-modules view of signatures
 to the Python representation.
 
 The PySide modules are not loaded in advance, but only after they appear
-in sys.modules. This minimises the loading overhead.
+in sys.modules. This minimizes the loading overhead.
 In principle, we need to re-load the module, when the imports change.
 But it is much easier to do it on demand, when we get an exception.
 See _resolve_value() in singature.py
@@ -71,7 +71,6 @@ FloatMatrix = typing.List[typing.List[float]]
 # Pair could be more specific, but we loose the info in the generator.
 Pair = typing.Tuple[typing.Any, typing.Any]
 MultiMap = typing.DefaultDict[str, typing.List[str]]
-Text = typing.Text
 
 # ulong_max is only 32 bit on windows.
 ulong_max = 2*sys.maxsize+1 if len(struct.pack("L", 1)) != 4 else 0xffffffff
@@ -153,7 +152,7 @@ type_map = {}
 
 def init_QtCore():
     import PySide2.QtCore
-    from PySide2.QtCore import Qt, QUrl, QDir, QGenericArgument
+    from PySide2.QtCore import Qt, QUrl, QDir
     from PySide2.QtCore import QRect, QSize, QPoint, QLocale, QByteArray
     from PySide2.QtCore import QMarginsF # 5.9
     try:
@@ -201,9 +200,8 @@ def init_QtCore():
         "ULONG_MAX": ulong_max,
         "quintptr": int,
         "PyCallable": typing.Callable,
-        "...": ellipsis, # no idea how this should be translated... maybe so?
         "PyTypeObject": type,
-        "PySequence": typing.Sequence,
+        "PySequence": typing.Iterable,  # important for numpy
         "qptrdiff": int,
         "true": True,
         "Qt.HANDLE": int, # be more explicit with some consts?
@@ -242,7 +240,7 @@ def init_QtCore():
             "QDir.SortFlags(QDir.Name | QDir.IgnoreCase)"),
         "PyBytes": bytes,
         "PyByteArray": bytearray,
-        "PyUnicode": Text,
+        "PyUnicode": typing.Text,
         "signed long": int,
         "PySide2.QtCore.int": int,
         "PySide2.QtCore.char": StringList, # A 'char **' is a list of strings.
@@ -259,13 +257,13 @@ def init_QtCore():
         "float[][]": FloatMatrix, # 5.9
         "PySide2.QtCore.unsigned int": int, # 5.9 Ubuntu
         "PySide2.QtCore.long long": int, # 5.9, MSVC 15
-        "QGenericArgument(nullptr)": QGenericArgument(None), # 5.10
+        "QGenericArgument(nullptr)": ellipsis, # 5.10
         "QModelIndex()": Invalid("PySide2.QtCore.QModelIndex"), # repr is btw. very wrong, fix it?!
-        "QGenericArgument((0))": None, # 5.6, RHEL 6.6. Is that ok?
-        "QGenericArgument()": None,
-        "QGenericArgument(0)": None,
-        "QGenericArgument(NULL)": None, # 5.6, MSVC
-        "QGenericArgument(Q_NULLPTR)": None,
+        "QGenericArgument((0))": ellipsis, # 5.6, RHEL 6.6. Is that ok?
+        "QGenericArgument()": ellipsis,
+        "QGenericArgument(0)": ellipsis,
+        "QGenericArgument(NULL)": ellipsis, # 5.6, MSVC
+        "QGenericArgument(Q_NULLPTR)": ellipsis,
         "zero(PySide2.QtCore.QObject)": None,
         "zero(PySide2.QtCore.QThread)": None,
         "zero(quintptr)": 0,
@@ -289,6 +287,8 @@ def init_QtCore():
         "zero(PySide2.QtCore.QEvent.Type)": None,
         "CheckIndexOption.NoOption": Instance(
             "PySide2.QtCore.QAbstractItemModel.CheckIndexOptions.NoOption"), # 5.11
+        "QVariantMap": dict,
+        "PySide2.QtCore.QCborStreamReader.StringResult": typing.AnyStr,
     })
     try:
         type_map.update({
@@ -301,7 +301,6 @@ def init_QtCore():
 
 def init_QtGui():
     import PySide2.QtGui
-    from PySide2.QtGui import QPageLayout, QPageSize # 5.9
     type_map.update({
         "QVector< QTextLayout.FormatRange >()": [], # do we need more structure?
         "USHRT_MAX": ushort_max,
@@ -313,7 +312,7 @@ def init_QtGui():
         "GL_COLOR_BUFFER_BIT": GL_COLOR_BUFFER_BIT,
         "GL_NEAREST": GL_NEAREST,
         "WId": WId,
-        "PySide2.QtGui.QPlatformSurface": Virtual("PySide2.QtGui.QPlatformSurface"), # hmm...
+        "PySide2.QtGui.QPlatformSurface": int, # a handle
         "QList< QTouchEvent.TouchPoint >()": [], # XXX improve?
         "QPixmap()": Default("PySide2.QtGui.QPixmap"), # can't create without qApp
         "PySide2.QtCore.uint8_t": int, # macOS 5.9
@@ -325,6 +324,7 @@ def init_QtGui():
         "zero(PySide2.QtGui.QTextLayout.FormatRange)": None,
         "zero(PySide2.QtGui.QTouchDevice)": None,
         "zero(PySide2.QtGui.QScreen)": None,
+        "PySide2.QtGui.QGenericMatrix": Missing("PySide2.QtGui.QGenericMatrix"),
     })
     return locals()
 
@@ -396,7 +396,6 @@ def init_QtMultimedia():
     import PySide2.QtMultimedia
     import PySide2.QtMultimediaWidgets
     type_map.update({
-        "QVariantMap": dict,
         "QGraphicsVideoItem": PySide2.QtMultimediaWidgets.QGraphicsVideoItem,
         "QVideoWidget": PySide2.QtMultimediaWidgets.QVideoWidget,
     })
diff --git a/sources/pyside2/PySide2/support/signature/parser.py b/sources/pyside2/PySide2/support/signature/parser.py
index 9313fb540..dd6640fde 100644
--- a/sources/pyside2/PySide2/support/signature/parser.py
+++ b/sources/pyside2/PySide2/support/signature/parser.py
@@ -1,6 +1,6 @@
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2018 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of Qt for Python.
@@ -48,6 +48,7 @@ import functools
 from .mapping import type_map, update_mapping, __dict__ as namespace
 
 _DEBUG = False
+LIST_KEYWORDS = False
 
 """
 parser.py
@@ -119,6 +120,8 @@ def _parse_line(line):
     for arg in arglist:
         name, ann = arg.split(":")
         if name in keyword.kwlist:
+            if LIST_KEYWORDS:
+                print("KEYWORD", ret)
             name = name + "_"
         if "=" in ann:
             ann, default = ann.split("=")
@@ -130,6 +133,10 @@ def _parse_line(line):
     multi = ret["multi"]
     if multi is not None:
         ret["multi"] = int(multi)
+    funcname = ret["funcname"]
+    parts = funcname.split(".")
+    if parts[-1] in keyword.kwlist:
+        ret["funcname"] = funcname + "_"
     return ret
 
 def make_good_value(thing, valtype):
@@ -192,8 +199,14 @@ def calculate_props(line):
     arglist = res["arglist"]
     annotations = {}
     _defaults = []
-    for tup in arglist:
+    for idx, tup in enumerate(arglist):
         name, ann = tup[:2]
+        if ann == "...":
+            name = "*args"
+            # copy the fields back :()
+            ann = 'NULL' # maps to None
+            tup = name, ann
+            arglist[idx] = tup
         annotations[name] = _resolve_type(ann, line)
         if len(tup) == 3:
             default = _resolve_value(tup[2], ann, line)
@@ -214,6 +227,31 @@ def calculate_props(line):
     props["multi"] = res["multi"]
     return props
 
+def fixup_multilines(sig_str):
+    lines = list(line.strip() for line in sig_str.strip().splitlines())
+    res = []
+    multi_lines = []
+    for line in lines:
+        multi = re.match(r"([0-9]+):", line)
+        if multi:
+            idx, rest = int(multi.group(1)), line[multi.end():]
+            multi_lines.append(rest)
+            if idx > 0:
+                continue
+            # remove duplicates
+            multi_lines = list(set(multi_lines))
+            # renumber or return a single line
+            nmulti = len(multi_lines)
+            if nmulti > 1:
+                for idx, line in enumerate(multi_lines):
+                    res.append("{}:{}".format(nmulti-idx-1, line))
+            else:
+                res.append(multi_lines[0])
+            multi_lines = []
+        else:
+            res.append(line)
+    return res
+
 def pyside_type_init(typemod, sig_str):
     dprint()
     if type(typemod) is types.ModuleType:
@@ -222,9 +260,10 @@ def pyside_type_init(typemod, sig_str):
         dprint("Initialization of type '{}.{}'".format(typemod.__module__,
                                                        typemod.__name__))
     update_mapping()
+    lines = fixup_multilines(sig_str)
     ret = {}
     multi_props = []
-    for line in sig_str.strip().splitlines():
+    for line in lines:
         props = calculate_props(line)
         shortname = props["name"]
         multi = props["multi"]
@@ -232,10 +271,10 @@ def pyside_type_init(typemod, sig_str):
             ret[shortname] = props
             dprint(props)
         else:
-            fullname = props.pop("fullname")
             multi_props.append(props)
             if multi > 0:
                 continue
+            fullname = props.pop("fullname")
             multi_props = {"multi": multi_props, "fullname": fullname}
             ret[shortname] = multi_props
             dprint(multi_props)