Unify signature registry for py2 / py3 and clean up

It turned out that there are tiny differences between Python2 and Python3
which make the versions of the registry almost, but not totally equal.

There are functions which are slot wrappers in Python2 instead of
method wrappers in Python3, and we currently don't support slot wrappers.
There are other tiny differences when we switch to Qt 5.9, too.

Initially, I thought to split the files for Python2 and Python3, but then
it turned out that the problems vanish when we ignore the 'next' and '__next__'
functions in both python versions.

The filter function is both applied to the generating function and the testing
function. Therefore we can keep the existing data intact.

I further removed an indentation leftover in cppgenerator.cpp,
fixed handling of duplicate entries and improved modularisation of the
signature enumerator and formatter. This part will later be moved into the signature
library.

Task-number: PYSIDE-510
Change-Id: I18f5e8f08fb9b07534003919abe55ab4dafeb2c2
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>

4 files changed, 159 insertions, 86 deletions

diff --git a/sources/pyside2/tests/registry/existence_test.py b/sources/pyside2/tests/registry/existence_test.py
index cdb6f362f..e7cadd54d 100644
--- a/sources/pyside2/tests/registry/existence_test.py
+++ b/sources/pyside2/tests/registry/existence_test.py
@@ -42,10 +42,9 @@ from __future__ import print_function, absolute_import
 import os
 import sys
 import unittest
-import warnings
 from textwrap import dedent
 from init_platform import enum_all, generate_all, is_ci, module, refpath
-from util import isolate_warnings, check_warnings, suppress_warnings
+from util import isolate_warnings, check_warnings, suppress_warnings, warn
 from PySide2 import *
 from PySide2.QtCore import __version__
 
@@ -88,10 +87,13 @@ class TestSignaturesExists(unittest.TestCase):
         found_sigs = enum_all()
         with isolate_warnings():
             for key, value in sig_exists.dict.items():
+                name = key.rsplit(".", 1)[-1]
+                if name in ("next", "__next__"): # ignore problematic cases
+                    continue
                 if key not in found_sigs:
-                    warnings.warn("missing key: '{}'".format(key), RuntimeWarning)
+                    warn("missing key: '{}'".format(key))
                 elif isinstance(value, list) and len(value) != len(found_sigs[key]):
-                    warnings.warn("different sig length: '{}'".format(key), RuntimeWarning)
+                    warn("multi-signature count mismatch: '{}'".format(key))
             if is_ci and check_warnings():
                 raise RuntimeError("There are errors, see above.")
 
@@ -101,10 +103,13 @@ class TestSignaturesExists(unittest.TestCase):
         found_sigs.pop(list(found_sigs.keys())[42])
         with isolate_warnings(), suppress_warnings():
             for key, value in sig_exists.dict.items():
+                name = key.rsplit(".", 1)[-1]
+                if name in ("next", "__next__"): # ignore problematic cases
+                    continue
                 if key not in found_sigs:
-                    warnings.warn("missing key: '{}'".format(key), RuntimeWarning)
+                    warn("missing key: '{}'".format(key))
                 elif isinstance(value, list) and len(value) != len(found_sigs[key]):
-                    warnings.warn("different sig length: '{}'".format(key), RuntimeWarning)
+                    warn("multi-signature count mismatch: '{}'".format(key))
             self.assertTrue(check_warnings())
 
 version = tuple(map(int, __version__.split(".")))
diff --git a/sources/pyside2/tests/registry/init_platform.py b/sources/pyside2/tests/registry/init_platform.py
index 5b6558806..2a0b66e06 100644
--- a/sources/pyside2/tests/registry/init_platform.py
+++ b/sources/pyside2/tests/registry/init_platform.py
@@ -49,6 +49,8 @@ One file is generated with all signatures of a platform and version.
 import sys
 import os
 import PySide2
+from contextlib import contextmanager
+from textwrap import dedent
 
 all_modules = list("PySide2." + x for x in PySide2.__all__)
 
@@ -56,29 +58,105 @@ from PySide2.support.signature import inspect
 from PySide2.QtCore import __version__
 
 version_id = __version__.replace(".", "_")
+is_py3 = sys.version_info[0] == 3
 is_ci = os.environ.get("QTEST_ENVIRONMENT", "") == "ci"
 # Python2 legacy: Correct 'linux2' to 'linux', recommended way.
 platform = 'linux' if sys.platform.startswith('linux') else sys.platform
-module = "exists_{}_{}{}".format(platform, version_id,
-                                     "_ci" if is_ci else "")
+module = "exists_{}_{}{}".format(platform, version_id, "_ci" if is_ci else "")
 refpath = os.path.join(os.path.dirname(__file__), module + ".py")
-outfile = None
-sourcepath = os.path.splitext(__file__)[0] + ".py"   # make sure not to get .pyc
+# Make sure not to get .pyc in Python2.
+sourcepath = os.path.splitext(__file__)[0] + ".py"
+
+
+class Formatter(object):
+    """
+    Formatter is formatting the signature listing of an enumerator.
+
+    It is written as context managers in order to avoid many callbacks.
+    The division in formatter and enumerator is done to keep the
+    unrelated tasks of enumeration and formatting apart.
+    """
+    def __init__(self, outfile):
+        self.outfile = outfile
+
+    def print(self, *args, **kw):
+        print(*args, file=self.outfile, **kw) if self.outfile else None
+
+    @contextmanager
+    def module(self, mod_name):
+        self.mod_name = mod_name
+        self.print("")
+        self.print("# Module", mod_name)
+        self.print('if "{}" in sys.modules:'.format(mod_name))
+        self.print("    dict.update({")
+        yield
+        self.print("    })")
+
+    @contextmanager
+    def klass(self, class_name):
+        self.class_name = class_name
+        self.print()
+        self.print("    # class {}.{}:".format(self.mod_name, class_name))
+        yield
+
+    @contextmanager
+    def function(self, func_name, signature):
+        key = viskey = "{}.{}".format(self.class_name, func_name)
+        if key.endswith("lY"):
+            # Some classes like PySide2.QtGui.QContextMenuEvent have functions
+            # globalX and the same with Y. The gerrit robot thinks that this
+            # is a badly written "globally". Convince it by hiding this word.
+            viskey = viskey[:-1] + '""Y'
+        self.print('        "{}": {},'.format(viskey, signature))
+        yield key
+
+
+class ExactEnumerator(object):
+    """
+    ExactEnumerator enumerates all signatures in a module as they are.
+
+    This class is used for generating complete listings of all signatures.
+    An appropriate formatter should be supplied, if printable output
+    is desired.
+    """
+    def __init__(self, formatter, result_type=dict):
+        self.fmt = formatter
+        self.result_type = result_type
+
+    def module(self, mod_name):
+        __import__(mod_name)
+        with self.fmt.module(mod_name):
+            module = sys.modules[mod_name]
+            members = inspect.getmembers(module, inspect.isclass)
+            ret = self.result_type()
+            for class_name, klass in members:
+                ret.update(self.klass(class_name, klass))
+            return ret
+
+    def klass(self, class_name, klass):
+        with self.fmt.klass(class_name):
+            ret = self.function("__init__", klass)
+            # class_members = inspect.getmembers(klass)
+            # gives us also the inherited things.
+            class_members = sorted(list(klass.__dict__.items()))
+            for func_name, func in class_members:
+                ret.update(self.function(func_name, func))
+            return ret
+
+    def function(self, func_name, func):
+        ret = self.result_type()
+        signature = getattr(func, '__signature__', None)
+        if signature is not None:
+            with self.fmt.function(func_name, signature) as key:
+                ret[key] = signature
+        return ret
 
-def xprint(*args, **kw):
-    if outfile:
-        print(*args, file=outfile, **kw)
 
 def simplify(signature):
     if isinstance(signature, list):
-        ret = list(simplify(sig) for sig in signature)
         # remove duplicates which still sometimes occour:
-        things = set(ret)
-        if len(things) != len(ret):
-            ret = list(things)
-            if len(ret) == 1:
-                ret = ret[0]
-        return sorted(ret)
+        ret = set(simplify(sig) for sig in signature)
+        return sorted(ret) if len(ret) > 1 else list(ret)[0]
     ret = []
     for pv in signature.parameters.values():
         txt = str(pv)
@@ -93,77 +171,60 @@ def simplify(signature):
         ret.append(txt)
     return tuple(ret)
 
-def begin_module(mod_name):
-    xprint("")
-    xprint("# Module", mod_name)
-    xprint('if "{}" in sys.modules:'.format(mod_name))
-    xprint("    dict.update({")
-
-def end_module(mod_name):
-    xprint("    })")
-
-def begin_class(mod_name, class_name):
-    xprint()
-    xprint("    # class {}.{}:".format(mod_name, class_name))
-
-def end_class(mod_name, class_name):
-    pass
-
-def show_signature(key, signature):
-    if key.endswith("lY"):
-        # make the robot shut up:
-        key = key[:-1] + '"+"Y'
-    xprint('        "{}": {},'.format(key, signature))
-
-def enum_module(mod_name):
-    __import__(mod_name)
-    begin_module(mod_name)
-    module = sys.modules[mod_name]
-    members = inspect.getmembers(module, inspect.isclass)
-    ret = {}
-    for class_name, klass in members:
-        begin_class(mod_name, class_name)
-        signature = getattr(klass, '__signature__', None)
-        # class_members = inspect.getmembers(klass)
-        # gives us also the inherited things.
-        if signature is not None:
-            signature = simplify(signature)
-            key = "{}.{}".format(class_name, "__init__")
-            ret[key] = signature
-            show_signature(key, signature)
-        class_members = sorted(list(klass.__dict__.items()))
-        for func_name, func in class_members:
-            signature = getattr(func, '__signature__', None)
-            if signature is not None:
-                signature = simplify(signature)
-                key = "{}.{}".format(class_name, func_name)
-                ret[key] = signature
-                show_signature(key, signature)
-        end_class(mod_name, class_name)
-    end_module(mod_name)
+
+class SimplifyingEnumerator(ExactEnumerator):
+    """
+    SimplifyingEnumerator enumerates all signatures in a module filtered.
+
+    There are no default values, no variable
+    names and no self parameter. Only types are present after simplification.
+    The functions 'next' resp. '__next__' are removed
+    to make the output identical for Python 2 and 3.
+    An appropriate formatter should be supplied, if printable output
+    is desired.
+    """
+
+    def function(self, func_name, func):
+        ret = self.result_type()
+        signature = getattr(func, '__signature__', None)
+        sig = simplify(signature) if signature is not None else None
+        if sig is not None and func_name not in ("next", "__next__"):
+            with self.fmt.function(func_name, sig) as key:
+                ret[key] = sig
+        return ret
+
+
+def enum_all():
+    fmt = Formatter(None)
+    enu = SimplifyingEnumerator(fmt)
+    ret = enu.result_type()
+    for mod_name in all_modules:
+        ret.update(enu.module(mod_name))
     return ret
 
 def generate_all():
-    global outfile
     with open(refpath, "w") as outfile, open(sourcepath) as f:
+        fmt = Formatter(outfile)
+        enu = SimplifyingEnumerator(fmt)
         lines = f.readlines()
         license_line = next((lno for lno, line in enumerate(lines)
                              if "$QT_END_LICENSE$" in line))
-        xprint("".join(lines[:license_line + 3]))
-        xprint("import sys")
-        xprint("")
-        xprint("dict = {}")
+        fmt.print("".join(lines[:license_line + 3]))
+        fmt.print(dedent('''\
+            """
+            This file contains the simplified signatures for all functions in PySide
+            for module '{}'. There are no default values, no variable
+            names and no self parameter. Only types are present after simplification.
+            The functions 'next' resp. '__next__' are removed
+            to make the output identical for Python 2 and 3.
+            """
+            '''.format(module)))
+        fmt.print("import sys")
+        fmt.print("")
+        fmt.print("dict = {}")
         for mod_name in all_modules:
-            enum_module(mod_name)
-        xprint("# eof")
-
-def enum_all():
-    global outfile
-    outfile = None
-    ret = {}
-    for mod_name in all_modules:
-        ret.update(enum_module(mod_name))
-    return ret
+            enu.module(mod_name)
+        fmt.print("# eof")
 
 def __main__():
     print("+++ generating {}. You should probably check this file in."
diff --git a/sources/pyside2/tests/registry/signature_test.py b/sources/pyside2/tests/registry/signature_test.py
index 949244dc3..105f46e15 100644
--- a/sources/pyside2/tests/registry/signature_test.py
+++ b/sources/pyside2/tests/registry/signature_test.py
@@ -85,9 +85,9 @@ def enum_module(mod_name):
         else:
             dprint("    def __init__" + str(signature))
         count += 1
-        class_members = list(klass.__dict__.items())
         have_sig = signature is not None
         have_members = 0
+        class_members = sorted(list(klass.__dict__.items()))
         for func_name, func in class_members:
             signature = getattr(func, '__signature__', None)
             if signature is not None:
@@ -110,8 +110,9 @@ def enum_all():
     for mod_name in all_modules:
         result[mod_name] = enum_module(mod_name)
         total += result[mod_name]
-    pprint(result if sys.version_info >= (3,) else list(result.items()))
-    print("Total", total)
+    pprint(result if sys.version_info >= (3,) else list(result.items()),
+           stream=sys.stderr)
+    print("Total", total, file=sys.stderr)
     return result
 
 
diff --git a/sources/pyside2/tests/registry/util.py b/sources/pyside2/tests/registry/util.py
index d873a7d66..7e048cb24 100644
--- a/sources/pyside2/tests/registry/util.py
+++ b/sources/pyside2/tests/registry/util.py
@@ -91,4 +91,10 @@ def check_warnings():
                     return True
     return False
 
+def warn(message, category=None, stacklevel=1):
+    """Issue a warning with the default 'RuntimeWarning'"""
+    if category is None:
+        category = UserWarning
+    warnings.warn(message, category, stacklevel)
+
 # eof