Signature: Simplify parser by not reloading mapping

The mapping had been reloading on demand. This is overkill,
since we only want to initialize the new constants.

This patch replaces reloading by explicit init functions. This
simplifies the parser and even the loader, because sys.path
is no longer relevant and no context manager is needed for a
single sys.path patch.

Task-number: PYSIDE-510
Change-Id: I9e3a45cb10570211183952bd517abb0084b94f47
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

3 files changed, 97 insertions, 154 deletions

diff --git a/sources/pyside2/PySide2/support/signature/loader.py b/sources/pyside2/PySide2/support/signature/loader.py
index f9c8b5c53..373cc8abe 100644
--- a/sources/pyside2/PySide2/support/signature/loader.py
+++ b/sources/pyside2/PySide2/support/signature/loader.py
@@ -51,40 +51,26 @@ This version does not use an embedded .zip file.
 
 import sys
 import os
-import functools
-from contextlib import contextmanager
-from distutils import sysconfig
-
-@contextmanager
-def add_path(path):
-    sys.path.insert(0, path)
-    yield
-    sys.path.pop(0)
 
 # Make sure that we always have the PySide containing package first.
 # This is crucial for the mapping during reload in the tests.
 package_dir = __file__
 for _ in "four":
     package_dir = os.path.dirname(package_dir)
-assured_site_packages = functools.partial(add_path, package_dir)
-
-with assured_site_packages():
-    if sys.version_info >= (3,):
-        from PySide2.support.signature import inspect
-        from PySide2.support.signature import typing
-    else:
-        import inspect
-        namespace = inspect.__dict__
-        from PySide2.support.signature import backport_inspect as inspect
-        inspect.__dict__.update(namespace)
-    from PySide2.support.signature import parser
-# Note also that during the tests we have a different encodind that would
-# break the Python license decorated files without an encoding line.
-
+sys.path.insert(0, package_dir)
+if sys.version_info >= (3,):
+    from PySide2.support.signature import inspect
+    from PySide2.support.signature import typing
+else:
+    import inspect
+    namespace = inspect.__dict__
+    from PySide2.support.signature import backport_inspect as inspect
+    inspect.__dict__.update(namespace)
 # name used in signature.cpp
-def pyside_type_init(*args, **kw):
-    with assured_site_packages():
-        return parser.pyside_type_init(*args, **kw)
+from PySide2.support.signature.parser import pyside_type_init
+sys.path.pop(0)
+# Note also that during the tests we have a different encoding that would
+# break the Python license decorated files without an encoding line.
 
 # name used in signature.cpp
 def create_signature(props, sig_kind):
diff --git a/sources/pyside2/PySide2/support/signature/mapping.py b/sources/pyside2/PySide2/support/signature/mapping.py
index d18ee561b..f6ec9868d 100644
--- a/sources/pyside2/PySide2/support/signature/mapping.py
+++ b/sources/pyside2/PySide2/support/signature/mapping.py
@@ -55,15 +55,16 @@ See _resolve_value() in singature.py
 import sys
 import collections
 import struct
+import PySide2
 
 PY3 = sys.version_info >= (3,)
 if PY3:
     from . import typing
-    exec("ellipsis = ...")
+    ellipsis = eval("...")
     Char = typing.Union[str, int]     # how do I model the limitation to 1 char?
     StringList = typing.List[str]
     Variant = typing.Union[str, int, float, Char, StringList, type(ellipsis)]
-              # much more, do we need that?
+              # Much more, do we need that? Do we better kill it?
     ModelIndexList = typing.List[int]
     QImageCleanupFunction = typing.Callable[[bytes], None]
 else:
@@ -74,8 +75,8 @@ else:
     ModelIndexList = list
     QImageCleanupFunction = object
 Pair = collections.namedtuple('Pair', ['first', 'second'])
-# ulong_max is the long size, which is only 32 bit on windows.
-ulong_max = sys.maxsize if len(struct.pack("L", 1)) != 4 else 0xffffffff
+# ulong_max is only 32 bit on windows.
+ulong_max = 2*sys.maxsize+1 if len(struct.pack("L", 1)) != 4 else 0xffffffff
 ushort_max = 0xffff
 
 GL_COLOR_BUFFER_BIT = 0x00004000
@@ -97,28 +98,27 @@ class Missing(str):
     def __repr__(self):
         return "Missing({})".format(self)
 
-TYPE_MAP_DOC = """
-    The type_map variable is central for the signature module.
-
-    PySide has a new function 'CppGenerator::writeSignatureInfo()'
-    that extracts the gathered information about the function arguments
-    and defaults as good as it can. But what PySide generates is still
-    very C-ish and has many constants that Python doesn't understand.
-
-    The function 'try_to_guess()' below understands a lot of PySide's
-    peculiar way to assume local context. If it is able to do the guess,
-    then the result is inserted into the dict, so the search happens
-    not again. For everything that is not covered by these automatic
-    guesses, we provide an entry in 'type_map' that resolves it.
-
-    In effect, 'type_map' maps text to real Python objects.
-"""
+class Reloader(object):
+    def __init__(self):
+        self.sys_module_count = 0
+        self.uninitialized = PySide2.__all__[:]
+
+    def update(self):
+        if self.sys_module_count == len(sys.modules):
+            return
+        self.sys_module_count = len(sys.modules)
+        for mod_name in self.uninitialized[:]:
+            if "PySide2." + mod_name in sys.modules:
+                self.uninitialized.remove(mod_name)
+                proc_name = "init_" + mod_name
+                if proc_name in globals():
+                    init_proc = globals()[proc_name]
+                    globals().update(init_proc())
+
+update_mapping = Reloader().update
 type_map = {}
 
-loaded_modules = sys.modules
-
-# QtCore
-if "PySide2.QtCore" in loaded_modules:
+def init_QtCore():
     import PySide2.QtCore
     from PySide2.QtCore import Qt, QUrl, QDir, QGenericArgument
     from PySide2.QtCore import QMarginsF # 5.9
@@ -169,7 +169,7 @@ if "PySide2.QtCore" in loaded_modules:
         "PyCallable": callable,
         "...": ellipsis, # no idea how this should be translated... maybe so?
         "PyTypeObject": type,
-        "PySequence": list, # could be more generic
+        "PySequence": list, # needs to be changed, QApplication for instance!
         "qptrdiff": int,
         "true": True,
         "Qt.HANDLE": int, # be more explicit with some consts?
@@ -215,7 +215,6 @@ if "PySide2.QtCore" in loaded_modules:
         "QGenericArgument((0))": None, # 5.6, RHEL 6.6. Is that ok?
         "4294967295UL": 4294967295, # 5.6, RHEL 6.6
     })
-
     try:
         type_map.update({
             "PySide2.QtCore.QMetaObject.Connection": PySide2.QtCore.Connection, # wrong!
@@ -223,9 +222,9 @@ if "PySide2.QtCore" in loaded_modules:
     except AttributeError:
         # this does not exist on 5.9 ATM.
         pass
+    return locals()
 
-# QtGui
-if "PySide2.QtGui" in loaded_modules:
+def init_QtGui():
     import PySide2.QtGui
     from PySide2.QtGui import QPageLayout, QPageSize # 5.9
     type_map.update({
@@ -244,9 +243,9 @@ if "PySide2.QtGui" in loaded_modules:
         "QList< QTouchEvent.TouchPoint >()": list,
         "QPixmap()": lambda:QPixmap(), # we cannot create this without qApp
     })
+    return locals()
 
-# QtWidgets
-if "PySide2.QtWidgets" in loaded_modules:
+def init_QtWidgets():
     import PySide2.QtWidgets
     from PySide2.QtWidgets import QWidget, QMessageBox, QStyleOption, QStyleHintReturn, QStyleOptionComplex
     type_map.update({
@@ -263,34 +262,34 @@ if "PySide2.QtWidgets" in loaded_modules:
         "SH_Default": QStyleHintReturn.SH_Default,
         "SO_Complex": QStyleOptionComplex.SO_Complex,
     })
+    return locals()
 
-# QtSql
-if "PySide2.QtSql" in loaded_modules:
+def init_QtSql():
     import PySide2.QtSql
     from PySide2.QtSql import QSqlDatabase
     type_map.update({
         "QLatin1String(defaultConnection)": QSqlDatabase.defaultConnection,
         "QVariant.Invalid": -1, # not sure what I should create, here...
     })
+    return locals()
 
-# QtNetwork
-if "PySide2.QtNetwork" in loaded_modules:
+def init_QtNetwork():
     import PySide2.QtNetwork
     type_map.update({
         "QMultiMap": typing.DefaultDict(list) if PY3 else {},
     })
+    return locals()
 
-# QtXmlPatterns
-if "PySide2.QtXmlPatterns" in loaded_modules:
+def init_QtXmlPatterns():
     import PySide2.QtXmlPatterns
     from PySide2.QtXmlPatterns import QXmlName
     type_map.update({
         "QXmlName.PrefixCode": Missing("PySide2.QtXmlPatterns.QXmlName.PrefixCode"),
         "QXmlName.NamespaceCode": Missing("PySide2.QtXmlPatterns.QXmlName.NamespaceCode")
     })
+    return locals()
 
-# QtMultimedia
-if "PySide2.QtMultimedia" in loaded_modules:
+def init_QtMultimedia():
     import PySide2.QtMultimedia
     import PySide2.QtMultimediaWidgets
     type_map.update({
@@ -298,9 +297,9 @@ if "PySide2.QtMultimedia" in loaded_modules:
         "QGraphicsVideoItem": PySide2.QtMultimediaWidgets.QGraphicsVideoItem,
         "QVideoWidget": PySide2.QtMultimediaWidgets.QVideoWidget,
     })
+    return locals()
 
-# QtOpenGL
-if "PySide2.QtOpenGL" in loaded_modules:
+def init_QtOpenGL():
     import PySide2.QtOpenGL
     type_map.update({
         "GLuint": int,
@@ -311,9 +310,9 @@ if "PySide2.QtOpenGL" in loaded_modules:
         "PySide2.QtOpenGL.GLuint": int,
         "GLfloat": float, # 5.6, MSVC 15
     })
+    return locals()
 
-# QtQml
-if "PySide2.QtQml" in loaded_modules:
+def init_QtQml():
     import PySide2.QtQml
     type_map.update({
         "QJSValueList()": [],
@@ -321,43 +320,46 @@ if "PySide2.QtQml" in loaded_modules:
         # from 5.9
         "QVariantHash()": {},
     })
+    return locals()
 
-# QtQml
-if "PySide2.QtQuick" in loaded_modules:
+def init_QtQuick():
     import PySide2.QtQuick
     type_map.update({
         "PySide2.QtQuick.QSharedPointer": int,
         "PySide2.QtCore.uint": int,
         "T": int,
     })
+    return locals()
 
-# QtScript
-if "PySide2.QtScript" in loaded_modules:
+def init_QtScript():
     import PySide2.QtScript
     type_map.update({
         "QScriptValueList()": [],
     })
+    return locals()
 
-# QtTest
-if "PySide2.QtTest" in loaded_modules:
+def init_QtTest():
     import PySide2.QtTest
     type_map.update({
         "PySide2.QtTest.QTouchEventSequence": PySide2.QtTest.QTest.QTouchEventSequence,
     })
+    return locals()
 
 # from 5.9
-if "PySide2.QtWebEngineWidgets" in loaded_modules:
+def init_QtWebEngineWidgets():
     import PySide2.QtWebEngineWidgets
     type_map.update({
         "PySide2.QtTest.QTouchEventSequence": PySide2.QtTest.QTest.QTouchEventSequence,
     })
+    return locals()
 
 # from 5.6, MSVC
-if "PySide2.QtWinExtras" in loaded_modules:
+def init_QtWinExtras():
     import PySide2.QtWinExtras
     type_map.update({
         "QList< QWinJumpListItem* >()": [],
     })
+    return locals()
 
 # Here was testbinding, actually the source of all evil.
 
diff --git a/sources/pyside2/PySide2/support/signature/parser.py b/sources/pyside2/PySide2/support/signature/parser.py
index 224ea6be8..c944fe856 100644
--- a/sources/pyside2/PySide2/support/signature/parser.py
+++ b/sources/pyside2/PySide2/support/signature/parser.py
@@ -39,49 +39,33 @@
 
 from __future__ import print_function, absolute_import
 
-"""
-signature.py
-
-This module is the python part of the PySide signature initialization.
-It is not for common use and should be called by shiboken's signature.cpp.
-It is initially written for Python 3, only.
-Meanwhile people say it works with Python 2.7, too. ;-)
-"""
-
 import sys
 import re
 import warnings
 import types
 import keyword
 import functools
-
-PY3 = sys.version_info >= (3,)
-if PY3:
-    try:
-        from importlib import reload
-    except ImportError:
-        from imp import reload
+from .mapping import type_map, update_mapping, __dict__ as namespace
 
 _DEBUG = False
 _BREAK_ON_ERROR = False
 
-class FakeMapping(object):
-    """
-    We do not import the mapping module directly:
+TYPE_MAP_DOC = """
+    The type_map variable is central for the signature package.
 
-    It is not clear from where the mapping is imported. When for instance
-    the mapping is imported by a test from the source directory, reload
-    would now reload from the PySide directory. This is weird and
-    wasteful. We fake the module instead and load it later.
-    """
-    def __init__(self):
-        self.type_map = {}
+    PySide has a new function 'CppGenerator::writeSignatureInfo()'
+    that extracts the gathered information about the function arguments
+    and defaults as good as it can. But what PySide generates is still
+    very C-ish and has many constants that Python doesn't understand.
 
-mapping = FakeMapping()
-namespace = mapping.__dict__
+    The function 'try_to_guess()' below understands a lot of PySide's
+    peculiar way to assume local context. If it is able to do the guess,
+    then the result is inserted into the dict, so the search happens
+    not again. For everything that is not covered by these automatic
+    guesses, we provide an entry in 'type_map' that resolves it.
 
-class _empty:
-    """ marks "no value found". We cannot use None here."""
+    In effect, 'type_map' maps text to real Python objects.
+"""
 
 def dprint(*args, **kw):
     if _DEBUG:
@@ -138,17 +122,17 @@ def _resolve_number(thing):
     try:
         return eval(thing, namespace)
     except Exception:
-        return _empty
+        return None
 
 def try_to_guess(thing, valtype):
     res = _resolve_number(thing)
-    if res is not _empty:
+    if res is not None:
         return res
     if "." not in thing and "(" not in thing:
         text = "{}.{}".format(valtype, thing)
         try:
             return eval(text, namespace)
-        except Exception as e:
+        except Exception:
             pass
     typewords = valtype.split(".")
     valwords = thing.split(".")
@@ -160,25 +144,23 @@ def try_to_guess(thing, valtype):
             text = ".".join(typewords[:idx] + valwords)
             try:
                 return eval(text, namespace)
-            except Exception as e:
+            except Exception:
                 pass
-    return _empty
+    return None
 
-def _resolve_value_reloaded(thing, valtype, type_map, line, maybe_redo):
+def _resolve_value(thing, valtype, line):
     if thing in type_map:
         return type_map[thing]
     try:
         res = eval(thing, namespace)
         type_map[thing] = res
         return res
-    except Exception as e:
+    except Exception:
         pass
-    res = try_to_guess(thing, valtype) if valtype else _empty
-    if res is not _empty:
+    res = try_to_guess(thing, valtype) if valtype else None
+    if res is not None:
         type_map[thing] = res
         return res
-    if maybe_redo:
-        return _empty
     warnings.warn("""pyside_type_init:
 
         UNRECOGNIZED:   {!r}
@@ -190,35 +172,10 @@ def _resolve_value_reloaded(thing, valtype, type_map, line, maybe_redo):
         raise RuntimeError
     return thing
 
-def _resolve_value(thing, valtype, type_map, line):
-    """
-    Load a value after eventually reloading.
-
-    If an error occurs, there is maybe a new module imported that we
-    don't have, yet. Reload the mapping module and try again.
-    """
-    try:
-        val = _resolve_value_reloaded(thing, valtype, type_map, line, True)
-    except Exception:
-        val = _empty
-    if val is not _empty:
-        return val
-    global mapping, namespace
-    if type(mapping) is not types.ModuleType:
-        # lazy import
-        from . import mapping
-        namespace = mapping.__dict__
-        type_map.update(mapping.type_map)
-        return _resolve_value(thing, valtype, type_map, line)
-    reload(mapping)
-    dprint("Matrix reloaded")
-    type_map.update(mapping.type_map)
-    return _resolve_value_reloaded(thing, valtype, type_map, line, False)
-
-def _resolve_type(thing, type_map, line):
-    return _resolve_value(thing, None, type_map, line)
+def _resolve_type(thing, line):
+    return _resolve_value(thing, None, line)
 
-def calculate_props(line, type_map):
+def calculate_props(line):
     line = line.strip()
     res = _parse_line(line)
     arglist = res["arglist"]
@@ -226,14 +183,14 @@ def calculate_props(line, type_map):
     _defaults = []
     for tup in arglist:
         name, ann = tup[:2]
-        annotations[name] = _resolve_type(ann, type_map, line)
+        annotations[name] = _resolve_type(ann, line)
         if len(tup) == 3:
-            default = _resolve_value(tup[2], ann, type_map, line)
+            default = _resolve_value(tup[2], ann, line)
             _defaults.append(default)
     defaults = tuple(_defaults)
     returntype = res["returntype"]
     if returntype is not None:
-        annotations["return"] = _resolve_type(returntype, type_map, line)
+        annotations["return"] = _resolve_type(returntype, line)
     props = {}
     props["defaults"] = defaults
     props["kwdefaults"] = {}
@@ -246,17 +203,18 @@ def calculate_props(line, type_map):
     props["multi"] = res["multi"]
     return props
 
-def pyside_type_init(typemod, sig_str, type_map):
+def pyside_type_init(typemod, sig_str):
     dprint()
     if type(typemod) is types.ModuleType:
         dprint("Initialization of module '{}'".format(typemod.__name__))
     else:
         dprint("Initialization of type '{}.{}'".format(typemod.__module__,
                                                        typemod.__name__))
+    update_mapping()
     ret = {}
     multi_props = []
     for line in sig_str.strip().splitlines():
-        props = calculate_props(line, type_map)
+        props = calculate_props(line)
         shortname = props["name"]
         multi = props["multi"]
         if multi is None:
@@ -273,7 +231,4 @@ def pyside_type_init(typemod, sig_str, type_map):
             multi_props = []
     return ret
 
-pyside_type_init = functools.partial(pyside_type_init,
-                                     type_map=mapping.type_map)
-
 # end of file