Generate Hinting Stubs Automatically

The script is now automatically called in the cmake build,
as part of the create_pyside_module macro.

The script runs after every module build and tries to
generate .pyi files. This does not need to succeed, but
will generate all files in the end. The script has been prepared
to allow partial runs without overhead.

After integration of the .pyi generation into cmake, these files
are also installed into the install directory by cmake.
For wheel building, setup.py has entries, too.

Building a full project with all modules revealed a bug in the
signature module that allowed unsupported function objects.
Module enum_sig had to be changed to suppress types
which have no ancestry in shiboken.

PYTHONPATH was avoided because it was not Windows
compatible. Instead, the script was changed to accept
"--sys-path" and "--lib-path" parameters. The latter evaluates
either to PATH or LD_LIBRARY_PATH.

The necessity to create .pyi files while the project is in the
build process showed a hard to track down error condition
in PySide_BuildSignatureProps.

Simple logging was added as a start of introducing logging
everywhere.

Task-number: PYSIDE-735
Change-Id: I6b3eec4b823d026583e902023badedeb06fe0961
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>

8 files changed, 76 insertions, 55 deletions

diff --git a/sources/shiboken2/libshiboken/pep384impl.h b/sources/shiboken2/libshiboken/pep384impl.h
index 78b9defb5..bfa8f38a7 100644
--- a/sources/shiboken2/libshiboken/pep384impl.h
+++ b/sources/shiboken2/libshiboken/pep384impl.h
@@ -333,10 +333,11 @@ LIBSHIBOKEN_API PyObject *PepFunction_Get(PyObject *, const char *);
 #define PyFunction_Check(op)        (Py_TYPE(op) == PepFunction_TypePtr)
 #define PyFunction_GET_CODE(func)   PyFunction_GetCode(func)
 
-#define PyFunction_GetCode(func)        PepFunction_Get((PyObject *)func, "__code__")
-#define PepFunction_GetName(func)    PepFunction_Get((PyObject *)func, "__name__")
+#define PyFunction_GetCode(func)    PepFunction_Get((PyObject *)func, "__code__")
+#define PepFunction_GetName(func)   PepFunction_Get((PyObject *)func, "__name__")
 #else
-#define PepFunction_GetName(func)    (((PyFunctionObject *)func)->func_name)
+#define PepFunction_TypePtr         (&PyFunction_Type)
+#define PepFunction_GetName(func)   (((PyFunctionObject *)func)->func_name)
 #endif
 
 /*****************************************************************************
diff --git a/sources/shiboken2/libshiboken/signature.cpp b/sources/shiboken2/libshiboken/signature.cpp
index 92ce3e50a..922f85906 100644
--- a/sources/shiboken2/libshiboken/signature.cpp
+++ b/sources/shiboken2/libshiboken/signature.cpp
@@ -120,6 +120,8 @@ pyside_sm_get___signature__(PyObject *sm, const char *modifier)
 {
     init_module_2();
     Shiboken::AutoDecRef func(PyObject_GetAttrString(sm, "__func__"));
+    if (Py_TYPE(func) == PepFunction_TypePtr)
+        Py_RETURN_NONE;
     return GetSignature_Function(func, modifier);
 }
 
@@ -299,28 +301,38 @@ GetClassKey(PyObject *ob)
     return Py_BuildValue("O", class_name.object());
 }
 
+static PyObject *empty_dict = nullptr;
+
+static PyObject *
+TypeKey_to_PropsDict(PyObject *type_key, PyObject *obtype)
+{
+    PyObject *dict = PyDict_GetItem(pyside_globals->arg_dict, type_key);
+    if (dict == nullptr) {
+        if (empty_dict == nullptr)
+            empty_dict = PyDict_New();
+        dict = empty_dict;
+    }
+    if (PyTuple_Check(dict))
+        dict = PySide_BuildSignatureProps(obtype);
+    return dict;
+}
+
 static PyObject *
 GetSignature_Function(PyObject *ob_func, const char *modifier)
 {
+    // make sure that we look into PyCFunction, only...
+    if (Py_TYPE(ob_func) == PepFunction_TypePtr)
+        Py_RETURN_NONE;
     Shiboken::AutoDecRef typemod(GetClassOfFunc(ob_func));
     Shiboken::AutoDecRef type_key(GetClassKey(typemod));
     if (type_key.isNull())
         Py_RETURN_NONE;
-    PyObject *dict = PyDict_GetItem(pyside_globals->arg_dict, type_key);
-    if (dict == NULL)
-        Py_RETURN_NONE;
-    if (PyTuple_Check(dict)) {
-        /*
-         * We do the initialization lazily.
-         * This has also the advantage that we can freely import PySide.
-         */
-        dict = PySide_BuildSignatureProps(typemod);
-        if (dict == NULL)
-            Py_RETURN_NONE;
-    }
+    PyObject *dict = TypeKey_to_PropsDict(type_key, typemod);
+    if (dict == nullptr)
+        return nullptr;
     Shiboken::AutoDecRef func_name(PyObject_GetAttrString(ob_func, "__name__"));
     PyObject *props = !func_name.isNull() ? PyDict_GetItem(dict, func_name) : nullptr;
-    if (props == NULL)
+    if (props == nullptr)
         Py_RETURN_NONE;
 
     int flags = PyCFunction_GET_FLAGS(ob_func);
@@ -333,8 +345,7 @@ GetSignature_Function(PyObject *ob_func, const char *modifier)
         sig_kind = "staticmethod";
     else
         sig_kind = "method";
-    PyObject *ret = GetSignature_Cached(props, sig_kind, modifier);
-    return ret;
+    return GetSignature_Cached(props, sig_kind, modifier);
 }
 
 static PyObject *
@@ -346,20 +357,11 @@ GetSignature_Wrapper(PyObject *ob, const char *modifier)
 
     if (func_name.isNull() || objclass.isNull() || class_key.isNull())
         return nullptr;
-    PyObject *dict = PyDict_GetItem(pyside_globals->arg_dict, class_key);
-    if (dict == NULL)
-        Py_RETURN_NONE;
-    if (PyTuple_Check(dict)) {
-        /*
-         * We do the initialization lazily.
-         * This has also the advantage that we can freely import PySide.
-         */
-        dict = PySide_BuildSignatureProps(objclass);
-        if (dict == NULL)
-            Py_RETURN_NONE;
-    }
+    PyObject *dict = TypeKey_to_PropsDict(class_key, objclass);
+    if (dict == nullptr)
+        return nullptr;
     PyObject *props = PyDict_GetItem(dict, func_name);
-    if (props == NULL)
+    if (props == nullptr)
         Py_RETURN_NONE;
     return GetSignature_Cached(props, "method", modifier);
 }
@@ -370,18 +372,11 @@ GetSignature_TypeMod(PyObject *ob, const char *modifier)
     Shiboken::AutoDecRef ob_name(PyObject_GetAttrString(ob, "__name__"));
     Shiboken::AutoDecRef ob_key(GetClassKey(ob));
 
-    PyObject *dict = PyDict_GetItem(pyside_globals->arg_dict, ob_key);
-    if (dict == NULL)
-        Py_RETURN_NONE;
-
-    if (PyTuple_Check(dict)) {
-        dict = PySide_BuildSignatureProps(ob);
-        if (dict == NULL) {
-            Py_RETURN_NONE;
-        }
-    }
+    PyObject *dict = TypeKey_to_PropsDict(ob_key, ob);
+    if (dict == nullptr)
+        return nullptr;
     PyObject *props = PyDict_GetItem(dict, ob_name);
-    if (props == NULL)
+    if (props == nullptr)
         Py_RETURN_NONE;
     return GetSignature_Cached(props, "method", modifier);
 }
@@ -587,6 +582,9 @@ get_signature(PyObject *self, PyObject *args)
 
     if (!PyArg_ParseTuple(args, "O|s", &ob, &modifier))
         return NULL;
+    if (Py_TYPE(ob) == PepFunction_TypePtr)
+        Py_RETURN_NONE;
+
     if (Py_TYPE(ob) == &PyCFunction_Type)
         return pyside_cf_get___signature__(ob, modifier);
     if (Py_TYPE(ob) == PepStaticMethod_TypePtr)
@@ -723,14 +721,13 @@ static PyMethodDef signature_methods[] = {
 static void
 init_module_2(void)
 {
-    static int init_done = 0, initializing = 0;
+    static int init_done = 0;
 
     if (!init_done) {
-        if (initializing)
-            Py_FatalError("Init 2 called recursively!");
-        init_phase_2(pyside_globals, signature_methods);
+        // Phase 2 will call __init__.py which touches a signature, itself.
+        // Therefore we set init_done prior to init_phase_2().
         init_done = 1;
-        initializing = 0;
+        init_phase_2(pyside_globals, signature_methods);
     }
 }
 
@@ -751,8 +748,14 @@ PySide_BuildSignatureProps(PyObject *classmod)
     if (arg_tup == nullptr)
         return nullptr;
     PyObject *dict = PyObject_CallObject(pyside_globals->sigparse_func, arg_tup);
-    if (dict == nullptr)
-        return nullptr;
+    if (dict == nullptr) {
+        if (PyErr_Occurred())
+            return nullptr;
+        // No error: return an empty dict.
+        if (empty_dict == nullptr)
+            empty_dict = PyDict_New();
+        return empty_dict;
+    }
 
     // We replace the arguments by the result dict.
     if (PyDict_SetItem(pyside_globals->arg_dict, type_key, dict) < 0)
diff --git a/sources/shiboken2/shibokenmodule/CMakeLists.txt b/sources/shiboken2/shibokenmodule/CMakeLists.txt
index 342b6d908..0eba3eaff 100644
--- a/sources/shiboken2/shibokenmodule/CMakeLists.txt
+++ b/sources/shiboken2/shibokenmodule/CMakeLists.txt
@@ -74,7 +74,7 @@ else()
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/support/signature/backport_inspect.py"
                    "${CMAKE_CURRENT_BINARY_DIR}/support/signature/backport_inspect.py" COPYONLY)
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/support/signature/typing27.py"
-                   "${CMAKE_CURRENT_BINARY_DIR}/support/signature/typing.py" COPYONLY)
+                   "${CMAKE_CURRENT_BINARY_DIR}/support/signature/typing27.py" COPYONLY)
 endif()
 install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/support"
         DESTINATION "${PYTHON_SITE_PACKAGES}/shiboken2")
diff --git a/sources/shiboken2/shibokenmodule/support/signature/__init__.py b/sources/shiboken2/shibokenmodule/support/signature/__init__.py
index 253ba98dc..d0791df04 100644
--- a/sources/shiboken2/shibokenmodule/support/signature/__init__.py
+++ b/sources/shiboken2/shibokenmodule/support/signature/__init__.py
@@ -41,4 +41,7 @@ from __future__ import print_function, absolute_import
 
 # Trigger initialization phase 2.
 _ = type.__signature__
-from signature_loader import get_signature, inspect, typing
+
+## from signature_loader import get_signature, inspect, typing
+# This causes a recursion in Python 2!
+# We do everything from signature_loader, instead.
diff --git a/sources/shiboken2/shibokenmodule/support/signature/lib/enum_sig.py b/sources/shiboken2/shibokenmodule/support/signature/lib/enum_sig.py
index f79f3266a..013ec36cc 100644
--- a/sources/shiboken2/shibokenmodule/support/signature/lib/enum_sig.py
+++ b/sources/shiboken2/shibokenmodule/support/signature/lib/enum_sig.py
@@ -81,6 +81,10 @@ class ExactEnumerator(object):
             return ret
 
     def klass(self, class_name, klass):
+        if not "Shiboken" in repr(klass.mro()):
+            # don't look into any foreign classes!
+            ret = self.result_type()
+            return ret
         bases_list = []
         for base in klass.__bases__:
             name = base.__name__
diff --git a/sources/shiboken2/shibokenmodule/support/signature/loader.py b/sources/shiboken2/shibokenmodule/support/signature/loader.py
index 170fb0a2a..de27d441c 100644
--- a/sources/shiboken2/shibokenmodule/support/signature/loader.py
+++ b/sources/shiboken2/shibokenmodule/support/signature/loader.py
@@ -146,7 +146,8 @@ with ensure_import_support():
     else:
         import inspect
         namespace = inspect.__dict__
-        from support.signature import typing
+        from support.signature import typing27 as typing
+        typing.__name__ = "typing"
         from support.signature import backport_inspect as inspect
         _doc = inspect.__doc__
         inspect.__dict__.update(namespace)
diff --git a/sources/shiboken2/shibokenmodule/support/signature/mapping.py b/sources/shiboken2/shibokenmodule/support/signature/mapping.py
index bca1ce307..3e76cd94a 100644
--- a/sources/shiboken2/shibokenmodule/support/signature/mapping.py
+++ b/sources/shiboken2/shibokenmodule/support/signature/mapping.py
@@ -104,7 +104,7 @@ class _NotCalled(str):
     real object is needed, the wrapper can simply be called.
     """
     def __repr__(self):
-        suppress = "PySide2.support.signature.typing."
+        suppress = "support.signature.typing."
         text = self[len(suppress):] if self.startswith(suppress) else self
         return "{}({})".format(type(self).__name__, text)
 
@@ -119,7 +119,8 @@ class Virtual(_NotCalled):
 
 # Other types I simply could not find.
 class Missing(_NotCalled):
-    pass
+    def __repr__(self):
+        return '{}("{}")'.format(type(self).__name__, self)
 
 class Invalid(_NotCalled):
     pass
@@ -148,7 +149,14 @@ class Reloader(object):
             g = globals()
         for mod_name in self.uninitialized[:]:
             for prefix in self._prefixes:
-                if prefix + mod_name in sys.modules:
+                import_name = prefix + mod_name
+                if import_name in sys.modules:
+                    # check if this is a real module
+                    obj = sys.modules[import_name]
+                    if not getattr(obj, "__file__", None) or os.path.isdir(obj.__file__):
+                        raise ImportError("Module '{mod_name}' is at most a "
+                                          "namespace!".format(**locals()))
+                    # module is real
                     self.uninitialized.remove(mod_name)
                     proc_name = "init_" + mod_name
                     if proc_name in g:
diff --git a/sources/shiboken2/shibokenmodule/support/signature/parser.py b/sources/shiboken2/shibokenmodule/support/signature/parser.py
index 4bb1bf234..5178d9ef9 100644
--- a/sources/shiboken2/shibokenmodule/support/signature/parser.py
+++ b/sources/shiboken2/shibokenmodule/support/signature/parser.py
@@ -76,6 +76,7 @@ def dprint(*args, **kw):
         import pprint
         for arg in args:
             pprint.pprint(arg)
+            sys.stdout.flush()
 
 def _parse_line(line):
     line_re = r"""