Implement Embedding To Make Signatures Always Available

Situation..

PySide works fine with normal applications. But when installers
are used to pack the application together, then the signature
extension cannot be loaded.

This is a problem that exists since the signature extension was
written. But starting with PySide 5.12.1, the signature extension
is very visible, because it is used to support the __doc__ attribute.

There have beed successful attempts to solve the problem for
PyInstaller and Py2App. But there are more packers available,
and they all need a change both in PySide and in the packer.

Solution..

To solve this problem once and for all, we embed the Python
support files in the binary shiboken package. When the Python
files are not normally accessible, they are unpacked from a ZIP
file.

Details..

- The embedded files shall only be used when the normal files
  are not available,

- The signature extension should no longer be lazily loaded.
  When the application starts, all files should be present.

- We drop support for shiboken2.support.signature and
  use a single, independen folder 'shibokensupport' (proposal).
  This avoids problems with multiple existence of the shiboken2
  folder. PySide2.support.signature remains the official interface,
  but it's only an alias when PySide2 exists.

- The embedding is used when the normal files cannot be
  loaded for some reason. It can be enforced by a sys variable
  "pyside_uses_embedding".

- Testcase is included.

- Tested with PyInstaller on macOS

Fixes: PYSIDE-932
Fixes: PYSIDE-942
Change-Id: Iaa62dc0623503a2e0943eedd10d2b2484c65cc2d
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

9 files changed, 133 insertions, 171 deletions

diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/backport_inspect.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/backport_inspect.py
index fbd82cc53..c690493b6 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/backport_inspect.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/backport_inspect.py
@@ -111,10 +111,9 @@ CO_NOFREE       = 0x0040
 
 # This function was changed: 'builtins' and 'qualname' don't exist.
 # We use '__builtin__' and '__name__' instead.
-# It is further changed because we use a local copy of typing
 def formatannotation(annotation, base_module=None):
-    if getattr(annotation, '__module__', None) == 'support.signature.typing27':
-        return repr(annotation).replace('support.signature.typing27', 'typing')
+    if getattr(annotation, '__module__', None) == 'typing':
+        return repr(annotation).replace('typing.', '')
     if isinstance(annotation, type):
         if annotation.__module__ in ('__builtin__', base_module):
             return annotation.__name__
@@ -378,7 +377,7 @@ class Parameter(object):
 
         # Add annotation and default value
         if self._annotation is not _empty:
-            formatted = '{}:{}'.format(formatted,
+            formatted = '{}: {}'.format(formatted,
                                        formatannotation(self._annotation))
 
         if self._default is not _empty:
@@ -892,65 +891,3 @@ class Signature(object):
 def signature(obj, follow_wrapped=True):
     """Get a signature object for the passed callable."""
     return Signature.from_callable(obj, follow_wrapped=follow_wrapped)
-
-
-def _main():
-    """ Logic for inspecting an object given at command line """
-    import argparse
-    import importlib
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'object',
-         help="The object to be analysed. "
-              "It supports the 'module:qualname' syntax")
-    parser.add_argument(
-        '-d', '--details', action='store_true',
-        help='Display info about the module rather than its source code')
-
-    args = parser.parse_args()
-
-    target = args.object
-    mod_name, has_attrs, attrs = target.partition(":")
-    try:
-        obj = module = importlib.import_module(mod_name)
-    except Exception as exc:
-        msg = "Failed to import {} ({}: {})".format(mod_name,
-                                                    type(exc).__name__,
-                                                    exc)
-        print(msg, file=sys.stderr)
-        exit(2)
-
-    if has_attrs:
-        parts = attrs.split(".")
-        obj = module
-        for part in parts:
-            obj = getattr(obj, part)
-
-    if module.__name__ in sys.builtin_module_names:
-        print("Can't get info for builtin modules.", file=sys.stderr)
-        exit(1)
-
-    if args.details:
-        print('Target: {}'.format(target))
-        print('Origin: {}'.format(getsourcefile(module)))
-        print('Cached: {}'.format(module.__cached__))
-        if obj is module:
-            print('Loader: {}'.format(repr(module.__loader__)))
-            if hasattr(module, '__path__'):
-                print('Submodule search path: {}'.format(module.__path__))
-        else:
-            try:
-                __, lineno = findsource(obj)
-            except Exception:
-                pass
-            else:
-                print('Line: {}'.format(lineno))
-
-        print('\n')
-    else:
-        print(getsource(obj))
-
-
-if __name__ == "__main__":
-    _main()
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/fix-complaints.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/fix-complaints.py
index cdd84f9be..3818ad602 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/fix-complaints.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/fix-complaints.py
@@ -1,3 +1,6 @@
+# This Python file uses the following encoding: utf-8
+# It has been edited by fix-complaints.py .
+
 #############################################################################
 ##
 ## Copyright (C) 2019 The Qt Company Ltd.
@@ -42,14 +45,15 @@ from __future__ import print_function, absolute_import
 """
 fix-complaints.py
 
-This module fixes the buildbot messages of external python modules.
+This module fixes the buildbot messages of external python files.
 Run it once after copying a new version. It is idem-potent, unless
 you are changing messages (what I did, of course :-) .
 """
 
 import os
+import glob
 
-patched_modules = "backport_inspect typing27"
+patched_file_patterns = "backport_inspect.py typing27.py python_minilib_*.py"
 
 offending_words = {
     "behavio""ur": "behavior",
@@ -79,9 +83,12 @@ def patch_file(fname):
             f.write("".join(lines))
 
 def doit():
-    dir = os.path.dirname(__file__)
-    for name in patched_modules.split():
-        fname = os.path.join(dir, name + ".py")
+    dirname = os.path.dirname(__file__)
+    patched_files = []
+    for name in patched_file_patterns.split():
+        pattern = os.path.join(dirname, name)
+        patched_files += glob.glob(pattern)
+    for fname in patched_files:
         print("Working on", fname)
         patch_file(fname)
 
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py
index 9447e9c25..ee541d0ea 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py
@@ -39,6 +39,4 @@
 
 from __future__ import print_function, absolute_import
 
-# from shibokensupport.signature import get_signature, inspect, typing
-# This gives a problem with Python 2. We do it in the loader, instead.
-__all__ = "get_signature inspect typing layout mapping lib".split()
+__all__ = "get_signature layout mapping lib".split()
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py
index 1443d7927..cb148830f 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py
@@ -56,7 +56,8 @@ enough to produce a useful ValueError.
 This matter will be improved in a later version.
 """
 
-from shibokensupport.signature import get_signature, inspect
+from shibokensupport.signature import inspect
+from shibokensupport.signature import get_signature
 from shibokensupport.signature.mapping import update_mapping, namespace
 from textwrap import dedent
 
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
index 90affbcfd..e6f6dc379 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
@@ -50,7 +50,8 @@ by producing a lot of clarity.
 """
 
 import sys
-from shibokensupport.signature import get_signature, inspect
+from shibokensupport.signature import inspect
+from shibokensupport.signature import get_signature
 
 
 class ExactEnumerator(object):
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
index 3a8d614db..a564aedb9 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
@@ -1,3 +1,6 @@
+# This Python file uses the following encoding: utf-8
+# It has been edited by fix-complaints.py .
+
 #############################################################################
 ##
 ## Copyright (C) 2019 The Qt Company Ltd.
@@ -42,33 +45,26 @@ from __future__ import print_function, absolute_import
 """
 loader.py
 
-The loader has to lazy-load the signature module and also provides a few
-Python modules to support Python 2.7 .
+The loader has to load the signature module completely at startup,
+to make sure that the functions are available when needed.
+This is meanwhile necessary to make the '__doc__' attribute work correctly.
+
+It does not mean that everything is initialized in advance. Only the modules
+are loaded completely after 'import PySide2'.
+
+This version uses both a normal directory, but has also an embedded ZIP file
+as a fallback solution. The ZIP file is generated by 'embedding_generator.py'
+and embedded into 'signature.cpp' as "embed/signature.inc".
 
-This version uses both a normal directory, but has also an embedded zip file
-as a fallback solution.
+Meanwhile, the ZIP file grew so much, that MSVC had problems
+with it's 64k string limit, so we had to break the string up.
+See 'zipped_string_sequence' in signature.cpp.
 """
 
 import sys
 import os
 import traceback
 import types
-from contextlib import contextmanager
-
-"""
-A note on the import problem (solved):
-
-During the tests, the shiboken build structure has the layout
-
-    shiboken2/shibokenmodule/shiboken2.abi3.so
-
-and the name "shiboken2" in sys.modules points directly to the binary
-file, hiding the outer shiboken2 module.
-
-To fix that, we temporarily remove the binary from sys.path,
-do the needed imports and then restore the binary.
-This action was put into a context manager for readability.
-"""
 
 # On Python 2, we only have ImportError, which is way too coarse.
 # When problems occour, please use Python 3, because it has the finer
@@ -79,31 +75,6 @@ try:
 except NameError:
     ModuleNotFoundError = ImportError
 
-@contextmanager
-def ensure_import_shibokensupport():
-    # Make sure that we always have the shibokensupport containing package first.
-    # Also remove any prior loaded module of this name, just in case.
-    sbk_support_dir = os.path.abspath(os.path.join(__file__, "..", "..", ".."))
-    assert os.path.basename(sbk_support_dir) == "files.dir"
-    sys.path.insert(0, sbk_support_dir)
-
-    sbk = "shibokensupport"
-    if sbk in sys.modules:
-        del sys.modules[sbk]
-    for key in list(key for key in sys.modules if key.startswith(sbk + ".")):
-        del sys.modules[key]
-    try:
-        import shibokensupport
-        yield
-    except Exception as e:
-        print("Problem importing shibokensupport:")
-        print(e)
-        traceback.print_exc()
-        sys.stdout.flush()
-        sys.exit(-1)
-    sys.path.remove(sbk_support_dir)
-
-
 # patching inspect's formatting to keep the word "typing":
 def formatannotation(annotation, base_module=None):
     # if getattr(annotation, '__module__', None) == 'typing':
@@ -143,53 +114,102 @@ def seterror_argument(args, func_name):
 def make_helptext(func):
     return errorhandler.make_helptext(func)
 
-with ensure_import_shibokensupport():
-    import signature_loader
-    import shibokensupport.signature
-    shibokensupport.signature.get_signature = signature_loader.get_signature
-    del signature_loader  # protect this dir, too?
+import signature_bootstrap
+from shibokensupport import signature
+signature.get_signature = signature_bootstrap.get_signature
+del signature_bootstrap
+
 
-    if sys.version_info >= (3,):
-        import typing
-        import inspect
-        inspect.formatannotation = formatannotation
+def _get_modname(mod):
+    return mod.__spec__.name if getattr(mod, "__spec__", None) else mod.__name__
+
+def _set_modname(mod, name):
+    if getattr(mod, "__spec__", None):
+        mod.__spec__.name = name
     else:
-        import inspect
-        namespace = inspect.__dict__
-        from shibokensupport.signature import typing27 as typing
-        typing.__name__ = "typing"
-        # Fix the module names in typing if possible. This is important since
-        # the typing names should be I/O compatible, so that typing.Dict
-        # shows itself as "typing.Dict".
-        for name, obj in typing.__dict__.items():
-            if hasattr(obj, "__module__"):
-                try:
-                    obj.__module__ = "typing"
-                except (TypeError, AttributeError):
-                    pass
-        from shibokensupport.signature import backport_inspect as inspect
-        _doc = inspect.__doc__
-        inspect.__dict__.update(namespace)
-        inspect.__doc__ += _doc
-        # force inspect to find all attributes. See "heuristic" in pydoc.py!
-        inspect.__all__ = list(x for x in dir(inspect) if not x.startswith("_"))
-    typing.TypeVar.__repr__ = _typevar__repr__
-
-    def put_into_package(module, package):
-        # take the last component of the module name
-        name = module.__name__.rsplit(".", 1)[-1]
-        # allow access as {package}.typing
+        mod.__name__ = name
+
+
+def put_into_package(package, module, override=None):
+    # take the last component of the module name
+    name = (override if override else _get_modname(module)).rsplit(".", 1)[-1]
+    # allow access as {package}.typing
+    if package:
         setattr(package, name, module)
-        # put into sys.modules as a package to allow all import options
-        fullname = "{}.{}".format(package.__name__, name)
-        sys.modules[fullname] = module
-
-    put_into_package(typing, shibokensupport.signature)
-    put_into_package(inspect, shibokensupport.signature)
-    from shibokensupport.signature import mapping
-    from shibokensupport.signature import errorhandler
-    from shibokensupport.signature import layout
-    from shibokensupport.signature.lib import enum_sig
-    from shibokensupport.signature.parser import pyside_type_init
+    # put into sys.modules as a package to allow all import options
+    fullname = "{}.{}".format(_get_modname(package), name) if package else name
+    _set_modname(module, fullname)
+    # publish new dotted name in sys.modules
+    sys.modules[fullname] = module
+
+
+# Debug: used to inspect what each step loads
+def list_modules(message):
+    ext_modules = {key:value for (key, value) in sys.modules.items()
+                              if hasattr(value, "__file__")}
+    print("SYS.MODULES", message, len(sys.modules), len(ext_modules))
+    for (name, module) in sorted(ext_modules.items()):
+        print("  {:23}".format(name), repr(module)[:70])
+
+
+if sys.version_info >= (3,):
+    import typing
+    import inspect
+    inspect.formatannotation = formatannotation
+else:
+    from shibokensupport import typing27 as typing
+    import inspect
+    namespace = inspect.__dict__
+    from shibokensupport import backport_inspect as inspect
+    _doc = inspect.__doc__
+    inspect.__dict__.update(namespace)
+    inspect.__doc__ += _doc
+    # force inspect to find all attributes. See "heuristic" in pydoc.py!
+    inspect.__all__ = list(x for x in dir(inspect) if not x.startswith("_"))
+typing.TypeVar.__repr__ = _typevar__repr__
+
+# Fix the module names in typing if possible. This is important since
+# the typing names should be I/O compatible, so that typing.Dict
+# shows itself as "typing.Dict".
+for name, obj in typing.__dict__.items():
+    if hasattr(obj, "__module__"):
+        try:
+            obj.__module__ = "typing"
+        except (TypeError, AttributeError):
+            pass
+
+import shibokensupport
+put_into_package(shibokensupport.signature, typing, "typing")
+put_into_package(shibokensupport.signature, inspect, "inspect")
+
+
+def move_into_pyside_package():
+    import PySide2
+    try:
+        import PySide2.support
+    except ModuleNotFoundError:
+        PySide2.support = types.ModuleType("PySide2.support")
+    put_into_package(PySide2.support, signature)
+    put_into_package(PySide2.support.signature, mapping)
+    put_into_package(PySide2.support.signature, errorhandler)
+    put_into_package(PySide2.support.signature, layout)
+    put_into_package(PySide2.support.signature, lib)
+    put_into_package(PySide2.support.signature, parser)
+    put_into_package(PySide2.support.signature.lib, enum_sig)
+
+    put_into_package(PySide2.support.signature, typing)
+    put_into_package(PySide2.support.signature, inspect)
+
+from shibokensupport.signature import mapping
+from shibokensupport.signature import errorhandler
+from shibokensupport.signature import layout
+from shibokensupport.signature import lib
+from shibokensupport.signature import parser
+from shibokensupport.signature.lib import enum_sig
+from shibokensupport.signature.parser import pyside_type_init
+
+if "PySide2" in sys.modules:
+    # We publish everything under "PySide2.support.signature", again.
+    move_into_pyside_package()
 
 # end of file
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
index 470dd676b..9a8ee4c4e 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
@@ -52,7 +52,6 @@ in sys.modules. This minimizes the loading overhead.
 import sys
 import struct
 import os
-import pkgutil
 
 from shibokensupport.signature import typing
 from shibokensupport.signature.typing import TypeVar, Generic
@@ -168,9 +167,7 @@ class _NotCalled(str):
     real object is needed, the wrapper can simply be called.
     """
     def __repr__(self):
-        suppress = "support.signature.typing27."
-        text = self[len(suppress):] if self.startswith(suppress) else self
-        return "{}({})".format(type(self).__name__, text)
+        return "{}({})".format(type(self).__name__, self)
 
     def __call__(self):
         from shibokensupport.signature.mapping import __dict__ as namespace
@@ -346,6 +343,7 @@ def init_other():
 def init_smart():
     type_map.update({
         "smart.SharedPtr": Missing("smart.SharedPtr"),  # bad object "SharedPtr<Obj >"
+        "smart.Smart.Integer2": int,
     })
     return locals()
 
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json
index 491ae8054..fbe4c51ab 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json
@@ -3,7 +3,7 @@
     "Name": "Python",
     "QDocModule": "QtForPython",
     "QtUsage": "Used for Qt for Python in the signature extension.",
-    "Description": "Qt for Python is an add-on for Python. The signature packages of PySide uses certain copied and adapted source files (backport_inspect.py, typing27.py). See the folder sources/pyside2/PySide2/support/signature .",
+    "Description": "Qt for Python is an add-on for Python. The signature packages of PySide uses certain copied and adapted source files (backport_inspect.py, typing27.py). See the folder sources/shiboken2/files.dir/shibokensupport/signature .",
     "Homepage": "http://www.python.org/",
     "Version": "3.7.0",
     "LicenseId": "Python-2.0",
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/typing27.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/typing27.py
index dba8f8c77..dba8f8c77 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/typing27.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/typing27.py