Implement Embedding To Make Signatures Always Available

Situation..

PySide works fine with normal applications. But when installers
are used to pack the application together, then the signature
extension cannot be loaded.

This is a problem that exists since the signature extension was
written. But starting with PySide 5.12.1, the signature extension
is very visible, because it is used to support the __doc__ attribute.

There have beed successful attempts to solve the problem for
PyInstaller and Py2App. But there are more packers available,
and they all need a change both in PySide and in the packer.

Solution..

To solve this problem once and for all, we embed the Python
support files in the binary shiboken package. When the Python
files are not normally accessible, they are unpacked from a ZIP
file.

Details..

- The embedded files shall only be used when the normal files
  are not available,

- The signature extension should no longer be lazily loaded.
  When the application starts, all files should be present.

- We drop support for shiboken2.support.signature and
  use a single, independen folder 'shibokensupport' (proposal).
  This avoids problems with multiple existence of the shiboken2
  folder. PySide2.support.signature remains the official interface,
  but it's only an alias when PySide2 exists.

- The embedding is used when the normal files cannot be
  loaded for some reason. It can be enforced by a sys variable
  "pyside_uses_embedding".

- Testcase is included.

- Tested with PyInstaller on macOS

Fixes: PYSIDE-932
Fixes: PYSIDE-942
Change-Id: Iaa62dc0623503a2e0943eedd10d2b2484c65cc2d
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

1 files changed, 110 insertions, 90 deletions

diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
index 3a8d614db..a564aedb9 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
@@ -1,3 +1,6 @@
+# This Python file uses the following encoding: utf-8
+# It has been edited by fix-complaints.py .
+
 #############################################################################
 ##
 ## Copyright (C) 2019 The Qt Company Ltd.
@@ -42,33 +45,26 @@ from __future__ import print_function, absolute_import
 """
 loader.py
 
-The loader has to lazy-load the signature module and also provides a few
-Python modules to support Python 2.7 .
+The loader has to load the signature module completely at startup,
+to make sure that the functions are available when needed.
+This is meanwhile necessary to make the '__doc__' attribute work correctly.
+
+It does not mean that everything is initialized in advance. Only the modules
+are loaded completely after 'import PySide2'.
+
+This version uses both a normal directory, but has also an embedded ZIP file
+as a fallback solution. The ZIP file is generated by 'embedding_generator.py'
+and embedded into 'signature.cpp' as "embed/signature.inc".
 
-This version uses both a normal directory, but has also an embedded zip file
-as a fallback solution.
+Meanwhile, the ZIP file grew so much, that MSVC had problems
+with it's 64k string limit, so we had to break the string up.
+See 'zipped_string_sequence' in signature.cpp.
 """
 
 import sys
 import os
 import traceback
 import types
-from contextlib import contextmanager
-
-"""
-A note on the import problem (solved):
-
-During the tests, the shiboken build structure has the layout
-
-    shiboken2/shibokenmodule/shiboken2.abi3.so
-
-and the name "shiboken2" in sys.modules points directly to the binary
-file, hiding the outer shiboken2 module.
-
-To fix that, we temporarily remove the binary from sys.path,
-do the needed imports and then restore the binary.
-This action was put into a context manager for readability.
-"""
 
 # On Python 2, we only have ImportError, which is way too coarse.
 # When problems occour, please use Python 3, because it has the finer
@@ -79,31 +75,6 @@ try:
 except NameError:
     ModuleNotFoundError = ImportError
 
-@contextmanager
-def ensure_import_shibokensupport():
-    # Make sure that we always have the shibokensupport containing package first.
-    # Also remove any prior loaded module of this name, just in case.
-    sbk_support_dir = os.path.abspath(os.path.join(__file__, "..", "..", ".."))
-    assert os.path.basename(sbk_support_dir) == "files.dir"
-    sys.path.insert(0, sbk_support_dir)
-
-    sbk = "shibokensupport"
-    if sbk in sys.modules:
-        del sys.modules[sbk]
-    for key in list(key for key in sys.modules if key.startswith(sbk + ".")):
-        del sys.modules[key]
-    try:
-        import shibokensupport
-        yield
-    except Exception as e:
-        print("Problem importing shibokensupport:")
-        print(e)
-        traceback.print_exc()
-        sys.stdout.flush()
-        sys.exit(-1)
-    sys.path.remove(sbk_support_dir)
-
-
 # patching inspect's formatting to keep the word "typing":
 def formatannotation(annotation, base_module=None):
     # if getattr(annotation, '__module__', None) == 'typing':
@@ -143,53 +114,102 @@ def seterror_argument(args, func_name):
 def make_helptext(func):
     return errorhandler.make_helptext(func)
 
-with ensure_import_shibokensupport():
-    import signature_loader
-    import shibokensupport.signature
-    shibokensupport.signature.get_signature = signature_loader.get_signature
-    del signature_loader  # protect this dir, too?
+import signature_bootstrap
+from shibokensupport import signature
+signature.get_signature = signature_bootstrap.get_signature
+del signature_bootstrap
+
 
-    if sys.version_info >= (3,):
-        import typing
-        import inspect
-        inspect.formatannotation = formatannotation
+def _get_modname(mod):
+    return mod.__spec__.name if getattr(mod, "__spec__", None) else mod.__name__
+
+def _set_modname(mod, name):
+    if getattr(mod, "__spec__", None):
+        mod.__spec__.name = name
     else:
-        import inspect
-        namespace = inspect.__dict__
-        from shibokensupport.signature import typing27 as typing
-        typing.__name__ = "typing"
-        # Fix the module names in typing if possible. This is important since
-        # the typing names should be I/O compatible, so that typing.Dict
-        # shows itself as "typing.Dict".
-        for name, obj in typing.__dict__.items():
-            if hasattr(obj, "__module__"):
-                try:
-                    obj.__module__ = "typing"
-                except (TypeError, AttributeError):
-                    pass
-        from shibokensupport.signature import backport_inspect as inspect
-        _doc = inspect.__doc__
-        inspect.__dict__.update(namespace)
-        inspect.__doc__ += _doc
-        # force inspect to find all attributes. See "heuristic" in pydoc.py!
-        inspect.__all__ = list(x for x in dir(inspect) if not x.startswith("_"))
-    typing.TypeVar.__repr__ = _typevar__repr__
-
-    def put_into_package(module, package):
-        # take the last component of the module name
-        name = module.__name__.rsplit(".", 1)[-1]
-        # allow access as {package}.typing
+        mod.__name__ = name
+
+
+def put_into_package(package, module, override=None):
+    # take the last component of the module name
+    name = (override if override else _get_modname(module)).rsplit(".", 1)[-1]
+    # allow access as {package}.typing
+    if package:
         setattr(package, name, module)
-        # put into sys.modules as a package to allow all import options
-        fullname = "{}.{}".format(package.__name__, name)
-        sys.modules[fullname] = module
-
-    put_into_package(typing, shibokensupport.signature)
-    put_into_package(inspect, shibokensupport.signature)
-    from shibokensupport.signature import mapping
-    from shibokensupport.signature import errorhandler
-    from shibokensupport.signature import layout
-    from shibokensupport.signature.lib import enum_sig
-    from shibokensupport.signature.parser import pyside_type_init
+    # put into sys.modules as a package to allow all import options
+    fullname = "{}.{}".format(_get_modname(package), name) if package else name
+    _set_modname(module, fullname)
+    # publish new dotted name in sys.modules
+    sys.modules[fullname] = module
+
+
+# Debug: used to inspect what each step loads
+def list_modules(message):
+    ext_modules = {key:value for (key, value) in sys.modules.items()
+                              if hasattr(value, "__file__")}
+    print("SYS.MODULES", message, len(sys.modules), len(ext_modules))
+    for (name, module) in sorted(ext_modules.items()):
+        print("  {:23}".format(name), repr(module)[:70])
+
+
+if sys.version_info >= (3,):
+    import typing
+    import inspect
+    inspect.formatannotation = formatannotation
+else:
+    from shibokensupport import typing27 as typing
+    import inspect
+    namespace = inspect.__dict__
+    from shibokensupport import backport_inspect as inspect
+    _doc = inspect.__doc__
+    inspect.__dict__.update(namespace)
+    inspect.__doc__ += _doc
+    # force inspect to find all attributes. See "heuristic" in pydoc.py!
+    inspect.__all__ = list(x for x in dir(inspect) if not x.startswith("_"))
+typing.TypeVar.__repr__ = _typevar__repr__
+
+# Fix the module names in typing if possible. This is important since
+# the typing names should be I/O compatible, so that typing.Dict
+# shows itself as "typing.Dict".
+for name, obj in typing.__dict__.items():
+    if hasattr(obj, "__module__"):
+        try:
+            obj.__module__ = "typing"
+        except (TypeError, AttributeError):
+            pass
+
+import shibokensupport
+put_into_package(shibokensupport.signature, typing, "typing")
+put_into_package(shibokensupport.signature, inspect, "inspect")
+
+
+def move_into_pyside_package():
+    import PySide2
+    try:
+        import PySide2.support
+    except ModuleNotFoundError:
+        PySide2.support = types.ModuleType("PySide2.support")
+    put_into_package(PySide2.support, signature)
+    put_into_package(PySide2.support.signature, mapping)
+    put_into_package(PySide2.support.signature, errorhandler)
+    put_into_package(PySide2.support.signature, layout)
+    put_into_package(PySide2.support.signature, lib)
+    put_into_package(PySide2.support.signature, parser)
+    put_into_package(PySide2.support.signature.lib, enum_sig)
+
+    put_into_package(PySide2.support.signature, typing)
+    put_into_package(PySide2.support.signature, inspect)
+
+from shibokensupport.signature import mapping
+from shibokensupport.signature import errorhandler
+from shibokensupport.signature import layout
+from shibokensupport.signature import lib
+from shibokensupport.signature import parser
+from shibokensupport.signature.lib import enum_sig
+from shibokensupport.signature.parser import pyside_type_init
+
+if "PySide2" in sys.modules:
+    # We publish everything under "PySide2.support.signature", again.
+    move_into_pyside_package()
 
 # end of file