aboutsummaryrefslogtreecommitdiffstats
path: root/sources/shiboken2/shibokenmodule
diff options
context:
space:
mode:
authorChristian Tismer <tismer@stackless.com>2019-02-26 09:44:01 +0100
committerChristian Tismer <tismer@stackless.com>2019-03-20 17:33:42 +0000
commit723aea37263571865edc9e0299fedbe266a47223 (patch)
tree388072fd4100c51eabecf7269880b6bbf10b3b55 /sources/shiboken2/shibokenmodule
parente8786b9cdccb63aa590af50fe5fa5496c6b216c7 (diff)
Implement Embedding To Make Signatures Always Available
Situation.. PySide works fine with normal applications. But when installers are used to pack the application together, then the signature extension cannot be loaded. This is a problem that exists since the signature extension was written. But starting with PySide 5.12.1, the signature extension is very visible, because it is used to support the __doc__ attribute. There have beed successful attempts to solve the problem for PyInstaller and Py2App. But there are more packers available, and they all need a change both in PySide and in the packer. Solution.. To solve this problem once and for all, we embed the Python support files in the binary shiboken package. When the Python files are not normally accessible, they are unpacked from a ZIP file. Details.. - The embedded files shall only be used when the normal files are not available, - The signature extension should no longer be lazily loaded. When the application starts, all files should be present. - We drop support for shiboken2.support.signature and use a single, independen folder 'shibokensupport' (proposal). This avoids problems with multiple existence of the shiboken2 folder. PySide2.support.signature remains the official interface, but it's only an alias when PySide2 exists. - The embedding is used when the normal files cannot be loaded for some reason. It can be enforced by a sys variable "pyside_uses_embedding". - Testcase is included. - Tested with PyInstaller on macOS Fixes: PYSIDE-932 Fixes: PYSIDE-942 Change-Id: Iaa62dc0623503a2e0943eedd10d2b2484c65cc2d Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org> Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
Diffstat (limited to 'sources/shiboken2/shibokenmodule')
-rw-r--r--sources/shiboken2/shibokenmodule/CMakeLists.txt8
-rw-r--r--sources/shiboken2/shibokenmodule/files.dir/shibokensupport/backport_inspect.py (renamed from sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/backport_inspect.py)69
-rw-r--r--sources/shiboken2/shibokenmodule/files.dir/shibokensupport/fix-complaints.py (renamed from sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/fix-complaints.py)17
-rw-r--r--sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py4
-rw-r--r--sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py3
-rw-r--r--sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py3
-rw-r--r--sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py200
-rw-r--r--sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py6
-rw-r--r--sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json2
-rw-r--r--sources/shiboken2/shibokenmodule/files.dir/shibokensupport/typing27.py (renamed from sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/typing27.py)0
10 files changed, 137 insertions, 175 deletions
diff --git a/sources/shiboken2/shibokenmodule/CMakeLists.txt b/sources/shiboken2/shibokenmodule/CMakeLists.txt
index 612055b3..61c38c5d 100644
--- a/sources/shiboken2/shibokenmodule/CMakeLists.txt
+++ b/sources/shiboken2/shibokenmodule/CMakeLists.txt
@@ -71,10 +71,10 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/
"${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/lib/enum_sig.py" COPYONLY)
if (PYTHON_VERSION_MAJOR EQUAL 3)
else()
- configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/backport_inspect.py"
- "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/backport_inspect.py" COPYONLY)
- configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/signature/typing27.py"
- "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/signature/typing27.py" COPYONLY)
+ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/backport_inspect.py"
+ "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/backport_inspect.py" COPYONLY)
+ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/files.dir/shibokensupport/typing27.py"
+ "${CMAKE_CURRENT_BINARY_DIR}/files.dir/shibokensupport/typing27.py" COPYONLY)
endif()
install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/files.dir"
DESTINATION "${PYTHON_SITE_PACKAGES}/shiboken2")
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/backport_inspect.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/backport_inspect.py
index fbd82cc5..c690493b 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/backport_inspect.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/backport_inspect.py
@@ -111,10 +111,9 @@ CO_NOFREE = 0x0040
# This function was changed: 'builtins' and 'qualname' don't exist.
# We use '__builtin__' and '__name__' instead.
-# It is further changed because we use a local copy of typing
def formatannotation(annotation, base_module=None):
- if getattr(annotation, '__module__', None) == 'support.signature.typing27':
- return repr(annotation).replace('support.signature.typing27', 'typing')
+ if getattr(annotation, '__module__', None) == 'typing':
+ return repr(annotation).replace('typing.', '')
if isinstance(annotation, type):
if annotation.__module__ in ('__builtin__', base_module):
return annotation.__name__
@@ -378,7 +377,7 @@ class Parameter(object):
# Add annotation and default value
if self._annotation is not _empty:
- formatted = '{}:{}'.format(formatted,
+ formatted = '{}: {}'.format(formatted,
formatannotation(self._annotation))
if self._default is not _empty:
@@ -892,65 +891,3 @@ class Signature(object):
def signature(obj, follow_wrapped=True):
"""Get a signature object for the passed callable."""
return Signature.from_callable(obj, follow_wrapped=follow_wrapped)
-
-
-def _main():
- """ Logic for inspecting an object given at command line """
- import argparse
- import importlib
-
- parser = argparse.ArgumentParser()
- parser.add_argument(
- 'object',
- help="The object to be analysed. "
- "It supports the 'module:qualname' syntax")
- parser.add_argument(
- '-d', '--details', action='store_true',
- help='Display info about the module rather than its source code')
-
- args = parser.parse_args()
-
- target = args.object
- mod_name, has_attrs, attrs = target.partition(":")
- try:
- obj = module = importlib.import_module(mod_name)
- except Exception as exc:
- msg = "Failed to import {} ({}: {})".format(mod_name,
- type(exc).__name__,
- exc)
- print(msg, file=sys.stderr)
- exit(2)
-
- if has_attrs:
- parts = attrs.split(".")
- obj = module
- for part in parts:
- obj = getattr(obj, part)
-
- if module.__name__ in sys.builtin_module_names:
- print("Can't get info for builtin modules.", file=sys.stderr)
- exit(1)
-
- if args.details:
- print('Target: {}'.format(target))
- print('Origin: {}'.format(getsourcefile(module)))
- print('Cached: {}'.format(module.__cached__))
- if obj is module:
- print('Loader: {}'.format(repr(module.__loader__)))
- if hasattr(module, '__path__'):
- print('Submodule search path: {}'.format(module.__path__))
- else:
- try:
- __, lineno = findsource(obj)
- except Exception:
- pass
- else:
- print('Line: {}'.format(lineno))
-
- print('\n')
- else:
- print(getsource(obj))
-
-
-if __name__ == "__main__":
- _main()
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/fix-complaints.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/fix-complaints.py
index cdd84f9b..3818ad60 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/fix-complaints.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/fix-complaints.py
@@ -1,3 +1,6 @@
+# This Python file uses the following encoding: utf-8
+# It has been edited by fix-complaints.py .
+
#############################################################################
##
## Copyright (C) 2019 The Qt Company Ltd.
@@ -42,14 +45,15 @@ from __future__ import print_function, absolute_import
"""
fix-complaints.py
-This module fixes the buildbot messages of external python modules.
+This module fixes the buildbot messages of external python files.
Run it once after copying a new version. It is idem-potent, unless
you are changing messages (what I did, of course :-) .
"""
import os
+import glob
-patched_modules = "backport_inspect typing27"
+patched_file_patterns = "backport_inspect.py typing27.py python_minilib_*.py"
offending_words = {
"behavio""ur": "behavior",
@@ -79,9 +83,12 @@ def patch_file(fname):
f.write("".join(lines))
def doit():
- dir = os.path.dirname(__file__)
- for name in patched_modules.split():
- fname = os.path.join(dir, name + ".py")
+ dirname = os.path.dirname(__file__)
+ patched_files = []
+ for name in patched_file_patterns.split():
+ pattern = os.path.join(dirname, name)
+ patched_files += glob.glob(pattern)
+ for fname in patched_files:
print("Working on", fname)
patch_file(fname)
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py
index 9447e9c2..ee541d0e 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/__init__.py
@@ -39,6 +39,4 @@
from __future__ import print_function, absolute_import
-# from shibokensupport.signature import get_signature, inspect, typing
-# This gives a problem with Python 2. We do it in the loader, instead.
-__all__ = "get_signature inspect typing layout mapping lib".split()
+__all__ = "get_signature layout mapping lib".split()
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py
index 1443d792..cb148830 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/errorhandler.py
@@ -56,7 +56,8 @@ enough to produce a useful ValueError.
This matter will be improved in a later version.
"""
-from shibokensupport.signature import get_signature, inspect
+from shibokensupport.signature import inspect
+from shibokensupport.signature import get_signature
from shibokensupport.signature.mapping import update_mapping, namespace
from textwrap import dedent
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
index 90affbcf..e6f6dc37 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py
@@ -50,7 +50,8 @@ by producing a lot of clarity.
"""
import sys
-from shibokensupport.signature import get_signature, inspect
+from shibokensupport.signature import inspect
+from shibokensupport.signature import get_signature
class ExactEnumerator(object):
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
index 3a8d614d..a564aedb 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/loader.py
@@ -1,3 +1,6 @@
+# This Python file uses the following encoding: utf-8
+# It has been edited by fix-complaints.py .
+
#############################################################################
##
## Copyright (C) 2019 The Qt Company Ltd.
@@ -42,33 +45,26 @@ from __future__ import print_function, absolute_import
"""
loader.py
-The loader has to lazy-load the signature module and also provides a few
-Python modules to support Python 2.7 .
+The loader has to load the signature module completely at startup,
+to make sure that the functions are available when needed.
+This is meanwhile necessary to make the '__doc__' attribute work correctly.
+
+It does not mean that everything is initialized in advance. Only the modules
+are loaded completely after 'import PySide2'.
+
+This version uses both a normal directory, but has also an embedded ZIP file
+as a fallback solution. The ZIP file is generated by 'embedding_generator.py'
+and embedded into 'signature.cpp' as "embed/signature.inc".
-This version uses both a normal directory, but has also an embedded zip file
-as a fallback solution.
+Meanwhile, the ZIP file grew so much, that MSVC had problems
+with it's 64k string limit, so we had to break the string up.
+See 'zipped_string_sequence' in signature.cpp.
"""
import sys
import os
import traceback
import types
-from contextlib import contextmanager
-
-"""
-A note on the import problem (solved):
-
-During the tests, the shiboken build structure has the layout
-
- shiboken2/shibokenmodule/shiboken2.abi3.so
-
-and the name "shiboken2" in sys.modules points directly to the binary
-file, hiding the outer shiboken2 module.
-
-To fix that, we temporarily remove the binary from sys.path,
-do the needed imports and then restore the binary.
-This action was put into a context manager for readability.
-"""
# On Python 2, we only have ImportError, which is way too coarse.
# When problems occour, please use Python 3, because it has the finer
@@ -79,31 +75,6 @@ try:
except NameError:
ModuleNotFoundError = ImportError
-@contextmanager
-def ensure_import_shibokensupport():
- # Make sure that we always have the shibokensupport containing package first.
- # Also remove any prior loaded module of this name, just in case.
- sbk_support_dir = os.path.abspath(os.path.join(__file__, "..", "..", ".."))
- assert os.path.basename(sbk_support_dir) == "files.dir"
- sys.path.insert(0, sbk_support_dir)
-
- sbk = "shibokensupport"
- if sbk in sys.modules:
- del sys.modules[sbk]
- for key in list(key for key in sys.modules if key.startswith(sbk + ".")):
- del sys.modules[key]
- try:
- import shibokensupport
- yield
- except Exception as e:
- print("Problem importing shibokensupport:")
- print(e)
- traceback.print_exc()
- sys.stdout.flush()
- sys.exit(-1)
- sys.path.remove(sbk_support_dir)
-
-
# patching inspect's formatting to keep the word "typing":
def formatannotation(annotation, base_module=None):
# if getattr(annotation, '__module__', None) == 'typing':
@@ -143,53 +114,102 @@ def seterror_argument(args, func_name):
def make_helptext(func):
return errorhandler.make_helptext(func)
-with ensure_import_shibokensupport():
- import signature_loader
- import shibokensupport.signature
- shibokensupport.signature.get_signature = signature_loader.get_signature
- del signature_loader # protect this dir, too?
+import signature_bootstrap
+from shibokensupport import signature
+signature.get_signature = signature_bootstrap.get_signature
+del signature_bootstrap
+
- if sys.version_info >= (3,):
- import typing
- import inspect
- inspect.formatannotation = formatannotation
+def _get_modname(mod):
+ return mod.__spec__.name if getattr(mod, "__spec__", None) else mod.__name__
+
+def _set_modname(mod, name):
+ if getattr(mod, "__spec__", None):
+ mod.__spec__.name = name
else:
- import inspect
- namespace = inspect.__dict__
- from shibokensupport.signature import typing27 as typing
- typing.__name__ = "typing"
- # Fix the module names in typing if possible. This is important since
- # the typing names should be I/O compatible, so that typing.Dict
- # shows itself as "typing.Dict".
- for name, obj in typing.__dict__.items():
- if hasattr(obj, "__module__"):
- try:
- obj.__module__ = "typing"
- except (TypeError, AttributeError):
- pass
- from shibokensupport.signature import backport_inspect as inspect
- _doc = inspect.__doc__
- inspect.__dict__.update(namespace)
- inspect.__doc__ += _doc
- # force inspect to find all attributes. See "heuristic" in pydoc.py!
- inspect.__all__ = list(x for x in dir(inspect) if not x.startswith("_"))
- typing.TypeVar.__repr__ = _typevar__repr__
-
- def put_into_package(module, package):
- # take the last component of the module name
- name = module.__name__.rsplit(".", 1)[-1]
- # allow access as {package}.typing
+ mod.__name__ = name
+
+
+def put_into_package(package, module, override=None):
+ # take the last component of the module name
+ name = (override if override else _get_modname(module)).rsplit(".", 1)[-1]
+ # allow access as {package}.typing
+ if package:
setattr(package, name, module)
- # put into sys.modules as a package to allow all import options
- fullname = "{}.{}".format(package.__name__, name)
- sys.modules[fullname] = module
-
- put_into_package(typing, shibokensupport.signature)
- put_into_package(inspect, shibokensupport.signature)
- from shibokensupport.signature import mapping
- from shibokensupport.signature import errorhandler
- from shibokensupport.signature import layout
- from shibokensupport.signature.lib import enum_sig
- from shibokensupport.signature.parser import pyside_type_init
+ # put into sys.modules as a package to allow all import options
+ fullname = "{}.{}".format(_get_modname(package), name) if package else name
+ _set_modname(module, fullname)
+ # publish new dotted name in sys.modules
+ sys.modules[fullname] = module
+
+
+# Debug: used to inspect what each step loads
+def list_modules(message):
+ ext_modules = {key:value for (key, value) in sys.modules.items()
+ if hasattr(value, "__file__")}
+ print("SYS.MODULES", message, len(sys.modules), len(ext_modules))
+ for (name, module) in sorted(ext_modules.items()):
+ print(" {:23}".format(name), repr(module)[:70])
+
+
+if sys.version_info >= (3,):
+ import typing
+ import inspect
+ inspect.formatannotation = formatannotation
+else:
+ from shibokensupport import typing27 as typing
+ import inspect
+ namespace = inspect.__dict__
+ from shibokensupport import backport_inspect as inspect
+ _doc = inspect.__doc__
+ inspect.__dict__.update(namespace)
+ inspect.__doc__ += _doc
+ # force inspect to find all attributes. See "heuristic" in pydoc.py!
+ inspect.__all__ = list(x for x in dir(inspect) if not x.startswith("_"))
+typing.TypeVar.__repr__ = _typevar__repr__
+
+# Fix the module names in typing if possible. This is important since
+# the typing names should be I/O compatible, so that typing.Dict
+# shows itself as "typing.Dict".
+for name, obj in typing.__dict__.items():
+ if hasattr(obj, "__module__"):
+ try:
+ obj.__module__ = "typing"
+ except (TypeError, AttributeError):
+ pass
+
+import shibokensupport
+put_into_package(shibokensupport.signature, typing, "typing")
+put_into_package(shibokensupport.signature, inspect, "inspect")
+
+
+def move_into_pyside_package():
+ import PySide2
+ try:
+ import PySide2.support
+ except ModuleNotFoundError:
+ PySide2.support = types.ModuleType("PySide2.support")
+ put_into_package(PySide2.support, signature)
+ put_into_package(PySide2.support.signature, mapping)
+ put_into_package(PySide2.support.signature, errorhandler)
+ put_into_package(PySide2.support.signature, layout)
+ put_into_package(PySide2.support.signature, lib)
+ put_into_package(PySide2.support.signature, parser)
+ put_into_package(PySide2.support.signature.lib, enum_sig)
+
+ put_into_package(PySide2.support.signature, typing)
+ put_into_package(PySide2.support.signature, inspect)
+
+from shibokensupport.signature import mapping
+from shibokensupport.signature import errorhandler
+from shibokensupport.signature import layout
+from shibokensupport.signature import lib
+from shibokensupport.signature import parser
+from shibokensupport.signature.lib import enum_sig
+from shibokensupport.signature.parser import pyside_type_init
+
+if "PySide2" in sys.modules:
+ # We publish everything under "PySide2.support.signature", again.
+ move_into_pyside_package()
# end of file
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
index 470dd676..9a8ee4c4 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/mapping.py
@@ -52,7 +52,6 @@ in sys.modules. This minimizes the loading overhead.
import sys
import struct
import os
-import pkgutil
from shibokensupport.signature import typing
from shibokensupport.signature.typing import TypeVar, Generic
@@ -168,9 +167,7 @@ class _NotCalled(str):
real object is needed, the wrapper can simply be called.
"""
def __repr__(self):
- suppress = "support.signature.typing27."
- text = self[len(suppress):] if self.startswith(suppress) else self
- return "{}({})".format(type(self).__name__, text)
+ return "{}({})".format(type(self).__name__, self)
def __call__(self):
from shibokensupport.signature.mapping import __dict__ as namespace
@@ -346,6 +343,7 @@ def init_other():
def init_smart():
type_map.update({
"smart.SharedPtr": Missing("smart.SharedPtr"), # bad object "SharedPtr<Obj >"
+ "smart.Smart.Integer2": int,
})
return locals()
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json
index 491ae805..fbe4c51a 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/qt_attribution.json
@@ -3,7 +3,7 @@
"Name": "Python",
"QDocModule": "QtForPython",
"QtUsage": "Used for Qt for Python in the signature extension.",
- "Description": "Qt for Python is an add-on for Python. The signature packages of PySide uses certain copied and adapted source files (backport_inspect.py, typing27.py). See the folder sources/pyside2/PySide2/support/signature .",
+ "Description": "Qt for Python is an add-on for Python. The signature packages of PySide uses certain copied and adapted source files (backport_inspect.py, typing27.py). See the folder sources/shiboken2/files.dir/shibokensupport/signature .",
"Homepage": "http://www.python.org/",
"Version": "3.7.0",
"LicenseId": "Python-2.0",
diff --git a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/typing27.py b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/typing27.py
index dba8f8c7..dba8f8c7 100644
--- a/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/signature/typing27.py
+++ b/sources/shiboken2/shibokenmodule/files.dir/shibokensupport/typing27.py