From 73a9168ad56559bb3fba0d93866b05a7fde812de Mon Sep 17 00:00:00 2001 From: Christian Tismer Date: Sat, 14 Jul 2018 15:10:56 +0200 Subject: Complete The Signature Introspection The signature module has been quite far developed. In the course of making things fit for the TypeErrors with the signature module, now also all signatures from all shiboken modules are queried. Instead of writing an extra signature existence test for shiboken, it made more sense to extend the existing init_platform.py by the shiboken modules. In fact, by this query a corner case was exploited that worked on Python 2 but assertion-crashed on Python 3. The mapping.py modules were also completed to support all new PySide2 modules. Special care had to be taken because the "shiboken2" module exists both as directory and as binary module. The fix was tricky, and I will add a task that replaces such workarounds by a better design. Task-number: PYSIDE-510 Change-Id: Ibf8e322d1905976a0044a702ea178b7f98629fb4 Reviewed-by: Christian Tismer --- ez_setup.py | 2 +- .../pyside2/PySide2/support/signature/mapping.py | 24 ++--- sources/pyside2/tests/QtGui/qmatrix_test.py | 3 +- sources/pyside2/tests/registry/existence_test.py | 14 ++- sources/pyside2/tests/registry/init_platform.py | 117 +++++++++++++++++---- sources/shiboken2/libshiboken/signature.cpp | 46 +++++--- .../support/signature/backport_inspect.py | 4 +- .../shibokenmodule/support/signature/mapping.py | 72 ++++++++++++- 8 files changed, 221 insertions(+), 61 deletions(-) diff --git a/ez_setup.py b/ez_setup.py index ddc630e7a..eb37479b9 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -224,7 +224,7 @@ def download_file_powershell(url, target): "[System.Net.CredentialCache]::DefaultCredentials; " "(new-object System.Net.WebClient).DownloadFile({}, {})".format( url, target)) - ) +# ) cmd = [ 'powershell', '-Command', diff --git a/sources/pyside2/PySide2/support/signature/mapping.py b/sources/pyside2/PySide2/support/signature/mapping.py index 1a769484d..c39821f05 100644 --- a/sources/pyside2/PySide2/support/signature/mapping.py +++ b/sources/pyside2/PySide2/support/signature/mapping.py @@ -56,7 +56,7 @@ from signature_loader.sbk_mapping import * Sbk_Reloader = Reloader class Reloader(Sbk_Reloader): - _uninitialized = Sbk_Reloader._uninitialized + PySide2.__all__ + _uninitialized = Sbk_Reloader._uninitialized + PySide2.__all__ + ["testbinding"] _prefixes = Sbk_Reloader._prefixes + ["PySide2."] def update(self): @@ -66,7 +66,6 @@ update_mapping = Reloader().update def init_QtCore(): - import PySide2.QtCore from PySide2.QtCore import Qt, QUrl, QDir from PySide2.QtCore import QRect, QSize, QPoint, QLocale, QByteArray from PySide2.QtCore import QMarginsF # 5.9 @@ -217,7 +216,6 @@ def init_QtCore(): def init_QtGui(): - import PySide2.QtGui from PySide2.QtGui import QPageLayout, QPageSize # 5.12 macOS type_map.update({ "QVector< QTextLayout.FormatRange >()": [], # do we need more structure? @@ -248,7 +246,6 @@ def init_QtGui(): def init_QtWidgets(): - import PySide2.QtWidgets from PySide2.QtWidgets import QWidget, QMessageBox, QStyleOption, QStyleHintReturn, QStyleOptionComplex from PySide2.QtWidgets import QGraphicsItem, QStyleOptionGraphicsItem # 5.9 GraphicsItemList = typing.List[QGraphicsItem] @@ -285,7 +282,6 @@ def init_QtWidgets(): def init_QtSql(): - import PySide2.QtSql from PySide2.QtSql import QSqlDatabase type_map.update({ "QLatin1String(defaultConnection)": QSqlDatabase.defaultConnection, @@ -295,7 +291,6 @@ def init_QtSql(): def init_QtNetwork(): - import PySide2.QtNetwork type_map.update({ "QMultiMap": MultiMap, "zero(unsigned short)": 0, @@ -306,7 +301,6 @@ def init_QtNetwork(): def init_QtXmlPatterns(): - import PySide2.QtXmlPatterns from PySide2.QtXmlPatterns import QXmlName type_map.update({ "QXmlName.PrefixCode": Missing("PySide2.QtXmlPatterns.QXmlName.PrefixCode"), @@ -315,7 +309,7 @@ def init_QtXmlPatterns(): return locals() -def init_QtMultimediaWidgets(): +def init_QtMultimedia(): import PySide2.QtMultimediaWidgets type_map.update({ "QGraphicsVideoItem": PySide2.QtMultimediaWidgets.QGraphicsVideoItem, @@ -325,7 +319,6 @@ def init_QtMultimediaWidgets(): def init_QtOpenGL(): - import PySide2.QtOpenGL type_map.update({ "GLuint": int, "GLenum": int, @@ -342,7 +335,6 @@ def init_QtOpenGL(): def init_QtQml(): - import PySide2.QtQml type_map.update({ "QJSValueList()": [], "PySide2.QtQml.bool volatile": bool, @@ -355,7 +347,6 @@ def init_QtQml(): def init_QtQuick(): - import PySide2.QtQuick type_map.update({ "PySide2.QtQuick.QSharedPointer": int, "PySide2.QtCore.uint": int, @@ -367,7 +358,6 @@ def init_QtQuick(): def init_QtScript(): - import PySide2.QtScript type_map.update({ "QScriptValueList()": [], }) @@ -375,7 +365,6 @@ def init_QtScript(): def init_QtTest(): - import PySide2.QtTest type_map.update({ "PySide2.QtTest.QTouchEventSequence": PySide2.QtTest.QTest.QTouchEventSequence, }) @@ -383,7 +372,6 @@ def init_QtTest(): # from 5.9 def init_QtWebEngineWidgets(): - import PySide2.QtWebEngineWidgets type_map.update({ "zero(PySide2.QtWebEngineWidgets.QWebEnginePage.FindFlags)": 0, }) @@ -391,7 +379,6 @@ def init_QtWebEngineWidgets(): # from 5.6, MSVC def init_QtWinExtras(): - import PySide2.QtWinExtras type_map.update({ "QList< QWinJumpListItem* >()": [], }) @@ -409,4 +396,11 @@ def init_QtDataVisualization(): }) return locals() + +def init_testbinding(): + type_map.update({ + "testbinding.PySideCPP2.TestObjectWithoutNamespace": testbinding.TestObjectWithoutNamespace, + }) + return locals() + # end of file diff --git a/sources/pyside2/tests/QtGui/qmatrix_test.py b/sources/pyside2/tests/QtGui/qmatrix_test.py index 7cfe9ea60..bc6a2b8ae 100644 --- a/sources/pyside2/tests/QtGui/qmatrix_test.py +++ b/sources/pyside2/tests/QtGui/qmatrix_test.py @@ -47,7 +47,8 @@ class QMatrixTest(unittest.TestCase): def testMatrixWithWrongType(self): matrix = QMatrix(11, 12, 21, 22, 100, 200) point = QPoint(3, 3) - self.assertRaises(TypeError, matrix.__mul__, point) + # This exception may move from a TypeError to a ValueError. + self.assertRaises((TypeError, ValueError), matrix.__mul__, point) def testMatrix2x2(self): matrix = QMatrix2x2([1.0, 2.0, 3.0, 4.0]) diff --git a/sources/pyside2/tests/registry/existence_test.py b/sources/pyside2/tests/registry/existence_test.py index 0d8014ad8..762a5888e 100644 --- a/sources/pyside2/tests/registry/existence_test.py +++ b/sources/pyside2/tests/registry/existence_test.py @@ -39,17 +39,23 @@ from __future__ import print_function, absolute_import +""" +existence_test.py + +A test that checks all function signatures if they still exist. +""" + import os import sys import unittest from textwrap import dedent from init_platform import (enum_all, generate_all, is_ci, - getEffectiveRefPath, getRefPath, qtVersion) + get_effective_refpath, get_refpath, qt_version) from util import isolate_warnings, check_warnings, suppress_warnings, warn from PySide2 import * -refPath = getRefPath() -effectiveRefPath = getEffectiveRefPath() +refPath = get_refpath() +effectiveRefPath = get_effective_refpath() effectiveRefPathRoot = os.path.splitext(effectiveRefPath)[0] pyc = effectiveRefPathRoot + ".pyc" if os.path.exists(pyc) and not os.path.exists(effectiveRefPath): @@ -132,7 +138,7 @@ class TestSignaturesExists(unittest.TestCase): tested_versions = (5, 6), (5, 9), (5, 11) #, (5, 12) # activate this, soon! -if not have_refmodule and is_ci and qtVersion()[:2] in tested_versions: +if not have_refmodule and is_ci and qt_version()[:2] in tested_versions: class TestFor_CI_Init(unittest.TestCase): """ This helper class generates the reference file for CI. diff --git a/sources/pyside2/tests/registry/init_platform.py b/sources/pyside2/tests/registry/init_platform.py index ded8ba81c..ca2b2cb68 100644 --- a/sources/pyside2/tests/registry/init_platform.py +++ b/sources/pyside2/tests/registry/init_platform.py @@ -40,21 +40,98 @@ from __future__ import print_function, absolute_import """ +init_platform.py + Existence registry +================== This is a registry for all existing function signatures. One file is generated with all signatures of a platform and version. + +The scope has been extended to generate all signatures from the +shiboken and pysidetest projects. """ import sys import os import re -import PySide2 from contextlib import contextmanager from textwrap import dedent +script_dir = os.path.normpath(os.path.join(__file__, *".. .. .. .. ..".split())) +history_dir = os.path.join(script_dir, 'build_history') + +# Find out if we have the build dir, already. Then use it. +look_for = os.path.join("pyside2", "tests", "pysidetest") +have_build_dir = [x for x in sys.path if x.endswith(look_for)] +if have_build_dir: + all_build_dir = os.path.normpath(os.path.join(have_build_dir[0], "..", "..", "..")) +elif os.path.exists(history_dir): + # Using the last build to find the build dir. + # Note: This is not reliable when building in parallel! + last_build = max(x for x in os.listdir(history_dir) if x.startswith("20")) + fpath = os.path.join(history_dir, last_build, "build_dir.txt") + if os.path.exists(fpath): + with open(fpath) as f: + all_build_dir = f.read().strip() +else: + print(dedent(""" + Can't find the build dir in the history. + Compile again and don't forget to specify "--build-tests". + """)) + sys.exit(1) + +if not os.path.exists(os.path.join(all_build_dir, look_for)): + print(dedent(""" + PySide has not been built with tests enabled. + Compile again and don't forget to specify "--build-tests". + """)) + sys.exit(1) + +pyside_build_dir = os.path.join(all_build_dir, "pyside2") +shiboken_build_dir = os.path.join(all_build_dir, "shiboken2") + +# now we compute all paths: +def set_ospaths(build_dir): + ps = os.pathsep + ospath_var = "PATH" if sys.platform == "win32" else "LD_LIBRARY_PATH" + old_val = os.environ.get(ospath_var, "") + lib_path = [os.path.join(build_dir, "pyside2", "libpyside"), + os.path.join(build_dir, "pyside2", "tests", "pysidetest"), + os.path.join(build_dir, "shiboken2", "tests", "libminimal"), + os.path.join(build_dir, "shiboken2", "tests", "libsample"), + os.path.join(build_dir, "shiboken2", "tests", "libother"), + os.path.join(build_dir, "shiboken2", "tests", "libsmart"), + os.path.join(build_dir, "shiboken2", "libshiboken")] + ospath = ps.join(lib_path + old_val.split(ps)) + os.environ[ospath_var] = ospath + +set_ospaths(all_build_dir) +sys.path[:0] = [os.path.join(shiboken_build_dir, "shibokenmodule"), + pyside_build_dir] + +import PySide2 + all_modules = list("PySide2." + x for x in PySide2.__all__) +# now we should be able to do all imports: +if not have_build_dir: + sys.path.insert(0, os.path.join(pyside_build_dir, "tests", "pysidetest")) +import testbinding +all_modules.append("testbinding") + +# Note: This is not the shiboken dir as usual, but the binary. +import shiboken2 as Shiboken +Shiboken.__name__ = "Shiboken" +sys.modules["Shiboken"] = sys.modules.pop("shiboken2") +all_modules.append("Shiboken") + +# 'sample' seems to be needed by 'other', so import it first. +for modname in "minimal sample other smart".split(): + sys.path.insert(0, os.path.join(shiboken_build_dir, "tests", modname + "binding")) + __import__(modname) + all_modules.append(modname) + from PySide2.QtCore import __version__ from PySide2.support.signature.lib.enum_sig import SimplifyingEnumerator @@ -79,36 +156,35 @@ else: # Make sure not to get .pyc in Python2. sourcepath = os.path.splitext(__file__)[0] + ".py" -def qtVersion(): +def qt_version(): return tuple(map(int, __version__.split("."))) -# Format a registry file name for version -def _registryFileName(version): +# Format a registry file name for version. +def _registry_filename(version): name = "exists_{}_{}_{}_{}{}.py".format(platform_name, version[0], version[1], version[2], "_ci" if is_ci else "") return os.path.join(os.path.dirname(__file__), name) -# Return the expected registry file name -def getRefPath(): - return _registryFileName(qtVersion()) +# Return the expected registry file name. +def get_refpath(): + return _registry_filename(qt_version()) # Return the registry file name, either that of the current -# version or fall back to a previous patch release -def getEffectiveRefPath(): - refpath = getRefPath() +# version or fall back to a previous patch release. +def get_effective_refpath(): + refpath = get_refpath() if os.path.exists(refpath): return refpath - version = qtVersion() - majorVersion = version[0] - minorVersion = version[1] - patchVersion = version[2] - while patchVersion >= 0: - file = _registryFileName((majorVersion, minorVersion, patchVersion)) + version = qt_version() + major, minor, patch = version[:3] + while patch >= 0: + file = _registry_filename((major, minor, patch)) if os.path.exists(file): return file - patchVersion = patchVersion - 1 + patch = patch - 1 return refpath + class Formatter(object): """ Formatter is formatting the signature listing of an enumerator. @@ -163,8 +239,9 @@ def enum_all(): ret.update(enu.module(mod_name)) return ret + def generate_all(): - refPath = getRefPath() + refPath = get_refpath() module = os.path.basename(os.path.splitext(refPath)[0]) with open(refPath, "w") as outfile, open(sourcepath) as f: fmt = Formatter(outfile) @@ -190,10 +267,12 @@ def generate_all(): enu.module(mod_name) fmt.print("# eof") + def __main__(): print("+++ generating {}. You should probably check this file in." - .format(getRefPath())) + .format(get_refpath())) generate_all() + if __name__ == "__main__": __main__() diff --git a/sources/shiboken2/libshiboken/signature.cpp b/sources/shiboken2/libshiboken/signature.cpp index 922f85906..564e5fcef 100644 --- a/sources/shiboken2/libshiboken/signature.cpp +++ b/sources/shiboken2/libshiboken/signature.cpp @@ -129,13 +129,16 @@ static PyObject * _get_class_of_cf(PyObject *ob_cf) { PyObject *selftype = PyCFunction_GET_SELF(ob_cf); - if (selftype == NULL) - selftype = PyDict_GetItem(pyside_globals->map_dict, (PyObject *)ob_cf); - if (selftype == NULL) { - if (!PyErr_Occurred()) - Py_RETURN_NONE; - return NULL; + if (selftype == nullptr) { + selftype = PyDict_GetItem(pyside_globals->map_dict, ob_cf); + if (selftype == nullptr) { + // This must be an overloaded function that we handled special. + Shiboken::AutoDecRef special(Py_BuildValue("(Os)", ob_cf, "overload")); + selftype = PyDict_GetItem(pyside_globals->map_dict, special); + } } + assert(selftype); + PyObject *typemod = (PyType_Check(selftype) || PyModule_Check(selftype)) ? selftype : (PyObject *)Py_TYPE(selftype); // do we support module functions? @@ -175,19 +178,26 @@ GetClassOfFunc(PyObject *ob) } static PyObject * -compute_name_key(PyObject *ob) +get_funcname(PyObject *ob) { - if (PyType_Check(ob)) - return GetClassKey(GetClassOfFunc(ob)); PyObject *func = ob; if (Py_TYPE(ob) == PepStaticMethod_TypePtr) func = PyObject_GetAttrString(ob, "__func__"); else Py_INCREF(func); - Shiboken::AutoDecRef func_name(PyObject_GetAttrString(func, "__name__")); + PyObject *func_name = PyObject_GetAttrString(func, "__name__"); Py_DECREF(func); - if (func_name.isNull()) + if (func_name == nullptr) Py_FatalError("unexpected name problem in compute_name_key"); + return func_name; +} + +static PyObject * +compute_name_key(PyObject *ob) +{ + if (PyType_Check(ob)) + return GetClassKey(ob); + Shiboken::AutoDecRef func_name(get_funcname(ob)); Shiboken::AutoDecRef type_key(GetClassKey(GetClassOfFunc(ob))); return Py_BuildValue("(OO)", type_key.object(), func_name.object()); } @@ -201,9 +211,11 @@ build_name_key_to_func(PyObject *obtype) if (meth == 0) return 0; + Shiboken::AutoDecRef type_key(GetClassKey(obtype)); for (; meth->ml_name != NULL; meth++) { Shiboken::AutoDecRef func(PyCFunction_NewEx(meth, obtype, NULL)); - Shiboken::AutoDecRef name_key(compute_name_key(func)); + Shiboken::AutoDecRef func_name(get_funcname(func)); + Shiboken::AutoDecRef name_key(Py_BuildValue("(OO)", type_key.object(), func_name.object())); if (func.isNull() || name_key.isNull() || PyDict_SetItem(pyside_globals->map_dict, name_key, func) < 0) return -1; @@ -224,7 +236,7 @@ name_key_to_func(PyObject *ob) Py_RETURN_NONE; PyObject *ret = PyDict_GetItem(pyside_globals->map_dict, name_key); - if (ret == NULL) { + if (ret == nullptr) { // do a lazy initialization Shiboken::AutoDecRef type_key(GetClassKey(GetClassOfFunc(ob))); PyObject *type = PyDict_GetItem(pyside_globals->map_dict, @@ -233,7 +245,7 @@ name_key_to_func(PyObject *ob) Py_RETURN_NONE; assert(PyType_Check(type)); if (build_name_key_to_func(type) < 0) - return NULL; + return nullptr; ret = PyDict_GetItem(pyside_globals->map_dict, name_key); } Py_XINCREF(ret); @@ -901,6 +913,12 @@ _build_func_to_type(PyObject *obtype) strcat(mangled_name, ".overload"); if (PyDict_SetItemString(dict, mangled_name, descr) < 0) return -1; + if (meth->ml_flags & METH_STATIC) { + // This is the special case where a static method is hidden. + Shiboken::AutoDecRef special(Py_BuildValue("(Os)", cfunc.object(), "overload")); + if (PyDict_SetItem(pyside_globals->map_dict, special, obtype) < 0) + return -1; + } if (PyDict_SetItemString(pyside_globals->map_dict, mangled_name, obtype) < 0) return -1; continue; diff --git a/sources/shiboken2/shibokenmodule/support/signature/backport_inspect.py b/sources/shiboken2/shibokenmodule/support/signature/backport_inspect.py index 6b97470e2..e890dcdcf 100644 --- a/sources/shiboken2/shibokenmodule/support/signature/backport_inspect.py +++ b/sources/shiboken2/shibokenmodule/support/signature/backport_inspect.py @@ -113,8 +113,8 @@ CO_NOFREE = 0x0040 # We use '__builtin__' and '__name__' instead. # It is further changed because we use a local copy of typing def formatannotation(annotation, base_module=None): - if getattr(annotation, '__module__', None) == 'support.signature.typing': - return repr(annotation).replace('support.signature.typing', 'typing') + if getattr(annotation, '__module__', None) == 'support.signature.typing27': + return repr(annotation).replace('support.signature.typing27', 'typing') if isinstance(annotation, type): if annotation.__module__ in ('__builtin__', base_module): return annotation.__name__ diff --git a/sources/shiboken2/shibokenmodule/support/signature/mapping.py b/sources/shiboken2/shibokenmodule/support/signature/mapping.py index 3e76cd94a..f638bc42b 100644 --- a/sources/shiboken2/shibokenmodule/support/signature/mapping.py +++ b/sources/shiboken2/shibokenmodule/support/signature/mapping.py @@ -88,6 +88,7 @@ WId = int GL_TEXTURE_2D = 0x0DE1 GL_RGBA = 0x1908 + class _NotCalled(str): """ Wrap some text with semantics @@ -104,7 +105,7 @@ class _NotCalled(str): real object is needed, the wrapper can simply be called. """ def __repr__(self): - suppress = "support.signature.typing." + suppress = "support.signature.typing27." text = self[len(suppress):] if self.startswith(suppress) else self return "{}({})".format(type(self).__name__, text) @@ -113,14 +114,20 @@ class _NotCalled(str): text = self if self.endswith(")") else self + "()" return eval(text, namespace) +USE_PEP563 = sys.version_info[:2] >= (3, 7) + + # Some types are abstract. They just show their name. class Virtual(_NotCalled): pass # Other types I simply could not find. class Missing(_NotCalled): - def __repr__(self): - return '{}("{}")'.format(type(self).__name__, self) + if not USE_PEP563: + # The string must be quoted, because the object does not exist. + def __repr__(self): + return '{}("{}")'.format(type(self).__name__, self) + class Invalid(_NotCalled): pass @@ -129,12 +136,20 @@ class Invalid(_NotCalled): class Default(_NotCalled): pass + class Instance(_NotCalled): pass class Reloader(object): - _uninitialized = ["sample"] + """ + Reloder class + + This is a singleton class which provides the update function for the + shiboken and PySide classes. + """ + ## Note: We needed to rename shiboken2 in order to avoid a name clash. + _uninitialized = "Shiboken minimal sample other smart".split() _prefixes = [""] def __init__(self): @@ -142,6 +157,14 @@ class Reloader(object): self.uninitialized = self._uninitialized def update(self, g=None): + """ + update is responsible to import all modules from shiboken and PySide + which are already in sys.modules. + The purpose is to follow all user imports without introducing new + ones. + This function is called by pyside_type_init to adapt imports + when the number of imported modules has changed. + """ if self.sys_module_count == len(sys.modules): return self.sys_module_count = len(sys.modules) @@ -160,6 +183,14 @@ class Reloader(object): self.uninitialized.remove(mod_name) proc_name = "init_" + mod_name if proc_name in g: + # Do the 'import {import_name}' first. + # 'top' is PySide2 when we do 'import PySide.QtCore' + # or Shiboken if we do 'import Shiboken'. + # Convince yourself that these two lines below have the same + # global effect as "import Shiboken" or "import PySide2.QtCore". + top = __import__(import_name) + g[top.__name__] = top + # Modules are in place, we can update the type_map. g.update(g[proc_name]()) @@ -167,8 +198,23 @@ update_mapping = Reloader().update type_map = {} +def init_Shiboken(): + type_map.update({ + "shiboken2.bool": bool, + "size_t": int, + "PyType": type, + }) + return locals() + + +def init_minimal(): + type_map.update({ + "MinBool": bool, + }) + return locals() + + def init_sample(): - import sample import datetime type_map.update({ "sample.int": int, @@ -207,4 +253,20 @@ def init_sample(): }) return locals() + +def init_other(): + import numbers + type_map.update({ + "other.Number": numbers.Number, + "other.ExtendsNoImplicitConversion": Missing("other.ExtendsNoImplicitConversion"), + }) + return locals() + + +def init_smart(): + type_map.update({ + "smart.SharedPtr": Missing("smart.SharedPtr"), # bad object "SharedPtr" + }) + return locals() + # end of file -- cgit v1.2.3