From 66615a89ef66c39d62a502df3eb1ff629aa39154 Mon Sep 17 00:00:00 2001 From: Christian Tismer Date: Sat, 14 Jul 2018 15:10:56 +0200 Subject: Prepare the Signature Module For More Applications This is the preparation for a number of planned applications and extensions using the signature module. This general overhaul contains: - Extraction of signature enumerations into enum_sigs.py, - a list of current keyword errors in arguments which are unsolved in shiboken, but temporarily fixed in parser.py (too many for XML), - fix spurious duplications in multiple signatures - corrections for keyword errors in function names which cannot be fixed by Python (quite few), - fixing "..." arguments into "*args", - supporting the "slot wrapper" type. This is necessary for methods like "__add__", "__mul__" etc. - Create an extra function "get_signature" that has a parameter to modify the appearance, i.e. without self, without returntype, etc. Task-number: PYSIDE-510 Change-Id: If16f7bf02c6e7cbbdc970058bb630ea4db2b854a Reviewed-by: Qt CI Bot Reviewed-by: Alexandru Croitor --- sources/shiboken2/libshiboken/pep384impl.cpp | 27 ++- sources/shiboken2/libshiboken/pep384impl.h | 8 +- sources/shiboken2/libshiboken/signature.cpp | 291 +++++++++++++++++++-------- sources/shiboken2/libshiboken/signature.h | 6 +- sources/shiboken2/libshiboken/typespec.cpp | 1 + 5 files changed, 246 insertions(+), 87 deletions(-) (limited to 'sources/shiboken2') diff --git a/sources/shiboken2/libshiboken/pep384impl.cpp b/sources/shiboken2/libshiboken/pep384impl.cpp index 4481f1cdd..18094c5c0 100644 --- a/sources/shiboken2/libshiboken/pep384impl.cpp +++ b/sources/shiboken2/libshiboken/pep384impl.cpp @@ -398,7 +398,10 @@ PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) return ret; } +#endif // Py_LIMITED_API + // This is only a simple local helper that returns a computed variable. +// Used also in Python 2. static PyObject * PepRun_GetResult(const char *command, const char *resvar) { @@ -415,6 +418,8 @@ PepRun_GetResult(const char *command, const char *resvar) return res; } +#ifdef Py_LIMITED_API + /***************************************************************************** * * Support for classobject.h @@ -499,13 +504,26 @@ PyTypeObject *PepStaticMethod_TypePtr = NULL; static PyTypeObject *getStaticMethodType(void) { + // this works for Python 3, only + // "StaticMethodType = type(str.__dict__['maketrans'])\n"; static const char prog[] = - "StaticMethodType = type(str.__dict__['maketrans'])\n"; - return (PyTypeObject *) PepRun_GetResult(prog, "StaticMethodType"); + "from xxsubtype import spamlist\n" + "StaticMethod_Type = type(spamlist.__dict__['staticmeth'])\n"; + return (PyTypeObject *) PepRun_GetResult(prog, "StaticMethod_Type"); } - #endif // Py_LIMITED_API +#if PY_VERSION_HEX < 0x03000000 +PyTypeObject *PepMethodDescr_TypePtr = NULL; + +static PyTypeObject *getMethodDescrType(void) +{ + static const char prog[] = + "MethodDescr_Type = type(str.split)\n"; + return (PyTypeObject *) PepRun_GetResult(prog, "MethodDescr_Type"); +} +#endif + /***************************************************************************** * * Common newly needed functions @@ -630,6 +648,9 @@ Pep384_Init() PepFunction_TypePtr = getFunctionType(); PepStaticMethod_TypePtr = getStaticMethodType(); #endif +#if PY_VERSION_HEX < 0x03000000 + PepMethodDescr_TypePtr = getMethodDescrType(); +#endif } } // extern "C" diff --git a/sources/shiboken2/libshiboken/pep384impl.h b/sources/shiboken2/libshiboken/pep384impl.h index b566a6218..6649fa95e 100644 --- a/sources/shiboken2/libshiboken/pep384impl.h +++ b/sources/shiboken2/libshiboken/pep384impl.h @@ -286,7 +286,7 @@ LIBSHIBOKEN_API PyObject *PyRun_String(const char *, int, PyObject *, PyObject * // But this is no problem as we check it's validity for every version. #define PYTHON_BUFFER_VERSION_COMPATIBLE (PY_VERSION_HEX >= 0x03030000 && \ - PY_VERSION_HEX < 0X0307FFFF) + PY_VERSION_HEX < 0x0307FFFF) #if !PYTHON_BUFFER_VERSION_COMPATIBLE # error Please check the buffer compatibility for this python version! #endif @@ -470,6 +470,12 @@ extern LIBSHIBOKEN_API PyTypeObject *PepStaticMethod_TypePtr; #else #define PepStaticMethod_TypePtr &PyStaticMethod_Type #endif +// Although not PEP specific, we resolve this similar issue, here: +#if PY_VERSION_HEX < 0x03000000 +extern LIBSHIBOKEN_API PyTypeObject *PepMethodDescr_TypePtr; +#else +#define PepMethodDescr_TypePtr &PyMethodDescr_Type +#endif /***************************************************************************** * diff --git a/sources/shiboken2/libshiboken/signature.cpp b/sources/shiboken2/libshiboken/signature.cpp index f0bb8e609..24d60acc4 100644 --- a/sources/shiboken2/libshiboken/signature.cpp +++ b/sources/shiboken2/libshiboken/signature.cpp @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2017 The Qt Company Ltd. +** Copyright (C) 2018 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of Qt for Python. @@ -77,18 +77,23 @@ typedef struct safe_globals_struc { static safe_globals pyside_globals = 0; -static PyObject *GetSignature_Function(PyCFunctionObject *); -static PyObject *GetSignature_TypeMod(PyObject *); +static PyObject *GetSignature_Function(PyCFunctionObject *, const char *); +static PyObject *GetSignature_TypeMod(PyObject *, const char *); +static PyObject *GetSignature_Wrapper(PyObject *, const char *); +static PyObject *get_signature(PyObject *self, PyObject *args); static PyObject *PySide_BuildSignatureProps(PyObject *class_mod); +static void init_module_1(void); +static void init_module_2(void); + const char helper_module_name[] = "signature_loader"; const char bootstrap_name[] = "bootstrap"; const char arg_name[] = "pyside_arg_dict"; const char func_name[] = "pyside_type_init"; static PyObject * -CreateSignature(PyObject *props, const char *sig_kind) +CreateSignature(PyObject *props, PyObject *key) { /* * Here is the new function to create all signatures. It simply calls @@ -97,22 +102,22 @@ CreateSignature(PyObject *props, const char *sig_kind) * to support '_signature_is_functionlike()'. */ return PyObject_CallFunction(pyside_globals->createsig_func, - (char *)"(Os)", props, sig_kind); + (char *)"(OO)", props, key); } static PyObject * -pyside_cf_get___signature__(PyObject *func) +pyside_cf_get___signature__(PyObject *func, const char *modifier) { - return GetSignature_Function((PyCFunctionObject *)func); + return GetSignature_Function((PyCFunctionObject *)func, modifier); } static PyObject * -pyside_sm_get___signature__(PyObject *sm) +pyside_sm_get___signature__(PyObject *sm, const char *modifier) { PyObject *func, *ret; func = PyObject_GetAttrString(sm, "__func__"); - ret = GetSignature_Function((PyCFunctionObject *)func); + ret = GetSignature_Function((PyCFunctionObject *)func, modifier); Py_XDECREF(func); return ret; } @@ -192,7 +197,7 @@ qualname_to_func(PyObject *ob) #endif static PyObject * -pyside_md_get___signature__(PyObject *ob) +pyside_md_get___signature__(PyObject *ob, const char *modifier) { PyObject *func; PyObject *result; @@ -218,21 +223,31 @@ pyside_md_get___signature__(PyObject *ob) return Py_None; if (func == NULL) Py_FatalError("missing mapping in MethodDescriptor"); - result = pyside_cf_get___signature__(func); + result = pyside_cf_get___signature__(func, modifier); Py_DECREF(func); return result; } static PyObject * -pyside_tp_get___signature__(PyObject *typemod) +pyside_wd_get___signature__(PyObject *ob, const char *modifier) +{ + return GetSignature_Wrapper(ob, modifier); +} + +static PyObject * +pyside_tp_get___signature__(PyObject *typemod, const char *modifier) { - return GetSignature_TypeMod(typemod); + return GetSignature_TypeMod(typemod, modifier); } +// forward +static PyObject * +GetSignature_Cached(PyObject *props, const char *sig_kind, const char *modifier); + static PyObject * -GetSignature_Function(PyCFunctionObject *func) +GetSignature_Function(PyCFunctionObject *func, const char *modifier) { - PyObject *typemod, *type_name, *dict, *props, *value, *selftype; + PyObject *typemod, *type_name, *dict, *props, *selftype; PyObject *func_name = PyObject_GetAttrString((PyObject *)func, "__name__"); const char *sig_kind; int flags; @@ -241,12 +256,8 @@ GetSignature_Function(PyCFunctionObject *func) if (selftype == NULL) selftype = PyDict_GetItem(pyside_globals->map_dict, (PyObject *)func); if (selftype == NULL) { - if (!PyErr_Occurred()) { - PyErr_Format(PyExc_SystemError, - "the signature for \"%s\" should exist", - PepCFunction_GET_NAMESTR(func) - ); - } + if (!PyErr_Occurred()) + Py_RETURN_NONE; return NULL; } if ((PyType_Check(selftype) || PyModule_Check(selftype))) @@ -279,24 +290,46 @@ GetSignature_Function(PyCFunctionObject *func) sig_kind = "staticmethod"; else sig_kind = "method"; - value = PyDict_GetItemString(props, sig_kind); - if (value == NULL) { - // we need to compute a signature object - value = CreateSignature(props, sig_kind); - if (value != NULL) { - if (PyDict_SetItemString(props, sig_kind, value) < 0) - return NULL; - } - else + return GetSignature_Cached(props, sig_kind, modifier); +} + +static PyObject * +GetSignature_Wrapper(PyObject *ob, const char *modifier) +{ + PyObject *dict, *props; + PyObject *func_name = PyObject_GetAttrString(ob, "__name__"); + PyObject *objclass = PyObject_GetAttrString(ob, "__objclass__"); + PyObject *class_name = PyObject_GetAttrString(objclass, "__name__"); + const char *sig_kind; + + if (func_name == nullptr || objclass == nullptr || class_name == nullptr) + return nullptr; + dict = PyDict_GetItem(pyside_globals->arg_dict, class_name); + if (dict == NULL) + Py_RETURN_NONE; + if (PyTuple_Check(dict)) { + /* + * We do the initialization lazily. + * This has also the advantage that we can freely import PySide. + */ + dict = PySide_BuildSignatureProps(objclass); + if (dict == NULL) Py_RETURN_NONE; } - return Py_INCREF(value), value; + props = PyDict_GetItem(dict, func_name); + Py_DECREF(func_name); + Py_DECREF(objclass); + Py_DECREF(class_name); + if (props == NULL) + Py_RETURN_NONE; + sig_kind = "method"; + return GetSignature_Cached(props, sig_kind, modifier); } static PyObject * -GetSignature_TypeMod(PyObject *ob) +GetSignature_TypeMod(PyObject *ob, const char *modifier) { - PyObject *ob_name, *dict, *props, *value; + PyObject *ob_name, *dict, *props; const char *sig_kind; ob_name = PyObject_GetAttrString(ob, "__name__"); @@ -314,37 +347,62 @@ GetSignature_TypeMod(PyObject *ob) if (props == NULL) Py_RETURN_NONE; sig_kind = "method"; - value = PyDict_GetItemString(props, sig_kind); - if (value == NULL) { + return GetSignature_Cached(props, sig_kind, modifier); +} + +static PyObject * +GetSignature_Cached(PyObject *props, const char *sig_kind, const char *modifier) +{ + PyObject *key, *value; + + if (modifier == nullptr) + key = Py_BuildValue("s", sig_kind); + else + key = Py_BuildValue("(ss)", sig_kind, modifier); + if (key == nullptr) + return nullptr; + value = PyDict_GetItem(props, key); + if (value == nullptr) { // we need to compute a signature object - value = CreateSignature(props, sig_kind); - if (value != NULL) { - if (PyDict_SetItemString(props, sig_kind, value) < 0) - return NULL; + value = CreateSignature(props, key); + if (value != nullptr) { + if (PyDict_SetItem(props, key, value) < 0) { + // this is an error + Py_DECREF(key); + return nullptr; + } } - else + else { + // key not found + Py_DECREF(key); Py_RETURN_NONE; + } } return Py_INCREF(value), value; } - static const char PySide_PythonCode[] = - "from __future__ import print_function, absolute_import\n" - "import sys, os, traceback\n" - - "pyside_package_dir = os.environ.get('PYSIDE_PACKAGE_DIR', '.')\n" - "__file__ = os.path.join(pyside_package_dir, 'support', 'signature', 'loader.py')\n" - - "def bootstrap():\n" - " try:\n" - " with open(__file__) as _f:\n" - " exec(compile(_f.read(), __file__, 'exec'))\n" - " except Exception as e:\n" - " print('Exception:', e)\n" - " traceback.print_exc(file=sys.stdout)\n" - " globals().update(locals())\n" - ; + "from __future__ import print_function, absolute_import\n" R"~(if True: + + import sys, os, traceback + + pyside_package_dir = os.environ.get('PYSIDE_PACKAGE_DIR') + if pyside_package_dir is None: + # This happens in shiboken running ctest. + from distutils.sysconfig import get_python_lib + pyside_package_dir = os.path.join(get_python_lib(), 'PySide2') + __file__ = os.path.join(pyside_package_dir, 'support', 'signature', 'loader.py') + + def bootstrap(): + try: + with open(__file__) as _f: + exec(compile(_f.read(), __file__, 'exec')) + except Exception as e: + print('Exception:', e) + traceback.print_exc(file=sys.stdout) + globals().update(locals()) + + )~"; static safe_globals_struc * init_phase_1(void) @@ -387,9 +445,10 @@ error: } static int -init_phase_2(safe_globals_struc *p) +init_phase_2(safe_globals_struc *p, PyMethodDef *methods) { - PyObject *bootstrap_func; + PyObject *bootstrap_func, *v = nullptr; + PyMethodDef *ml; bootstrap_func = PyObject_GetAttrString(p->helper_module, bootstrap_name); if (bootstrap_func == NULL) @@ -403,9 +462,22 @@ init_phase_2(safe_globals_struc *p) p->createsig_func = PyObject_GetAttrString(p->helper_module, "create_signature"); if (p->createsig_func == NULL) goto error; + + // The single function to be called, but maybe more to come. + for (ml = methods; ml->ml_name != NULL; ml++) { + v = PyCFunction_NewEx(ml, nullptr, nullptr); + if (v == nullptr) { + goto error; + } + if (PyObject_SetAttrString(p->helper_module, ml->ml_name, v) != 0) { + goto error; + } + Py_DECREF(v); + } return 0; error: + Py_XDECREF(v); PyErr_SetString(PyExc_SystemError, "could not initialize part 2"); return -1; } @@ -413,8 +485,11 @@ error: static int add_more_getsets(PyTypeObject *type, PyGetSetDef *gsp) { - PyObject *dict = type->tp_dict; + PyObject *dict; + assert(PyType_Check(type)); + PyType_Ready(type); + dict = type->tp_dict; for (; gsp->name != NULL; gsp++) { PyObject *descr; if (PyDict_GetItemString(dict, gsp->name)) @@ -463,6 +538,45 @@ static PyGetSetDef new_PyType_getsets[] = { {0} }; +static PyGetSetDef new_PyWrapperDescr_getsets[] = { + {(char *) "__signature__", (getter)pyside_wd_get___signature__}, + {0} +}; + +//////////////////////////////////////////////////////////////////////////// +// +// get_signature -- providing a superior interface +// +// Additionally to the interface via __signature__, we also provide +// a general function, which allows for different signature layouts. +// The "modifier" argument is a string that is passed in from loader.py . +// Configuration what the modifiers mean is completely in Python. +// + +static PyObject * +get_signature(PyObject *self, PyObject *args) +{ + PyObject *ob; + const char *modifier = nullptr; + + init_module_1(); + init_module_2(); + + if (!PyArg_ParseTuple(args, "O|s", &ob, &modifier)) + return NULL; + if (Py_TYPE(ob) == &PyCFunction_Type) + return pyside_cf_get___signature__(ob, modifier); + if (Py_TYPE(ob) == PepStaticMethod_TypePtr) + return pyside_sm_get___signature__(ob, modifier); + if (Py_TYPE(ob) == PepMethodDescr_TypePtr) + return pyside_md_get___signature__(ob, modifier); + if (PyType_Check(ob)) + return pyside_tp_get___signature__(ob, modifier); + if (Py_TYPE(ob) == &PyWrapperDescr_Type) + return pyside_wd_get___signature__(ob, modifier); + Py_RETURN_NONE; +} + //////////////////////////////////////////////////////////////////////////// // // This special Type_Ready does certain initializations earlier with @@ -497,23 +611,23 @@ void handler(int sig) { static int PySideType_Ready(PyTypeObject *type) { - PyObject *md; + PyObject *md, *wd; static int init_done = 0; if (!init_done) { - // Python2 does not expose certain types. We look them up: - // PyMethodDescr_Type 'type(str.__dict__["split"])' - // PyClassMethodDescr_Type. 'type(dict.__dict__["fromkeys"])' - // The latter is not needed until we use class methods in PySide. - md = PyObject_GetAttrString((PyObject *)&PyString_Type, "split"); - if (md == NULL + md = PyObject_GetAttrString((PyObject *)&PyString_Type, "split"); // method-descriptor + wd = PyObject_GetAttrString((PyObject *)Py_TYPE(Py_True), "__add__"); // wrapper-descriptor + if (md == nullptr || wd == nullptr || PyType_Ready(Py_TYPE(md)) < 0 - || add_more_getsets(Py_TYPE(md), new_PyMethodDescr_getsets) < 0 + || add_more_getsets(PepMethodDescr_TypePtr, new_PyMethodDescr_getsets) < 0 || add_more_getsets(&PyCFunction_Type, new_PyCFunction_getsets) < 0 || add_more_getsets(PepStaticMethod_TypePtr, new_PyStaticMethod_getsets) < 0 - || add_more_getsets(&PyType_Type, new_PyType_getsets) < 0) + || add_more_getsets(&PyType_Type, new_PyType_getsets) < 0 + || add_more_getsets(Py_TYPE(wd), new_PyWrapperDescr_getsets) < 0 + ) return -1; Py_DECREF(md); + Py_DECREF(wd); #ifndef _WIN32 // We enable the stack trace in CI, only. const char *testEnv = getenv("QTEST_ENVIRONMENT"); @@ -550,20 +664,26 @@ build_func_to_type(PyObject *obtype) return 0; } +static void +init_module_1(void) +{ + static int init_done = 0; + + if (!init_done) { + pyside_globals = init_phase_1(); + if (pyside_globals != nullptr) + init_done = 1; + } +} + static int PySide_BuildSignatureArgs(PyObject *module, PyObject *type, const char *signatures) { PyObject *type_name, *arg_tup; const char *name = NULL; - static int init_done = 0; - if (!init_done) { - pyside_globals = init_phase_1(); - if (pyside_globals == NULL) - return -1; - init_done = 1; - } + init_module_1();; arg_tup = Py_BuildValue("(Os)", type, signatures); if (arg_tup == NULL) return -1; @@ -599,23 +719,34 @@ PySide_BuildSignatureArgs(PyObject *module, PyObject *type, return 0; } -static PyObject * -PySide_BuildSignatureProps(PyObject *classmod) +static PyMethodDef signature_methods[] = { + {"get_signature", (PyCFunction)get_signature, METH_VARARGS, + "get the __signature__, but pass an optional string parameter"}, + {NULL, NULL} +}; + +static void +init_module_2(void) { - PyObject *arg_tup, *dict, *type_name; static int init_done = 0; if (!init_done) { - if (init_phase_2(pyside_globals) < 0) - return NULL; + init_phase_2(pyside_globals, signature_methods); init_done = 1; } +} + +static PyObject * +PySide_BuildSignatureProps(PyObject *classmod) +{ + PyObject *arg_tup, *dict, *type_name; /* * Here is the second part of the function. * This part will be called on-demand when needed by some attribute. * We simply pick up the arguments that we stored here and replace * them by the function result. */ + init_module_2(); type_name = PyObject_GetAttrString(classmod, "__name__"); if (type_name == NULL) return NULL; diff --git a/sources/shiboken2/libshiboken/signature.h b/sources/shiboken2/libshiboken/signature.h index 3a229cb5c..b65317662 100644 --- a/sources/shiboken2/libshiboken/signature.h +++ b/sources/shiboken2/libshiboken/signature.h @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2017 The Qt Company Ltd. +** Copyright (C) 2018 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of Qt for Python. @@ -45,8 +45,8 @@ extern "C" { -LIBSHIBOKEN_API int SbkSpecial_Type_Ready(PyObject *, PyTypeObject *, const char*); -LIBSHIBOKEN_API void FinishSignatureInitialization(PyObject *, const char*); +LIBSHIBOKEN_API int SbkSpecial_Type_Ready(PyObject *, PyTypeObject *, const char *); //WS +LIBSHIBOKEN_API void FinishSignatureInitialization(PyObject *, const char *); } // extern "C" diff --git a/sources/shiboken2/libshiboken/typespec.cpp b/sources/shiboken2/libshiboken/typespec.cpp index 9123a09e0..a67daf12d 100644 --- a/sources/shiboken2/libshiboken/typespec.cpp +++ b/sources/shiboken2/libshiboken/typespec.cpp @@ -37,6 +37,7 @@ ** ****************************************************************************/ +#include "sbkpython.h" #include "typespec.h" #include -- cgit v1.2.3