diff options
Diffstat (limited to 'sources/shiboken2/libshiboken')
30 files changed, 1115 insertions, 475 deletions
diff --git a/sources/shiboken2/libshiboken/CMakeLists.txt b/sources/shiboken2/libshiboken/CMakeLists.txt index 7cbb22978..a38da8d89 100644 --- a/sources/shiboken2/libshiboken/CMakeLists.txt +++ b/sources/shiboken2/libshiboken/CMakeLists.txt @@ -30,8 +30,8 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/embed/signature_bootstrap.py" "${CMAKE_CURRENT_BINARY_DIR}/embed/signature_bootstrap.py" @ONLY) add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/embed/signature_bootstrap.inc" - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/embed/signature.inc" + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/embed/signature_bootstrap_inc.h" + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/embed/signature_inc.h" COMMAND ${PYTHON_EXECUTABLE} -E "${CMAKE_CURRENT_SOURCE_DIR}/embed/embedding_generator.py" --cmake-dir "${CMAKE_CURRENT_BINARY_DIR}/embed" @@ -53,6 +53,7 @@ sbkconverter.cpp sbkenum.cpp sbkmodule.cpp sbkstring.cpp +sbkstaticstrings.cpp bindingmanager.cpp threadstatesaver.cpp shibokenbuffer.cpp @@ -62,8 +63,8 @@ pep384impl.cpp voidptr.cpp typespec.cpp bufferprocs_py37.cpp -embed/signature_bootstrap.inc -embed/signature.inc +embed/signature_bootstrap_inc.h +embed/signature_inc.h ) get_numpy_location() @@ -129,6 +130,7 @@ install(FILES python25compat.h sbkdbg.h sbkstring.h + sbkstaticstrings.h shiboken.h shibokenmacros.h threadstatesaver.h diff --git a/sources/shiboken2/libshiboken/autodecref.h b/sources/shiboken2/libshiboken/autodecref.h index d3353b1e4..498b1aec4 100644 --- a/sources/shiboken2/libshiboken/autodecref.h +++ b/sources/shiboken2/libshiboken/autodecref.h @@ -96,8 +96,10 @@ public: */ void reset(PyObject *other) { - Py_XDECREF(m_pyObj); + // Safely decref m_pyObj. See Py_XSETREF in object.h . + PyObject *_py_tmp = m_pyObj; m_pyObj = other; + Py_XDECREF(_py_tmp); } private: PyObject *m_pyObj; diff --git a/sources/shiboken2/libshiboken/basewrapper.cpp b/sources/shiboken2/libshiboken/basewrapper.cpp index 9d233b847..78f4cbe8b 100644 --- a/sources/shiboken2/libshiboken/basewrapper.cpp +++ b/sources/shiboken2/libshiboken/basewrapper.cpp @@ -44,6 +44,8 @@ #include "sbkconverter.h" #include "sbkenum.h" #include "sbkstring.h" +#include "sbkstaticstrings.h" +#include "sbkstaticstrings_p.h" #include "autodecref.h" #include "gilstate.h" #include <string> @@ -57,6 +59,10 @@ #include "qapp_macro.h" #include "voidptr.h" +#if defined(__APPLE__) +#include <dlfcn.h> +#endif + namespace { void _destroyParentInfo(SbkObject *obj, bool keepReference); } @@ -73,6 +79,17 @@ static void callDestructor(const Shiboken::DtorAccumulatorVisitor::DestructorEnt extern "C" { +// PYSIDE-939: A general replacement for object_dealloc. +void Sbk_object_dealloc(PyObject *self) +{ + if (PepRuntime_38_flag) { + // PYSIDE-939: Handling references correctly. + // This was not needed before Python 3.8 (Python issue 35810) + Py_DECREF(Py_TYPE(self)); + } + Py_TYPE(self)->tp_free(self); +} + static void SbkObjectTypeDealloc(PyObject *pyObj); static PyObject *SbkObjectTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds); @@ -85,6 +102,23 @@ static PyGetSetDef SbkObjectType_Type_getsetlist[] = { {nullptr} // Sentinel }; +#if PY_VERSION_HEX < 0x03000000 + +static PyObject *SbkObjectType_repr(PyObject *type) +{ + Shiboken::AutoDecRef mod(PyObject_GetAttr(type, Shiboken::PyMagicName::module())); + if (mod.isNull()) + return nullptr; + Shiboken::AutoDecRef name(PyObject_GetAttr(type, Shiboken::PyMagicName::qualname())); + if (name.isNull()) + return nullptr; + return PyString_FromFormat("<class '%s.%s'>", + PyString_AS_STRING(mod.object()), + PyString_AS_STRING(name.object())); +} + +#endif // PY_VERSION_HEX < 0x03000000 + static PyType_Slot SbkObjectType_Type_slots[] = { {Py_tp_dealloc, reinterpret_cast<void *>(SbkObjectTypeDealloc)}, {Py_tp_setattro, reinterpret_cast<void *>(PyObject_GenericSetAttr)}, @@ -93,10 +127,13 @@ static PyType_Slot SbkObjectType_Type_slots[] = { {Py_tp_new, reinterpret_cast<void *>(SbkObjectTypeTpNew)}, {Py_tp_free, reinterpret_cast<void *>(PyObject_GC_Del)}, {Py_tp_getset, reinterpret_cast<void *>(SbkObjectType_Type_getsetlist)}, +#if PY_VERSION_HEX < 0x03000000 + {Py_tp_repr, reinterpret_cast<void *>(SbkObjectType_repr)}, +#endif {0, nullptr} }; static PyType_Spec SbkObjectType_Type_spec = { - "Shiboken.ObjectType", + "1:Shiboken.ObjectType", 0, // basicsize (inserted later) sizeof(PyMemberDef), Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, @@ -160,11 +197,10 @@ patch_tp_new_wrapper(PyTypeObject *type) * The old tp_new_wrapper is added to all types that have tp_new. * We patch that with a version that ignores the heaptype flag. */ - static PyObject *__new__ = nullptr; + auto newMethod = Shiboken::PyMagicName::new_(); if (old_tp_new_wrapper == nullptr) { - if ((__new__ = Py_BuildValue("s", "__new__")) == nullptr) - return -1; - PyObject *func = PyDict_GetItem(PyType_Type.tp_dict, __new__); + PyObject *func = PyDict_GetItem(PyType_Type.tp_dict, newMethod); + assert(func); PyCFunctionObject *pycf_ob = reinterpret_cast<PyCFunctionObject *>(func); old_tp_new_wrapper = reinterpret_cast<ternaryfunc>(pycf_ob->m_ml->ml_meth); } @@ -172,7 +208,7 @@ patch_tp_new_wrapper(PyTypeObject *type) Py_ssize_t i, n = PyTuple_GET_SIZE(mro); for (i = 0; i < n; i++) { type = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(mro, i)); - PyObject *existing = PyDict_GetItem(type->tp_dict, __new__); + PyObject *existing = PyDict_GetItem(type->tp_dict, newMethod); if (existing && PyCFunction_Check(existing) && type->tp_flags & Py_TPFLAGS_HEAPTYPE) { auto *pycf_ob = reinterpret_cast<PyCFunctionObject *>(existing); @@ -182,7 +218,7 @@ patch_tp_new_wrapper(PyTypeObject *type) if (existing_wrapper == old_tp_new_wrapper) { PyObject *ob_type = reinterpret_cast<PyObject *>(type); Shiboken::AutoDecRef func(PyCFunction_New(tp_new_methoddef, ob_type)); - if (func.isNull() || PyDict_SetItem(type->tp_dict, __new__, func)) + if (func.isNull() || PyDict_SetItem(type->tp_dict, newMethod, func)) return -1; } } @@ -199,7 +235,7 @@ PyTypeObject *SbkObjectType_TypeF(void) if (!type) { SbkObjectType_Type_spec.basicsize = PepHeapType_SIZE + sizeof(SbkObjectTypePrivate); - type = reinterpret_cast<PyTypeObject *>(PyType_FromSpec(&SbkObjectType_Type_spec)); + type = reinterpret_cast<PyTypeObject *>(SbkType_FromSpec(&SbkObjectType_Type_spec)); #if PY_VERSION_HEX < 0x03000000 if (patch_tp_new_wrapper(type) < 0) return nullptr; @@ -273,7 +309,7 @@ static PyType_Slot SbkObject_Type_slots[] = { {0, nullptr} }; static PyType_Spec SbkObject_Type_spec = { - "Shiboken.Object", + "1:Shiboken.Object", sizeof(SbkObject), 0, Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, @@ -285,7 +321,7 @@ SbkObjectType *SbkObject_TypeF(void) { static PyTypeObject *type = nullptr; if (!type) { - type = reinterpret_cast<PyTypeObject *>(PyType_FromSpec(&SbkObject_Type_spec)); + type = reinterpret_cast<PyTypeObject *>(SbkType_FromSpec(&SbkObject_Type_spec)); Py_TYPE(type) = SbkObjectType_TypeF(); Py_INCREF(Py_TYPE(type)); type->tp_weaklistoffset = offsetof(SbkObject, weakreflist); @@ -309,8 +345,32 @@ static void SbkDeallocWrapperCommon(PyObject *pyObj, bool canDelete) // Need to decref the type if this is the dealloc func; if type // is subclassed, that dealloc func will decref (see subtype_dealloc // in typeobject.c in the python sources) - bool needTypeDecref = (PyType_GetSlot(pyType, Py_tp_dealloc) == SbkDeallocWrapper + bool needTypeDecref = (false + || PyType_GetSlot(pyType, Py_tp_dealloc) == SbkDeallocWrapper || PyType_GetSlot(pyType, Py_tp_dealloc) == SbkDeallocWrapperWithPrivateDtor); + if (PepRuntime_38_flag) { + // PYSIDE-939: Additional rule: Also when a subtype is heap allocated, + // then the subtype_dealloc deref will be suppressed, and we need again + // to supply a decref. + needTypeDecref |= (pyType->tp_base->tp_flags & Py_TPFLAGS_HEAPTYPE) != 0; + } + +#if defined(__APPLE__) + // Just checking once that our assumptions are right. + if (false) { + void *p = PyType_GetSlot(pyType, Py_tp_dealloc); + Dl_info dl_info; + dladdr(p, &dl_info); + fprintf(stderr, "tp_dealloc is %s\n", dl_info.dli_sname); + } + // Gives one of our functions + // "Sbk_object_dealloc" + // "SbkDeallocWrapperWithPrivateDtor" + // "SbkDeallocQAppWrapper" + // "SbkDeallocWrapper" + // but for typedealloc_test.py we get + // "subtype_dealloc" +#endif // Ensure that the GC is no longer tracking this object to avoid a // possible reentrancy problem. Since there are multiple steps involved @@ -369,6 +429,11 @@ static void SbkDeallocWrapperCommon(PyObject *pyObj, bool canDelete) if (needTypeDecref) Py_DECREF(pyType); + if (PepRuntime_38_flag) { + // PYSIDE-939: Handling references correctly. + // This was not needed before Python 3.8 (Python issue 35810) + Py_DECREF(pyType); + } } void SbkDeallocWrapper(PyObject *pyObj) @@ -380,7 +445,7 @@ void SbkDeallocQAppWrapper(PyObject *pyObj) { SbkDeallocWrapper(pyObj); // PYSIDE-571: make sure to create a singleton deleted qApp. - MakeSingletonQAppWrapper(nullptr); + Py_DECREF(MakeQAppWrapper(nullptr)); } void SbkDeallocWrapperWithPrivateDtor(PyObject *self) @@ -412,6 +477,11 @@ void SbkObjectTypeDealloc(PyObject *pyObj) #ifndef Py_LIMITED_API Py_TRASHCAN_SAFE_END(pyObj); #endif + if (PepRuntime_38_flag) { + // PYSIDE-939: Handling references correctly. + // This was not needed before Python 3.8 (Python issue 35810) + Py_DECREF(Py_TYPE(pyObj)); + } } PyObject *SbkObjectTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds) @@ -453,7 +523,18 @@ PyObject *SbkObjectTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *k // The meta type creates a new type when the Python programmer extends a wrapped C++ class. auto type_new = reinterpret_cast<newfunc>(PyType_Type.tp_new); + + // PYSIDE-939: This is a temporary patch that circumvents the problem + // with Py_TPFLAGS_METHOD_DESCRIPTOR until this is finally solved. + // PyType_Ready uses mro(). We need to temporarily remove the flag from it's type. + // We cannot use PyMethodDescr_Type since it is not exported by Python 2.7 . + static PyTypeObject *PyMethodDescr_TypePtr = Py_TYPE( + PyObject_GetAttr(reinterpret_cast<PyObject *>(&PyType_Type), Shiboken::PyName::mro())); + auto hold = PyMethodDescr_TypePtr->tp_flags; + PyMethodDescr_TypePtr->tp_flags &= ~Py_TPFLAGS_METHOD_DESCRIPTOR; auto *newType = reinterpret_cast<SbkObjectType *>(type_new(metatype, args, kwds)); + PyMethodDescr_TypePtr->tp_flags = hold; + if (!newType) return nullptr; #if PY_VERSION_HEX < 0x03000000 @@ -550,18 +631,11 @@ PyObject *SbkQAppTpNew(PyTypeObject *subtype, PyObject *, PyObject *) subtype->tp_free = PyObject_Del; } #endif - auto self = reinterpret_cast<SbkObject *>(MakeSingletonQAppWrapper(subtype)); + auto self = reinterpret_cast<SbkObject *>(MakeQAppWrapper(subtype)); return self == nullptr ? nullptr : _setupNew(self, subtype); } -void -object_dealloc(PyObject *self) -{ - Py_TYPE(self)->tp_free(self); -} - -PyObject * -SbkDummyNew(PyTypeObject *type, PyObject *, PyObject *) +PyObject *SbkDummyNew(PyTypeObject *type, PyObject *, PyObject *) { // PYSIDE-595: Give the same error as type_call does when tp_new is NULL. PyErr_Format(PyExc_TypeError, @@ -570,6 +644,51 @@ SbkDummyNew(PyTypeObject *type, PyObject *, PyObject *) return nullptr; } +PyObject *SbkType_FromSpec(PyType_Spec *spec) +{ + return SbkType_FromSpecWithBases(spec, nullptr); +} + +PyObject *SbkType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) +{ + // PYSIDE-1286: Generate correct __module__ and __qualname__ + // The name field can now be extended by an "n:" prefix which is + // the number of modules in the name. The default is 1. + // + // Example: + // "2:mainmod.submod.mainclass.subclass" + // results in + // __module__ : "mainmod.submod" + // __qualname__ : "mainclass.subclass" + // __name__ : "subclass" + + PyType_Spec new_spec = *spec; + const char *colon = strchr(spec->name, ':'); + assert(colon); + int package_level = atoi(spec->name); + const char *mod = new_spec.name = colon + 1; + + PyObject *type = PyType_FromSpecWithBases(&new_spec, bases); + if (type == nullptr) + return nullptr; + + const char *qual = mod; + for (int idx = package_level; idx > 0; --idx) { + const char *dot = strchr(qual, '.'); + if (!dot) + break; + qual = dot + 1; + } + int mlen = qual - mod - 1; + Shiboken::AutoDecRef module(Shiboken::String::fromCString(mod, mlen)); + Shiboken::AutoDecRef qualname(Shiboken::String::fromCString(qual)); + if (PyObject_SetAttr(type, Shiboken::PyMagicName::module(), module) < 0) + return nullptr; + if (PyObject_SetAttr(type, Shiboken::PyMagicName::qualname(), qualname) < 0) + return nullptr; + return type; +} + } //extern "C" @@ -691,13 +810,13 @@ void init() Shiboken::ObjectType::initPrivateData(SbkObject_TypeF()); if (PyType_Ready(SbkEnumType_TypeF()) < 0) - Py_FatalError("[libshiboken] Failed to initialise Shiboken.SbkEnumType metatype."); + Py_FatalError("[libshiboken] Failed to initialize Shiboken.SbkEnumType metatype."); if (PyType_Ready(SbkObjectType_TypeF()) < 0) - Py_FatalError("[libshiboken] Failed to initialise Shiboken.BaseWrapperType metatype."); + Py_FatalError("[libshiboken] Failed to initialize Shiboken.BaseWrapperType metatype."); if (PyType_Ready(reinterpret_cast<PyTypeObject *>(SbkObject_TypeF())) < 0) - Py_FatalError("[libshiboken] Failed to initialise Shiboken.BaseWrapper type."); + Py_FatalError("[libshiboken] Failed to initialize Shiboken.BaseWrapper type."); VoidPtr::init(); @@ -847,7 +966,7 @@ introduceWrapperType(PyObject *enclosingObject, { typeSpec->slots[0].pfunc = reinterpret_cast<void *>(baseType ? baseType : SbkObject_TypeF()); - PyObject *heaptype = PyType_FromSpecWithBases(typeSpec, baseTypes); + PyObject *heaptype = SbkType_FromSpecWithBases(typeSpec, baseTypes); Py_TYPE(heaptype) = SbkObjectType_TypeF(); Py_INCREF(Py_TYPE(heaptype)); auto *type = reinterpret_cast<SbkObjectType *>(heaptype); diff --git a/sources/shiboken2/libshiboken/basewrapper.h b/sources/shiboken2/libshiboken/basewrapper.h index 7faf223bd..4132b5cc5 100644 --- a/sources/shiboken2/libshiboken/basewrapper.h +++ b/sources/shiboken2/libshiboken/basewrapper.h @@ -64,6 +64,9 @@ struct LIBSHIBOKEN_API SbkObject }; +/// PYSIDE-939: A general replacement for object_dealloc. +LIBSHIBOKEN_API void Sbk_object_dealloc(PyObject *self); + /// Dealloc the python object \p pyObj and the C++ object represented by it. LIBSHIBOKEN_API void SbkDeallocWrapper(PyObject *pyObj); LIBSHIBOKEN_API void SbkDeallocQAppWrapper(PyObject *pyObj); @@ -116,9 +119,13 @@ LIBSHIBOKEN_API PyObject *SbkQAppTpNew(PyTypeObject *subtype, PyObject *args, Py * nullptr. But the default before conversion to heaptypes was to assign * object_dealloc. This seems to be a bug in the Limited API. */ -LIBSHIBOKEN_API void object_dealloc(PyObject *); +/// PYSIDE-939: Replaced by Sbk_object_dealloc. LIBSHIBOKEN_API PyObject *SbkDummyNew(PyTypeObject *type, PyObject *, PyObject *); +/// PYSIDE-1286: Generate correct __module__ and __qualname__ +LIBSHIBOKEN_API PyObject *SbkType_FromSpec(PyType_Spec *); +LIBSHIBOKEN_API PyObject *SbkType_FromSpecWithBases(PyType_Spec *, PyObject *); + } // extern "C" namespace Shiboken diff --git a/sources/shiboken2/libshiboken/bindingmanager.cpp b/sources/shiboken2/libshiboken/bindingmanager.cpp index 725150e87..1f18ed60a 100644 --- a/sources/shiboken2/libshiboken/bindingmanager.cpp +++ b/sources/shiboken2/libshiboken/bindingmanager.cpp @@ -37,6 +37,7 @@ ** ****************************************************************************/ +#include "autodecref.h" #include "basewrapper.h" #include "basewrapper_p.h" #include "bindingmanager.h" @@ -288,7 +289,7 @@ PyObject *BindingManager::getOverride(const void *cptr, const char *methodName) } } - PyObject *pyMethodName = Shiboken::String::fromCString(methodName); + Shiboken::AutoDecRef pyMethodName(Shiboken::String::fromCString(methodName)); PyObject *method = PyObject_GetAttr(reinterpret_cast<PyObject *>(wrapper), pyMethodName); if (method && PyMethod_Check(method) @@ -302,16 +303,13 @@ PyObject *BindingManager::getOverride(const void *cptr, const char *methodName) auto *parent = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(mro, i)); if (parent->tp_dict) { defaultMethod = PyDict_GetItem(parent->tp_dict, pyMethodName); - if (defaultMethod && PyMethod_GET_FUNCTION(method) != defaultMethod) { - Py_DECREF(pyMethodName); + if (defaultMethod && PyMethod_GET_FUNCTION(method) != defaultMethod) return method; - } } } } Py_XDECREF(method); - Py_DECREF(pyMethodName); return nullptr; } diff --git a/sources/shiboken2/libshiboken/bufferprocs_py37.cpp b/sources/shiboken2/libshiboken/bufferprocs_py37.cpp index ddb07390e..da6bb00a3 100644 --- a/sources/shiboken2/libshiboken/bufferprocs_py37.cpp +++ b/sources/shiboken2/libshiboken/bufferprocs_py37.cpp @@ -47,7 +47,7 @@ #ifdef Py_LIMITED_API -#include "pep384impl.h" +#include "sbkpython.h" /* Buffer C-API for Python 3.0 */ int diff --git a/sources/shiboken2/libshiboken/embed/embedding_generator.py b/sources/shiboken2/libshiboken/embed/embedding_generator.py index 77aa5c329..15f63649b 100644 --- a/sources/shiboken2/libshiboken/embed/embedding_generator.py +++ b/sources/shiboken2/libshiboken/embed/embedding_generator.py @@ -88,7 +88,7 @@ def create_zipfile(limited_api): and make a chunked base64 encoded file from it. """ zip_name = "signature.zip" - inc_name = "signature.inc" + inc_name = "signature_inc.h" flag = '-b' if sys.version_info >= (3,) else '' os.chdir(work_dir) @@ -131,7 +131,7 @@ def create_zipfile(limited_api): tmp.close() # also generate a simple embeddable .pyc file for signature_bootstrap.pyc boot_name = "signature_bootstrap.py" if limited_api else "signature_bootstrap.pyc" - with open(boot_name, "rb") as ldr, open("signature_bootstrap.inc", "w") as inc: + with open(boot_name, "rb") as ldr, open("signature_bootstrap_inc.h", "w") as inc: _embed_bytefile(ldr, inc, limited_api) os.chdir(cur_dir) diff --git a/sources/shiboken2/libshiboken/gilstate.cpp b/sources/shiboken2/libshiboken/gilstate.cpp index a59c6f01e..76a4d0e61 100644 --- a/sources/shiboken2/libshiboken/gilstate.cpp +++ b/sources/shiboken2/libshiboken/gilstate.cpp @@ -63,5 +63,12 @@ void GilState::release() } } +// Abandon the lock: Only for special situations, like termination of a +// POSIX thread (PYSIDE 1282). +void GilState::abandon() +{ + m_locked = false; +} + } // namespace Shiboken diff --git a/sources/shiboken2/libshiboken/gilstate.h b/sources/shiboken2/libshiboken/gilstate.h index d22f688ba..fbf39ead0 100644 --- a/sources/shiboken2/libshiboken/gilstate.h +++ b/sources/shiboken2/libshiboken/gilstate.h @@ -57,6 +57,7 @@ public: GilState(); ~GilState(); void release(); + void abandon(); private: PyGILState_STATE m_gstate; bool m_locked = false; diff --git a/sources/shiboken2/libshiboken/helper.cpp b/sources/shiboken2/libshiboken/helper.cpp index fac72d56f..013080b6e 100644 --- a/sources/shiboken2/libshiboken/helper.cpp +++ b/sources/shiboken2/libshiboken/helper.cpp @@ -39,6 +39,7 @@ #include "helper.h" #include "sbkstring.h" +#include "sbkstaticstrings.h" #include <stdarg.h> #ifdef _WIN32 @@ -78,7 +79,7 @@ bool listToArgcArgv(PyObject *argList, int *argc, char ***argv, const char *defa if (hasEmptyArgList) { // Try to get the script name PyObject *globals = PyEval_GetGlobals(); - PyObject *appName = PyDict_GetItemString(globals, "__file__"); + PyObject *appName = PyDict_GetItem(globals, Shiboken::PyMagicName::file()); (*argv)[0] = strdup(appName ? Shiboken::String::toCString(appName) : defaultAppName); } else { for (int i = 0; i < numArgs; ++i) { diff --git a/sources/shiboken2/libshiboken/pep384impl.cpp b/sources/shiboken2/libshiboken/pep384impl.cpp index fe7157d24..f07cac613 100644 --- a/sources/shiboken2/libshiboken/pep384impl.cpp +++ b/sources/shiboken2/libshiboken/pep384impl.cpp @@ -37,8 +37,12 @@ ** ****************************************************************************/ -#include "pep384impl.h" +#include "sbkpython.h" #include "autodecref.h" +#include "sbkstaticstrings.h" +#include "sbkstaticstrings_p.h" +#include <stdlib.h> + extern "C" { @@ -46,26 +50,19 @@ extern "C" /* * The documentation is located in pep384impl_doc.rst */ - -/***************************************************************************** - * - * Support for object.h - * - */ - +#if PY_VERSION_HEX < 0x03000000 +#define IS_PY2 +#endif // PY_VERSION_HEX < 0x03000000 /* * Here is the verification code for PyTypeObject. * We create a type object and check if its fields * appear at the right offsets. */ +#ifdef Py_LIMITED_API #define make_dummy_int(x) (x * sizeof(void *)) #define make_dummy(x) (reinterpret_cast<void *>(make_dummy_int(x))) -#ifdef Py_LIMITED_API -datetime_struc *PyDateTimeAPI = NULL; -#endif - static PyObject * dummy_func(PyObject * /* self */, PyObject * /* args */) { @@ -81,27 +78,33 @@ static PyGetSetDef probe_getseters[] = { {nullptr} /* Sentinel */ }; -#define probe_tp_call make_dummy(1) -#define probe_tp_str make_dummy(2) -#define probe_tp_traverse make_dummy(3) -#define probe_tp_clear make_dummy(4) +#define probe_tp_dealloc make_dummy(1) +#define probe_tp_repr make_dummy(2) +#define probe_tp_call make_dummy(3) +#define probe_tp_str make_dummy(4) +#define probe_tp_traverse make_dummy(5) +#define probe_tp_clear make_dummy(6) +#define probe_tp_iternext make_dummy(7) #define probe_tp_methods probe_methoddef #define probe_tp_getset probe_getseters -#define probe_tp_descr_get make_dummy(7) -#define probe_tp_init make_dummy(8) -#define probe_tp_alloc make_dummy(9) -#define probe_tp_new make_dummy(10) -#define probe_tp_free make_dummy(11) -#define probe_tp_is_gc make_dummy(12) +#define probe_tp_descr_get make_dummy(10) +#define probe_tp_init make_dummy(11) +#define probe_tp_alloc make_dummy(12) +#define probe_tp_new make_dummy(13) +#define probe_tp_free make_dummy(14) +#define probe_tp_is_gc make_dummy(15) #define probe_tp_name "type.probe" #define probe_tp_basicsize make_dummy_int(42) static PyType_Slot typeprobe_slots[] = { + {Py_tp_dealloc, probe_tp_dealloc}, + {Py_tp_repr, probe_tp_repr}, {Py_tp_call, probe_tp_call}, {Py_tp_str, probe_tp_str}, {Py_tp_traverse, probe_tp_traverse}, {Py_tp_clear, probe_tp_clear}, + {Py_tp_iternext, probe_tp_iternext}, {Py_tp_methods, probe_tp_methods}, {Py_tp_getset, probe_tp_getset}, {Py_tp_descr_get, probe_tp_descr_get}, @@ -125,24 +128,27 @@ check_PyTypeObject_valid() { auto *obtype = reinterpret_cast<PyObject *>(&PyType_Type); auto *probe_tp_base = reinterpret_cast<PyTypeObject *>( - PyObject_GetAttrString(obtype, "__base__")); - PyObject *probe_tp_bases = PyObject_GetAttrString(obtype, "__bases__"); + PyObject_GetAttr(obtype, Shiboken::PyMagicName::base())); + auto *probe_tp_bases = PyObject_GetAttr(obtype, Shiboken::PyMagicName::bases()); auto *check = reinterpret_cast<PyTypeObject *>( PyType_FromSpecWithBases(&typeprobe_spec, probe_tp_bases)); auto *typetype = reinterpret_cast<PyTypeObject *>(obtype); - PyObject *w = PyObject_GetAttrString(obtype, "__weakrefoffset__"); + PyObject *w = PyObject_GetAttr(obtype, Shiboken::PyMagicName::weakrefoffset()); long probe_tp_weakrefoffset = PyLong_AsLong(w); - PyObject *d = PyObject_GetAttrString(obtype, "__dictoffset__"); + PyObject *d = PyObject_GetAttr(obtype, Shiboken::PyMagicName::dictoffset()); long probe_tp_dictoffset = PyLong_AsLong(d); - PyObject *probe_tp_mro = PyObject_GetAttrString(obtype, "__mro__"); + PyObject *probe_tp_mro = PyObject_GetAttr(obtype, Shiboken::PyMagicName::mro()); if (false || strcmp(probe_tp_name, check->tp_name) != 0 || probe_tp_basicsize != check->tp_basicsize + || probe_tp_dealloc != check->tp_dealloc + || probe_tp_repr != check->tp_repr || probe_tp_call != check->tp_call || probe_tp_str != check->tp_str || probe_tp_traverse != check->tp_traverse || probe_tp_clear != check->tp_clear || probe_tp_weakrefoffset != typetype->tp_weaklistoffset + || probe_tp_iternext != check->tp_iternext || probe_tp_methods != check->tp_methods || probe_tp_getset != check->tp_getset || probe_tp_base != typetype->tp_base @@ -156,7 +162,8 @@ check_PyTypeObject_valid() || probe_tp_free != check->tp_free || probe_tp_is_gc != check->tp_is_gc || probe_tp_bases != typetype->tp_bases - || probe_tp_mro != typetype->tp_mro) + || probe_tp_mro != typetype->tp_mro + || Py_TPFLAGS_DEFAULT != (check->tp_flags & Py_TPFLAGS_DEFAULT)) Py_FatalError("The structure of type objects has changed!"); Py_DECREF(check); Py_DECREF(probe_tp_base); @@ -166,18 +173,18 @@ check_PyTypeObject_valid() Py_DECREF(probe_tp_mro); } - -#ifdef Py_LIMITED_API - #if PY_VERSION_HEX < PY_ISSUE33738_SOLVED #include "pep384_issue33738.cpp" #endif +#endif // Py_LIMITED_API + /***************************************************************************** * * Support for unicodeobject.h * */ +#ifdef Py_LIMITED_API char * _PepUnicode_AsString(PyObject *str) @@ -193,15 +200,15 @@ _PepUnicode_AsString(PyObject *str) #define TOSTRING(x) STRINGIFY(x) #define AT __FILE__ ":" TOSTRING(__LINE__) - static PyObject *cstring_dict = NULL; - if (cstring_dict == NULL) { + static PyObject *cstring_dict = nullptr; + if (cstring_dict == nullptr) { cstring_dict = PyDict_New(); - if (cstring_dict == NULL) + if (cstring_dict == nullptr) Py_FatalError("Error in " AT); } - PyObject *bytesStr = PyUnicode_AsEncodedString(str, "utf8", NULL); - PyObject *entry = PyDict_GetItem(cstring_dict, bytesStr); - if (entry == NULL) { + PyObject *bytesStr = PyUnicode_AsEncodedString(str, "utf8", nullptr); + PyObject *entry = PyDict_GetItemWithError(cstring_dict, bytesStr); + if (entry == nullptr) { int e = PyDict_SetItem(cstring_dict, bytesStr, bytesStr); if (e != 0) Py_FatalError("Error in " AT); @@ -211,12 +218,14 @@ _PepUnicode_AsString(PyObject *str) Py_DECREF(bytesStr); return PyBytes_AsString(entry); } +#endif // Py_LIMITED_API /***************************************************************************** * * Support for longobject.h * */ +#ifdef Py_LIMITED_API /* * This is the original Python function _PyLong_AsInt() from longobject.c . @@ -239,15 +248,18 @@ _PepLong_AsInt(PyObject *obj) "Python int too large to convert to C int"); return -1; } - return (int)result; + return int(result); } +#endif // Py_LIMITED_API /***************************************************************************** * * Support for pydebug.h * */ -static PyObject *sys_flags = NULL; +#ifdef Py_LIMITED_API + +static PyObject *sys_flags = nullptr; int Pep_GetFlag(const char *name) @@ -257,13 +269,13 @@ Pep_GetFlag(const char *name) if (!initialized) { sys_flags = PySys_GetObject("flags"); - // func gives no error if NULL is returned and does not incref. + // func gives no error if nullptr is returned and does not incref. Py_XINCREF(sys_flags); initialized = 1; } - if (sys_flags != NULL) { + if (sys_flags != nullptr) { PyObject *ob_ret = PyObject_GetAttrString(sys_flags, name); - if (ob_ret != NULL) { + if (ob_ret != nullptr) { long long_ret = PyLong_AsLong(ob_ret); Py_DECREF(ob_ret); ret = (int) long_ret; @@ -285,12 +297,14 @@ Pep_GetVerboseFlag() } return verbose_flag; } +#endif // Py_LIMITED_API /***************************************************************************** * * Support for code.h * */ +#ifdef Py_LIMITED_API int PepCode_Get(PyCodeObject *co, const char *name) @@ -300,28 +314,32 @@ PepCode_Get(PyCodeObject *co, const char *name) int ret = -1; ob_ret = PyObject_GetAttrString(ob, name); - if (ob_ret != NULL) { + if (ob_ret != nullptr) { long long_ret = PyLong_AsLong(ob_ret); Py_DECREF(ob_ret); ret = (int) long_ret; } return ret; } +#endif // Py_LIMITED_API /***************************************************************************** * * Support for datetime.h * */ +#ifdef Py_LIMITED_API + +datetime_struc *PyDateTimeAPI = nullptr; static PyTypeObject *dt_getCheck(const char *name) { PyObject *op = PyObject_GetAttrString(PyDateTimeAPI->module, name); - if (op == NULL) { + if (op == nullptr) { fprintf(stderr, "datetime.%s not found\n", name); Py_FatalError("aborting"); } - return (PyTypeObject *)op; + return reinterpret_cast<PyTypeObject *>(op); } // init_DateTime is called earlier than our module init. @@ -332,10 +350,10 @@ init_DateTime(void) static int initialized = 0; if (!initialized) { PyDateTimeAPI = (datetime_struc *)malloc(sizeof(datetime_struc)); - if (PyDateTimeAPI == NULL) + if (PyDateTimeAPI == nullptr) Py_FatalError("PyDateTimeAPI malloc error, aborting"); PyDateTimeAPI->module = PyImport_ImportModule("datetime"); - if (PyDateTimeAPI->module == NULL) + if (PyDateTimeAPI->module == nullptr) Py_FatalError("datetime module not found, aborting"); PyDateTimeAPI->DateType = dt_getCheck("date"); PyDateTimeAPI->DateTimeType = dt_getCheck("datetime"); @@ -354,7 +372,7 @@ PyDateTime_Get(PyObject *ob, const char *name) int ret = -1; ob_ret = PyObject_GetAttrString(ob, name); - if (ob_ret != NULL) { + if (ob_ret != nullptr) { long long_ret = PyLong_AsLong(ob_ret); Py_DECREF(ob_ret); ret = (int) long_ret; @@ -384,21 +402,23 @@ PyTime_FromTime(int hour, int min, int sec, int usec) return PyObject_CallFunction((PyObject *)PyDateTimeAPI->TimeType, (char *)"(iiii)", hour, min, sec, usec); } +#endif // Py_LIMITED_API /***************************************************************************** * * Support for pythonrun.h * */ +#ifdef Py_LIMITED_API // Flags are ignored in these simple helpers. PyObject * PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) { PyObject *code = Py_CompileString(str, "pyscript", start); - PyObject *ret = NULL; + PyObject *ret = nullptr; - if (code != NULL) { + if (code != nullptr) { ret = PyEval_EvalCode(code, globals, locals); } Py_XDECREF(code); @@ -409,41 +429,41 @@ PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) // This is only a simple local helper that returns a computed variable. // Used also in Python 2. -#if defined(Py_LIMITED_API) || PY_VERSION_HEX < 0x03000000 +#if defined(Py_LIMITED_API) || defined(IS_PY2) static PyObject * -PepRun_GetResult(const char *command, const char *resvar) +PepRun_GetResult(const char *command) { PyObject *d, *v, *res; d = PyDict_New(); - if (d == NULL || PyDict_SetItemString(d, "__builtins__", - PyEval_GetBuiltins()) < 0) - return NULL; + if (d == nullptr + || PyDict_SetItem(d, Shiboken::PyMagicName::builtins(), PyEval_GetBuiltins()) < 0) { + return nullptr; + } v = PyRun_String(command, Py_file_input, d, d); - res = v ? PyDict_GetItemString(d, resvar) : NULL; + res = v ? PyDict_GetItem(d, Shiboken::PyName::result()) : nullptr; Py_XDECREF(v); Py_DECREF(d); return res; } -#endif // Py_LIMITED_API || Python 2 - -#ifdef Py_LIMITED_API +#endif // defined(Py_LIMITED_API) || defined(IS_PY2) /***************************************************************************** * * Support for classobject.h * */ +#ifdef Py_LIMITED_API -PyTypeObject *PepMethod_TypePtr = NULL; +PyTypeObject *PepMethod_TypePtr = nullptr; static PyTypeObject *getMethodType(void) { static const char prog[] = "class _C:\n" " def _m(self): pass\n" - "MethodType = type(_C()._m)\n"; - return (PyTypeObject *) PepRun_GetResult(prog, "MethodType"); + "result = type(_C()._m)\n"; + return reinterpret_cast<PyTypeObject *>(PepRun_GetResult(prog)); } // We have no access to PyMethod_New and must call types.MethodType, instead. @@ -457,7 +477,7 @@ PyMethod_New(PyObject *func, PyObject *self) PyObject * PyMethod_Function(PyObject *im) { - PyObject *ret = PyObject_GetAttrString(im, "__func__"); + PyObject *ret = PyObject_GetAttr(im, Shiboken::PyMagicName::func()); // We have to return a borrowed reference. Py_DECREF(ret); @@ -467,19 +487,21 @@ PyMethod_Function(PyObject *im) PyObject * PyMethod_Self(PyObject *im) { - PyObject *ret = PyObject_GetAttrString(im, "__self__"); + PyObject *ret = PyObject_GetAttr(im, Shiboken::PyMagicName::self()); // We have to return a borrowed reference. // If we don't obey that here, then we get a test error! Py_DECREF(ret); return ret; } +#endif // Py_LIMITED_API /***************************************************************************** * * Support for funcobject.h * */ +#ifdef Py_LIMITED_API PyObject * PepFunction_Get(PyObject *ob, const char *name) @@ -494,22 +516,64 @@ PepFunction_Get(PyObject *ob, const char *name) // This became necessary after Windows was activated. -PyTypeObject *PepFunction_TypePtr = NULL; +PyTypeObject *PepFunction_TypePtr = nullptr; static PyTypeObject *getFunctionType(void) { static const char prog[] = - "from types import FunctionType\n"; - return (PyTypeObject *) PepRun_GetResult(prog, "FunctionType"); + "from types import FunctionType as result\n"; + return reinterpret_cast<PyTypeObject *>(PepRun_GetResult(prog)); +} +#endif // Py_LIMITED_API || Python 2 + +/***************************************************************************** + * + * Support for dictobject.h + * + */ + +// PYSIDE-803, PYSIDE-813: We need that GIL-free version from Python 2.7.12 . +#ifdef IS_PY2 + +/* Variant of PyDict_GetItem() that doesn't suppress exceptions. + This returns NULL *with* an exception set if an exception occurred. + It returns NULL *without* an exception set if the key wasn't present. +*/ +PyObject * +PyDict_GetItemWithError(PyObject *op, PyObject *key) +{ + long hash; + PyDictObject *mp = reinterpret_cast<PyDictObject *>(op); + PyDictEntry *ep; + if (!PyDict_Check(op)) { + PyErr_BadInternalCall(); + return nullptr; + } + if (!PyString_CheckExact(key) || + (hash = (reinterpret_cast<PyStringObject *>(key))->ob_shash) == -1) + { + hash = PyObject_Hash(key); + if (hash == -1) { + return nullptr; + } + } + + ep = (mp->ma_lookup)(mp, key, hash); + if (ep == nullptr) { + return nullptr; + } + return ep->me_value; } +#endif // IS_PY2 /***************************************************************************** * * Extra support for signature.cpp * */ +#ifdef Py_LIMITED_API -PyTypeObject *PepStaticMethod_TypePtr = NULL; +PyTypeObject *PepStaticMethod_TypePtr = nullptr; static PyTypeObject * getStaticMethodType(void) @@ -518,8 +582,8 @@ getStaticMethodType(void) // "StaticMethodType = type(str.__dict__['maketrans'])\n"; static const char prog[] = "from xxsubtype import spamlist\n" - "StaticMethod_Type = type(spamlist.__dict__['staticmeth'])\n"; - return (PyTypeObject *) PepRun_GetResult(prog, "StaticMethod_Type"); + "result = type(spamlist.__dict__['staticmeth'])\n"; + return reinterpret_cast<PyTypeObject *>(PepRun_GetResult(prog)); } typedef struct { @@ -533,25 +597,25 @@ PyStaticMethod_New(PyObject *callable) { staticmethod *sm = (staticmethod *) PyType_GenericAlloc(PepStaticMethod_TypePtr, 0); - if (sm != NULL) { + if (sm != nullptr) { Py_INCREF(callable); sm->sm_callable = callable; } - return (PyObject *)sm; + return reinterpret_cast<PyObject *>(sm); } #endif // Py_LIMITED_API -#if PY_VERSION_HEX < 0x03000000 -PyTypeObject *PepMethodDescr_TypePtr = NULL; +#ifdef IS_PY2 +PyTypeObject *PepMethodDescr_TypePtr = nullptr; static PyTypeObject * getMethodDescrType(void) { static const char prog[] = - "MethodDescr_Type = type(str.split)\n"; - return (PyTypeObject *) PepRun_GetResult(prog, "MethodDescr_Type"); + "result = type(str.split)\n"; + return reinterpret_cast<PyTypeObject *>(PepRun_GetResult(prog)); } -#endif +#endif // IS_PY2 /***************************************************************************** * @@ -574,15 +638,47 @@ PepType_GetNameStr(PyTypeObject *type) /***************************************************************************** * + * Newly introduced convenience functions + * + */ +#if PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) + +PyObject * +PyImport_GetModule(PyObject *name) +{ + PyObject *m; + PyObject *modules = PyImport_GetModuleDict(); + if (modules == NULL) { + PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); + return NULL; + } + Py_INCREF(modules); + if (PyDict_CheckExact(modules)) { + m = PyDict_GetItemWithError(modules, name); /* borrowed */ + Py_XINCREF(m); + } + else { + m = PyObject_GetItem(modules, name); + if (m == NULL && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + } + } + Py_DECREF(modules); + return m; +} + +#endif // PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) +/***************************************************************************** + * * Extra support for name mangling * */ #ifdef Py_LIMITED_API // We keep these definitions local, because they don't work in Python 2. -#define PyUnicode_GET_LENGTH(op) PyUnicode_GetLength((PyObject *)(op)) -#define PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar((PyObject *)(u), (i)) -#endif +# define PyUnicode_GET_LENGTH(op) PyUnicode_GetLength((PyObject *)(op)) +# define PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar((PyObject *)(u), (i)) +#endif // Py_LIMITED_API PyObject * _Pep_PrivateMangle(PyObject *self, PyObject *name) @@ -592,9 +688,9 @@ _Pep_PrivateMangle(PyObject *self, PyObject *name) * This function is modelled after _Py_Mangle, but is optimized * a little for our purpose. */ -#if PY_VERSION_HEX < 0X03000000 +#ifdef IS_PY2 const char *namestr = PyString_AsString(name); - if (namestr == NULL || namestr[0] != '_' || namestr[1] != '_') { + if (namestr == nullptr || namestr[0] != '_' || namestr[1] != '_') { Py_INCREF(name); return name; } @@ -619,9 +715,9 @@ _Pep_PrivateMangle(PyObject *self, PyObject *name) Py_INCREF(name); return name; } -#endif - Shiboken::AutoDecRef privateobj(PyObject_GetAttrString( - reinterpret_cast<PyObject *>(Py_TYPE(self)), "__name__")); +#endif // IS_PY2 + Shiboken::AutoDecRef privateobj(PyObject_GetAttr( + reinterpret_cast<PyObject *>(Py_TYPE(self)), Shiboken::PyMagicName::name())); #ifndef Py_LIMITED_API return _Py_Mangle(privateobj, name); #else @@ -640,7 +736,7 @@ _Pep_PrivateMangle(PyObject *self, PyObject *name) if (plen + nlen >= PY_SSIZE_T_MAX - 1) { PyErr_SetString(PyExc_OverflowError, "private identifier too large to be mangled"); - return NULL; + return nullptr; } size_t const amount = ipriv + 1 + plen + nlen; size_t const big_stack = 1000; @@ -658,7 +754,26 @@ _Pep_PrivateMangle(PyObject *self, PyObject *name) if (amount > big_stack) free(resbuf); return result; -#endif // Py_LIMITED_API +#endif // else Py_LIMITED_API +} + +/***************************************************************************** + * + * Runtime support for Python 3.8 incompatibilities + * + */ + +int PepRuntime_38_flag = 0; + +static void +init_PepRuntime() +{ + // We expect a string of the form "\d\.\d+\." + const char *version = Py_GetVersion(); + if (version[0] < '3') + return; + if (std::atoi(version + 2) >= 8) + PepRuntime_38_flag = 1; } /***************************************************************************** @@ -670,16 +785,18 @@ _Pep_PrivateMangle(PyObject *self, PyObject *name) void Pep384_Init() { - check_PyTypeObject_valid(); + init_PepRuntime(); #ifdef Py_LIMITED_API + check_PyTypeObject_valid(); Pep_GetVerboseFlag(); PepMethod_TypePtr = getMethodType(); PepFunction_TypePtr = getFunctionType(); PepStaticMethod_TypePtr = getStaticMethodType(); -#endif -#if PY_VERSION_HEX < 0x03000000 +#endif // Py_LIMITED_API + +#ifdef IS_PY2 PepMethodDescr_TypePtr = getMethodDescrType(); -#endif +#endif // IS_PY2 } } // extern "C" diff --git a/sources/shiboken2/libshiboken/pep384impl.h b/sources/shiboken2/libshiboken/pep384impl.h index 93f718988..541b0e775 100644 --- a/sources/shiboken2/libshiboken/pep384impl.h +++ b/sources/shiboken2/libshiboken/pep384impl.h @@ -40,8 +40,6 @@ #ifndef PEP384IMPL_H #define PEP384IMPL_H -#include "sbkpython.h" - extern "C" { @@ -88,12 +86,12 @@ typedef struct _typeobject { const char *tp_name; Py_ssize_t tp_basicsize; void *X03; // Py_ssize_t tp_itemsize; - void *X04; // destructor tp_dealloc; - void *X05; // printfunc tp_print; + destructor tp_dealloc; + void *X05; // Py_ssize_t tp_vectorcall_offset; void *X06; // getattrfunc tp_getattr; void *X07; // setattrfunc tp_setattr; void *X08; // PyAsyncMethods *tp_as_async; - void *X09; // reprfunc tp_repr; + reprfunc tp_repr; void *X10; // PyNumberMethods *tp_as_number; void *X11; // PySequenceMethods *tp_as_sequence; void *X12; // PyMappingMethods *tp_as_mapping; @@ -103,14 +101,14 @@ typedef struct _typeobject { void *X16; // getattrofunc tp_getattro; void *X17; // setattrofunc tp_setattro; void *X18; // PyBufferProcs *tp_as_buffer; - void *X19; // unsigned long tp_flags; + unsigned long tp_flags; void *X20; // const char *tp_doc; traverseproc tp_traverse; inquiry tp_clear; void *X23; // richcmpfunc tp_richcompare; Py_ssize_t tp_weaklistoffset; void *X25; // getiterfunc tp_iter; - void *X26; // iternextfunc tp_iternext; + iternextfunc tp_iternext; struct PyMethodDef *tp_methods; void *X28; // struct PyMemberDef *tp_members; struct PyGetSetDef *tp_getset; @@ -129,6 +127,13 @@ typedef struct _typeobject { } PyTypeObject; +#ifndef PyObject_IS_GC +/* Test if an object has a GC head */ +#define PyObject_IS_GC(o) \ + (PyType_IS_GC(Py_TYPE(o)) \ + && (Py_TYPE(o)->tp_is_gc == NULL || Py_TYPE(o)->tp_is_gc(o))) +#endif + // This was a macro error in the limited API from the beginning. // It was fixed in Python master, but did make it only in Python 3.8 . #define PY_ISSUE33738_SOLVED 0x03080000 @@ -197,17 +202,36 @@ LIBSHIBOKEN_API int Pep_GetVerboseFlag(void); * RESOLVED: unicodeobject.h * */ -#ifdef Py_LIMITED_API - -LIBSHIBOKEN_API char *_PepUnicode_AsString(PyObject *); +/////////////////////////////////////////////////////////////////////// +// +// PYSIDE-813: About The Length Of Unicode Objects +// ----------------------------------------------- +// +// In Python 2 and before Python 3.3, the macro PyUnicode_GET_SIZE +// worked fine and really like a macro. +// +// Meanwhile, the unicode objects have changed their layout very much, +// and the former cheap macro call has become a real function call +// that converts objects and needs PyMemory. +// +// That is not only inefficient, but also requires the GIL! +// This problem was visible by debug Python and qdatastream_test.py . +// It was found while fixing the refcount problem of PYSIDE-813 which +// needed a debug Python. +// + +// PyUnicode_GetSize is deprecated in favor of PyUnicode_GetLength. #if PY_VERSION_HEX < 0x03000000 -#define PyUnicode_GET_SIZE(op) PyUnicode_GetSize((PyObject *)(op)) +#define PepUnicode_GetLength(op) PyUnicode_GetSize((PyObject *)(op)) #else -// PyUnicode_GetSize is deprecated in favor of PyUnicode_GetLength -#define PyUnicode_GET_SIZE(op) PyUnicode_GetLength((PyObject *)(op)) +#define PepUnicode_GetLength(op) PyUnicode_GetLength((PyObject *)(op)) #endif +#ifdef Py_LIMITED_API + +LIBSHIBOKEN_API char *_PepUnicode_AsString(PyObject *); + #else #define _PepUnicode_AsString PyUnicode_AsUTF8 #endif @@ -255,6 +279,17 @@ LIBSHIBOKEN_API char *_PepUnicode_AsString(PyObject *); /***************************************************************************** * + * RESOLVED: dictobject.h + * + * PYSIDE-803, PYSIDE-813: We need PyDict_GetItemWithError in order to + * avoid the GIL. + */ +#if PY_VERSION_HEX < 0x03000000 +LIBSHIBOKEN_API PyObject *PyDict_GetItemWithError(PyObject *mp, PyObject *key); +#endif + +/***************************************************************************** + * * RESOLVED: methodobject.h * */ @@ -292,7 +327,7 @@ LIBSHIBOKEN_API PyObject *PyRun_String(const char *, int, PyObject *, PyObject * // But this is no problem as we check it's validity for every version. #define PYTHON_BUFFER_VERSION_COMPATIBLE (PY_VERSION_HEX >= 0x03030000 && \ - PY_VERSION_HEX < 0x0307FFFF) + PY_VERSION_HEX < 0x0308FFFF) #if !PYTHON_BUFFER_VERSION_COMPATIBLE # error Please check the buffer compatibility for this python version! #endif @@ -488,6 +523,29 @@ extern LIBSHIBOKEN_API PyTypeObject *PepMethodDescr_TypePtr; /***************************************************************************** * + * Newly introduced convenience functions + * + * This is not defined if Py_LIMITED_API is defined. + */ +#if PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) +LIBSHIBOKEN_API PyObject *PyImport_GetModule(PyObject *name); +#endif // PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) + +/***************************************************************************** + * + * Runtime support for Python 3.8 incompatibilities + * + */ + +#ifndef Py_TPFLAGS_METHOD_DESCRIPTOR +/* Objects behave like an unbound method */ +#define Py_TPFLAGS_METHOD_DESCRIPTOR (1UL << 17) +#endif + +extern LIBSHIBOKEN_API int PepRuntime_38_flag; + +/***************************************************************************** + * * Module Initialization * */ diff --git a/sources/shiboken2/libshiboken/pep384impl_doc.rst b/sources/shiboken2/libshiboken/pep384impl_doc.rst index 2f3b7ea97..d8ebdbe70 100644 --- a/sources/shiboken2/libshiboken/pep384impl_doc.rst +++ b/sources/shiboken2/libshiboken/pep384impl_doc.rst @@ -70,8 +70,10 @@ supported. We redefined it as macro ``Py_VerboseFlag`` which calls ``Pep_Verbose unicodeobject.h --------------- -The macro ``PyUnicode_GET_SIZE`` was redefined to call into ``PyUnicode_GetSize`` -for Python 2, and ``PyUnicode_GetLength`` for Python 3. +The macro ``PyUnicode_GET_SIZE`` was removed and replaced by ``PepUnicode_GetLength`` +which evaluates to ``PyUnicode_GetSize`` for Python 2 and ``PyUnicode_GetLength`` for Python 3. +Since Python 3.3, ``PyUnicode_GetSize`` would have the bad side effect of requiring the GIL! + Function ``_PyUnicode_AsString`` is unavailable and was replaced by a macro that calls ``_PepUnicode_AsString``. The implementation was a bit involved, and it would be better to change the code and replace this function. @@ -104,6 +106,16 @@ listobject.h function calls. +dictobject.h +------------ + +``PyDict_GetItem`` also exists in a ``PyDict_GetItemWithError`` version that does +not suppress errors. This suppression has the side effect of touching global +structures. This function exists in Python 2 only since Python 2.7.12 and has +a different name. We simply implemented the function. +Needed to avoid the GIL when accessing dictionaries. + + methodobject.h -------------- diff --git a/sources/shiboken2/libshiboken/qapp_macro.cpp b/sources/shiboken2/libshiboken/qapp_macro.cpp index 12af9613c..3ef3a51c6 100644 --- a/sources/shiboken2/libshiboken/qapp_macro.cpp +++ b/sources/shiboken2/libshiboken/qapp_macro.cpp @@ -51,206 +51,50 @@ extern "C" // // qApp is a macro in Qt5. In Python, we simulate that a little by a // variable that monitors Q*Application.instance(). -// This variable is also able to destroy the app by deleting qApp. +// This variable is also able to destroy the app by qApp.shutdown(). // -static int -qApp_module_index(PyObject *module) -{ - const char *name = PyModule_GetName(module); - int ret = 0; - - if (strcmp(name, "PySide2.QtCore") == 0) - ret = 1; - else if (strcmp(name, "PySide2.QtGui") == 0) - ret = 2; - else if (strcmp(name, "PySide2.QtWidgets") == 0) - ret = 3; - return ret; -} - -#define PYTHON_IS_PYTHON3 (PY_VERSION_HEX >= 0x03000000) -#define PYTHON_IS_PYTHON2 (!PYTHON_IS_PYTHON3) -#define Py_NONE_TYPE Py_TYPE(Py_None) - -#if PYTHON_IS_PYTHON3 -# define BRACE_OPEN { -# define BRACE_CLOSE } -#else -# define BRACE_OPEN -# define BRACE_CLOSE -#endif - -static SbkObject _Py_ChameleonQAppWrapper_Struct = { - BRACE_OPEN - _PyObject_EXTRA_INIT - 1, Py_NONE_TYPE - BRACE_CLOSE -}; static PyObject *qApp_var = nullptr; -static PyObject *qApp_content = reinterpret_cast<PyObject *>(&_Py_ChameleonQAppWrapper_Struct); -static PyObject *qApp_moduledicts[5] = {nullptr, nullptr, nullptr, nullptr, nullptr}; -static int qApp_var_ref = 0; -static int qApp_content_ref = 0; +static PyObject *qApp_content = nullptr; -static int -reset_qApp_var(void) +static PyObject * +monitor_qApp_var(PyObject *qApp) { - PyObject **mod_ptr; + static bool init_done; + static PyObject *builtins = PyEval_GetBuiltins(); - for (mod_ptr = qApp_moduledicts; *mod_ptr != nullptr; mod_ptr++) { - // We respect whatever the user may have set. - PyObject *existing = PyDict_GetItem(*mod_ptr, qApp_var); - if (existing == nullptr || Py_TYPE(existing) == Py_NONE_TYPE) { - if (PyDict_SetItem(*mod_ptr, qApp_var, qApp_content) < 0) - return -1; - } + if (!init_done) { + qApp_var = Py_BuildValue("s", "qApp"); + if (qApp_var == nullptr) + return nullptr; + // This is a borrowed reference + Py_INCREF(builtins); + init_done = true; } - return 0; + + if (PyDict_SetItem(builtins, qApp_var, qApp) < 0) + return nullptr; + qApp_content = qApp; + Py_INCREF(qApp); + return qApp; } -/* - * Note: - * The PYSIDE-585 problem was that shutdown is called one more often - * than Q*Application is created. We could special-case that last - * shutdown or add a refcount, initially, but actually it was easier - * and more intuitive in that context to make the refcount of - * qApp_content equal to the refcount of Py_None. - */ PyObject * -MakeSingletonQAppWrapper(PyTypeObject *type) +MakeQAppWrapper(PyTypeObject *type) { if (type == nullptr) - type = Py_NONE_TYPE; - if (!(type == Py_NONE_TYPE || Py_TYPE(qApp_content) == Py_NONE_TYPE)) { + type = Py_TYPE(Py_None); + if (!(type == Py_TYPE(Py_None) || Py_TYPE(qApp_content) == Py_TYPE(Py_None))) { const char *res_name = PepType_GetNameStr(Py_TYPE(qApp_content)); const char *type_name = PepType_GetNameStr(type); PyErr_Format(PyExc_RuntimeError, "Please destroy the %s singleton before" " creating a new %s instance.", res_name, type_name); return nullptr; } - if (reset_qApp_var() < 0) - return nullptr; - // always know the max of the refs - if (Py_REFCNT(qApp_var) > qApp_var_ref) - qApp_var_ref = Py_REFCNT(qApp_var); - if (Py_REFCNT(qApp_content) > qApp_content_ref) - qApp_content_ref = Py_REFCNT(qApp_content); - - if (Py_TYPE(qApp_content) != Py_NONE_TYPE) { - // Remove the "_" variable which might hold a reference to qApp. - Shiboken::AutoDecRef pymain(PyImport_ImportModule("__main__")); - if (pymain.object() && PyObject_HasAttrString(pymain.object(), "_")) - PyObject_DelAttrString(pymain.object(), "_"); - Py_REFCNT(qApp_var) = 1; // fuse is armed... - } - if (type == Py_NONE_TYPE) { - // Debug mode showed that we need to do more than just remove the - // reference. To keep everything in the right order, it is easiest - // to do a full shutdown, using QtCore.__moduleShutdown(). - // restore the "None-state" - PyObject *__moduleShutdown = PyDict_GetItemString(qApp_moduledicts[1], - "__moduleShutdown"); - // PYSIDE-585: It was crucial to update the refcounts *before* - // calling the shutdown. - Py_TYPE(qApp_content) = Py_NONE_TYPE; - Py_REFCNT(qApp_var) = qApp_var_ref; - Py_REFCNT(qApp_content) = Py_REFCNT(Py_None); - if (__moduleShutdown != nullptr) - Py_XDECREF(PyObject_CallFunction(__moduleShutdown, const_cast<char *>("()"))); - } - else - (void)PyObject_INIT(qApp_content, type); - Py_INCREF(qApp_content); - return qApp_content; -} - -#if PYTHON_IS_PYTHON2 - -// Install support in Py_NONE_TYPE for Python 2: 'bool(qApp) == False'. -static int -none_bool(PyObject *v) -{ - return 0; -} - -static PyNumberMethods none_as_number = { - nullptr, /* nb_add */ - nullptr, /* nb_subtract */ - nullptr, /* nb_multiply */ - nullptr, /* nb_divide */ - nullptr, /* nb_remainder */ - nullptr, /* nb_divmod */ - nullptr, /* nb_power */ - nullptr, /* nb_negative */ - nullptr, /* nb_positive */ - nullptr, /* nb_absolute */ - reinterpret_cast<inquiry>(none_bool), /* nb_nonzero */ -}; - -#endif - -static int -setup_qApp_var(PyObject *module) -{ - int module_index; - static int init_done = 0; - - if (!init_done) { -#if PYTHON_IS_PYTHON2 - Py_NONE_TYPE->tp_as_number = &none_as_number; -#endif - qApp_var = Py_BuildValue("s", "qApp"); - if (qApp_var == nullptr) - return -1; - // This is a borrowed reference - qApp_moduledicts[0] = PyEval_GetBuiltins(); - Py_INCREF(qApp_moduledicts[0]); - init_done = 1; - } - - // Initialize qApp. We insert it into __dict__ for "import *" and also - // into __builtins__, to let it appear like a real macro. - module_index = qApp_module_index(module); - if (module_index) { - // This line gets a borrowed reference - qApp_moduledicts[module_index] = PyModule_GetDict(module); - Py_INCREF(qApp_moduledicts[module_index]); - if (reset_qApp_var() < 0) - return -1; - } - return 0; + PyObject *self = type != Py_TYPE(Py_None) ? PyObject_New(PyObject, type) : Py_None; + return monitor_qApp_var(self); } -void -NotifyModuleForQApp(PyObject *module, void *qApp) -{ - setup_qApp_var(module); - /* - * PYSIDE-571: Check if an QApplication instance exists before the import. - * This happens in scriptableapplication and application_test.py . - * - * Crucial Observation - * =================== - * - * A Q*Application object from C++ does not have a wrapper or constructor - * like instances created by Python. It makes no sense to support - * deletion or special features like qApp resurrection. - * - * Therefore, the implementation is very simple and just redirects the - * qApp_contents variable and assigns the instance, instead of vice-versa. - */ - PyObject *coreDict = qApp_moduledicts[1]; - if (qApp != nullptr && coreDict != nullptr) { - PyObject *coreApp = PyDict_GetItemString(coreDict, "QCoreApplication"); - if (coreApp != nullptr) { - qApp_content = PyObject_CallMethod(coreApp, "instance", ""); - reset_qApp_var(); - } - } -} - - } //extern "C" // end of module diff --git a/sources/shiboken2/libshiboken/qapp_macro.h b/sources/shiboken2/libshiboken/qapp_macro.h index be45241de..9abd17c17 100644 --- a/sources/shiboken2/libshiboken/qapp_macro.h +++ b/sources/shiboken2/libshiboken/qapp_macro.h @@ -45,8 +45,7 @@ extern "C" { -LIBSHIBOKEN_API PyObject *MakeSingletonQAppWrapper(PyTypeObject *type); -LIBSHIBOKEN_API void NotifyModuleForQApp(PyObject *module, void *qApp); +LIBSHIBOKEN_API PyObject *MakeQAppWrapper(PyTypeObject *type); } // extern "C" diff --git a/sources/shiboken2/libshiboken/sbkenum.cpp b/sources/shiboken2/libshiboken/sbkenum.cpp index 2dc785884..36f2f48f9 100644 --- a/sources/shiboken2/libshiboken/sbkenum.cpp +++ b/sources/shiboken2/libshiboken/sbkenum.cpp @@ -39,6 +39,8 @@ #include "sbkenum.h" #include "sbkstring.h" +#include "sbkstaticstrings.h" +#include "sbkstaticstrings_p.h" #include "sbkconverter.h" #include "basewrapper.h" #include "sbkdbg.h" @@ -105,16 +107,18 @@ static PyObject *SbkEnum_tp_new(PyTypeObject *type, PyObject *args, PyObject *) if (!self) return nullptr; self->ob_value = itemValue; - PyObject *item = Shiboken::Enum::getEnumItemFromValue(type, itemValue); - if (item) { - self->ob_name = SbkEnumObject_name(item, nullptr); - Py_XDECREF(item); - } else { - self->ob_name = nullptr; - } + Shiboken::AutoDecRef item(Shiboken::Enum::getEnumItemFromValue(type, itemValue)); + self->ob_name = item.object() ? SbkEnumObject_name(item, nullptr) : nullptr; return reinterpret_cast<PyObject *>(self); } +void enum_object_dealloc(PyObject *ob) +{ + auto self = reinterpret_cast<SbkEnumObject *>(ob); + Py_XDECREF(self->ob_name); + Sbk_object_dealloc(ob); +} + static PyObject *enum_op(enum_func f, PyObject *a, PyObject *b) { PyObject *valA = a; PyObject *valB = b; @@ -258,6 +262,23 @@ static PyGetSetDef SbkEnumGetSetList[] = { {nullptr, nullptr, nullptr, nullptr, nullptr} // Sentinel }; +#if PY_VERSION_HEX < 0x03000000 + +static PyObject *SbkEnumType_repr(PyObject *type) +{ + Shiboken::AutoDecRef mod(PyObject_GetAttr(type, Shiboken::PyMagicName::module())); + if (mod.isNull()) + return nullptr; + Shiboken::AutoDecRef name(PyObject_GetAttr(type, Shiboken::PyMagicName::qualname())); + if (name.isNull()) + return nullptr; + return PyString_FromFormat("<class '%s.%s'>", + PyString_AS_STRING(mod.object()), + PyString_AS_STRING(name.object())); +} + +#endif // PY_VERSION_HEX < 0x03000000 + static void SbkEnumTypeDealloc(PyObject *pyObj); static PyObject *SbkEnumTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds); @@ -285,10 +306,13 @@ static PyType_Slot SbkEnumType_Type_slots[] = { {Py_tp_alloc, (void *)PyType_GenericAlloc}, {Py_tp_new, (void *)SbkEnumTypeTpNew}, {Py_tp_free, (void *)PyObject_GC_Del}, +#if PY_VERSION_HEX < 0x03000000 + {Py_tp_repr, (void *)SbkEnumType_repr}, +#endif {0, nullptr} }; static PyType_Spec SbkEnumType_Type_spec = { - "Shiboken.EnumType", + "1:Shiboken.EnumType", 0, // filled in later sizeof(PyMemberDef), Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_CHECKTYPES, @@ -302,7 +326,7 @@ PyTypeObject *SbkEnumType_TypeF(void) if (!type) { SbkEnumType_Type_spec.basicsize = PepHeapType_SIZE + sizeof(SbkEnumTypePrivate); - type = (PyTypeObject *)PyType_FromSpec(&SbkEnumType_Type_spec); + type = reinterpret_cast<PyTypeObject *>(SbkType_FromSpec(&SbkEnumType_Type_spec)); } return type; } @@ -321,6 +345,11 @@ void SbkEnumTypeDealloc(PyObject *pyObj) #ifndef Py_LIMITED_API Py_TRASHCAN_SAFE_END(pyObj); #endif + if (PepRuntime_38_flag) { + // PYSIDE-939: Handling references correctly. + // This was not needed before Python 3.8 (Python issue 35810) + Py_DECREF(Py_TYPE(pyObj)); + } } PyObject *SbkEnumTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds) @@ -334,6 +363,107 @@ PyObject *SbkEnumTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwd } // extern "C" +/////////////////////////////////////////////////////////////// +// +// PYSIDE-15: Pickling Support for Qt Enum objects +// This works very well and fixes the issue. +// +extern "C" { + +static void init_enum(); // forward + +static PyObject *enum_unpickler = nullptr; + +// Pickling: reduce the Qt Enum object +static PyObject *enum___reduce__(PyObject *obj) +{ + init_enum(); + return Py_BuildValue("O(Ni)", + enum_unpickler, + Py_BuildValue("s", Py_TYPE(obj)->tp_name), + PyInt_AS_LONG(obj)); +} + +} // extern "C" + +namespace Shiboken { namespace Enum { + +// Unpickling: rebuild the Qt Enum object +PyObject *unpickleEnum(PyObject *enum_class_name, PyObject *value) +{ + Shiboken::AutoDecRef parts(PyObject_CallMethod(enum_class_name, + const_cast<char *>("split"), const_cast<char *>("s"), ".")); + if (parts.isNull()) + return nullptr; + PyObject *top_name = PyList_GetItem(parts, 0); // borrowed ref + if (top_name == nullptr) + return nullptr; + PyObject *module = PyImport_GetModule(top_name); + if (module == nullptr) { + PyErr_Format(PyExc_ImportError, "could not import module %.200s", + Shiboken::String::toCString(top_name)); + return nullptr; + } + Shiboken::AutoDecRef cur_thing(module); + int len = PyList_Size(parts); + for (int idx = 1; idx < len; ++idx) { + PyObject *name = PyList_GetItem(parts, idx); // borrowed ref + PyObject *thing = PyObject_GetAttr(cur_thing, name); + if (thing == nullptr) { + PyErr_Format(PyExc_ImportError, "could not import Qt Enum type %.200s", + Shiboken::String::toCString(enum_class_name)); + return nullptr; + } + cur_thing.reset(thing); + } + PyObject *klass = cur_thing; + return PyObject_CallFunctionObjArgs(klass, value, nullptr); +} + +} // namespace Enum +} // namespace Shiboken + +extern "C" { + +// Initialization +static bool _init_enum() +{ + static PyObject *shiboken_name = Py_BuildValue("s", "shiboken2"); + if (shiboken_name == nullptr) + return false; + Shiboken::AutoDecRef shibo(PyImport_GetModule(shiboken_name)); + if (shibo.isNull()) + return false; + Shiboken::AutoDecRef sub(PyObject_GetAttr(shibo, shiboken_name)); + PyObject *mod = sub.object(); + if (mod == nullptr) { + // We are in the build dir and already in shiboken. + PyErr_Clear(); + mod = shibo.object(); + } + enum_unpickler = PyObject_GetAttrString(mod, "_unpickle_enum"); + if (enum_unpickler == nullptr) + return false; + return true; +} + +static void init_enum() +{ + if (!(enum_unpickler || _init_enum())) + Py_FatalError("could not load enum pickling helper function"); +} + +static PyMethodDef SbkEnumObject_Methods[] = { + {const_cast<char *>("__reduce__"), reinterpret_cast<PyCFunction>(enum___reduce__), + METH_NOARGS, nullptr}, + {nullptr, nullptr, 0, nullptr} // Sentinel +}; + +} // extern "C" + +// +/////////////////////////////////////////////////////////////// + namespace Shiboken { class DeclaredEnumTypes @@ -364,12 +494,12 @@ PyObject *getEnumItemFromValue(PyTypeObject *enumType, long itemValue) { PyObject *key, *value; Py_ssize_t pos = 0; - PyObject *values = PyDict_GetItemString(enumType->tp_dict, const_cast<char *>("values")); + PyObject *values = PyDict_GetItem(enumType->tp_dict, Shiboken::PyName::values()); while (PyDict_Next(values, &pos, &key, &value)) { auto *obj = reinterpret_cast<SbkEnumObject *>(value); if (obj->ob_value == itemValue) { - Py_INCREF(obj); + Py_INCREF(value); return value; } } @@ -381,19 +511,25 @@ static PyTypeObject *createEnum(const char *fullName, const char *cppName, PyTypeObject *flagsType) { PyTypeObject *enumType = newTypeWithName(fullName, cppName, flagsType); - if (PyType_Ready(enumType) < 0) + if (PyType_Ready(enumType) < 0) { + Py_XDECREF(enumType); return nullptr; + } return enumType; } PyTypeObject *createGlobalEnum(PyObject *module, const char *name, const char *fullName, const char *cppName, PyTypeObject *flagsType) { PyTypeObject *enumType = createEnum(fullName, cppName, name, flagsType); - if (enumType && PyModule_AddObject(module, name, reinterpret_cast<PyObject *>(enumType)) < 0) + if (enumType && PyModule_AddObject(module, name, reinterpret_cast<PyObject *>(enumType)) < 0) { + Py_DECREF(enumType); return nullptr; + } if (flagsType && PyModule_AddObject(module, PepType_GetNameStr(flagsType), - reinterpret_cast<PyObject *>(flagsType)) < 0) + reinterpret_cast<PyObject *>(flagsType)) < 0) { + Py_DECREF(enumType); return nullptr; + } return enumType; } @@ -401,58 +537,48 @@ PyTypeObject *createScopedEnum(SbkObjectType *scope, const char *name, const cha { PyTypeObject *enumType = createEnum(fullName, cppName, name, flagsType); if (enumType && PyDict_SetItemString(reinterpret_cast<PyTypeObject *>(scope)->tp_dict, name, - reinterpret_cast<PyObject *>(enumType)) < 0) + reinterpret_cast<PyObject *>(enumType)) < 0) { + Py_DECREF(enumType); return nullptr; + } if (flagsType && PyDict_SetItemString(reinterpret_cast<PyTypeObject *>(scope)->tp_dict, PepType_GetNameStr(flagsType), - reinterpret_cast<PyObject *>(flagsType)) < 0) + reinterpret_cast<PyObject *>(flagsType)) < 0) { + Py_DECREF(enumType); return nullptr; + } return enumType; } static PyObject *createEnumItem(PyTypeObject *enumType, const char *itemName, long itemValue) { - char mangled[20]; - if (strcmp(itemName, "None") == 0 - || strcmp(itemName, "False") == 0 || strcmp(itemName, "True") == 0) { - strcpy(mangled, itemName); - strcat(mangled, "_"); - itemName = mangled; - } PyObject *enumItem = newItem(enumType, itemValue, itemName); - if (PyDict_SetItemString(enumType->tp_dict, itemName, enumItem) < 0) + if (PyDict_SetItemString(enumType->tp_dict, itemName, enumItem) < 0) { + Py_DECREF(enumItem); return nullptr; - Py_DECREF(enumItem); + } return enumItem; } bool createGlobalEnumItem(PyTypeObject *enumType, PyObject *module, const char *itemName, long itemValue) { PyObject *enumItem = createEnumItem(enumType, itemName, itemValue); - if (enumItem) { - if (PyModule_AddObject(module, itemName, enumItem) < 0) - return false; - // @TODO This Py_DECREF causes crashes on exit with a debug Python interpreter, essentially - // causing a use-after-free in the GC. This is now commented out to cause a memory leak - // instead of a crash. Proper memory management of Enum types and items should be - // implemented. See PYSIDE-488. This will require proper allocation and deallocation of - // the underlying Enum PyHeapType, which is currently just deallocated at application exit. - // Py_DECREF(enumItem); - return true; - } - return false; + if (!enumItem) + return false; + int ok = PyModule_AddObject(module, itemName, enumItem); + Py_DECREF(enumItem); + return ok >= 0; } bool createScopedEnumItem(PyTypeObject *enumType, PyTypeObject *scope, const char *itemName, long itemValue) { - if (PyObject *enumItem = createEnumItem(enumType, itemName, itemValue)) { - if (PyDict_SetItemString(reinterpret_cast<PyTypeObject *>(scope)->tp_dict, itemName, enumItem) < 0) - return false; - Py_DECREF(enumItem); - return true; - } - return false; + PyObject *enumItem = createEnumItem(enumType, itemName, itemValue); + if (!enumItem) + return false; + int ok = PyDict_SetItemString(reinterpret_cast<PyTypeObject *>(scope)->tp_dict, itemName, enumItem); + Py_DECREF(enumItem); + return ok >= 0; } bool createScopedEnumItem(PyTypeObject *enumType, SbkObjectType *scope, const char *itemName, long itemValue) @@ -482,11 +608,11 @@ newItem(PyTypeObject *enumType, long itemValue, const char *itemName) enumObj->ob_value = itemValue; if (newValue) { - PyObject *values = PyDict_GetItemString(enumType->tp_dict, const_cast<char *>("values")); + PyObject *values = PyDict_GetItem(enumType->tp_dict, Shiboken::PyName::values()); if (!values) { values = PyDict_New(); - PyDict_SetItemString(enumType->tp_dict, const_cast<char *>("values"), values); - Py_DECREF(values); // ^ values still alive, because setitemstring incref it + PyDict_SetItem(enumType->tp_dict, Shiboken::PyName::values(), values); + Py_DECREF(values); // ^ values still alive, because setitem increfs it } PyDict_SetItemString(values, itemName, reinterpret_cast<PyObject *>(enumObj)); } @@ -498,6 +624,7 @@ static PyType_Slot SbkNewType_slots[] = { {Py_tp_repr, (void *)SbkEnumObject_repr}, {Py_tp_str, (void *)SbkEnumObject_repr}, {Py_tp_getset, (void *)SbkEnumGetSetList}, + {Py_tp_methods, (void *)SbkEnumObject_Methods}, {Py_tp_new, (void *)SbkEnum_tp_new}, {Py_nb_add, (void *)enum_add}, {Py_nb_subtract, (void *)enum_subtract}, @@ -519,7 +646,7 @@ static PyType_Slot SbkNewType_slots[] = { {Py_nb_index, (void *)enum_int}, {Py_tp_richcompare, (void *)enum_richcompare}, {Py_tp_hash, (void *)enum_hash}, - {Py_tp_dealloc, (void *)object_dealloc}, + {Py_tp_dealloc, (void *)enum_object_dealloc}, {0, nullptr} }; static PyType_Spec SbkNewType_spec = { @@ -591,7 +718,7 @@ newTypeWithName(const char *name, const char *cppName, PyTypeObject *numbers_fromFlag) { - // Careful: PyType_FromSpec does not allocate the string. + // Careful: SbkType_FromSpec does not allocate the string. PyType_Slot newslots[99] = {}; // enough but not too big for the stack auto *newspec = new PyType_Spec; newspec->name = strdup(name); @@ -608,9 +735,8 @@ newTypeWithName(const char *name, if (numbers_fromFlag) copyNumberMethods(numbers_fromFlag, newslots, &idx); newspec->slots = newslots; - auto *type = reinterpret_cast<PyTypeObject *>(PyType_FromSpec(newspec)); + auto *type = reinterpret_cast<PyTypeObject *>(SbkType_FromSpec(newspec)); Py_TYPE(type) = SbkEnumType_TypeF(); - Py_INCREF(Py_TYPE(type)); auto *enumType = reinterpret_cast<SbkEnumType *>(type); PepType_SETP(enumType)->cppName = cppName; @@ -656,7 +782,7 @@ DeclaredEnumTypes::DeclaredEnumTypes() = default; DeclaredEnumTypes::~DeclaredEnumTypes() { /* - * PYSIDE-595: This was "delete *it;" before introducing 'PyType_FromSpec'. + * PYSIDE-595: This was "delete *it;" before introducing 'SbkType_FromSpec'. * XXX what should I do now? * Refcounts in tests are 30 or 0 at end. * When I add the default tp_dealloc, we get negative refcounts! diff --git a/sources/shiboken2/libshiboken/sbkenum.h b/sources/shiboken2/libshiboken/sbkenum.h index 759d72636..c294c17d9 100644 --- a/sources/shiboken2/libshiboken/sbkenum.h +++ b/sources/shiboken2/libshiboken/sbkenum.h @@ -114,6 +114,8 @@ namespace Enum LIBSHIBOKEN_API void setTypeConverter(PyTypeObject *enumType, SbkConverter *converter); /// Returns the converter assigned to the enum \p type. LIBSHIBOKEN_API SbkConverter *getTypeConverter(PyTypeObject *enumType); + + LIBSHIBOKEN_API PyObject *unpickleEnum(PyObject *, PyObject *); } } // namespace Shiboken diff --git a/sources/shiboken2/libshiboken/sbkpython.h b/sources/shiboken2/libshiboken/sbkpython.h index f06b0b19e..9dd1e712e 100644 --- a/sources/shiboken2/libshiboken/sbkpython.h +++ b/sources/shiboken2/libshiboken/sbkpython.h @@ -72,6 +72,7 @@ extern "C" { // Now we have the usual variables from Python.h . # include "python25compat.h" # include "shibokenmacros.h" +// "pep384impl.h" may nowhere be included but in this file. # include "pep384impl.h" # include "typespec.h" # pragma pop_macro("slots") @@ -98,6 +99,7 @@ extern "C" { // Now we have the usual variables from Python.h . # include "python25compat.h" # include "shibokenmacros.h" +// "pep384impl.h" may nowhere be included but in this file. # include "pep384impl.h" # include "typespec.h" #endif diff --git a/sources/shiboken2/libshiboken/sbkstaticstrings.cpp b/sources/shiboken2/libshiboken/sbkstaticstrings.cpp new file mode 100644 index 000000000..c19665176 --- /dev/null +++ b/sources/shiboken2/libshiboken/sbkstaticstrings.cpp @@ -0,0 +1,96 @@ +/**************************************************************************** +** +** Copyright (C) 2019 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt for Python. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "sbkstaticstrings.h" +#include "sbkstaticstrings_p.h" +#include "sbkstring.h" + +#define STATIC_STRING_IMPL(funcName, value) \ +PyObject *funcName() \ +{ \ + static PyObject *const s = Shiboken::String::createStaticString(value); \ + return s; \ +} + +namespace Shiboken +{ +namespace PyName { +// exported: +STATIC_STRING_IMPL(dumps, "dumps") +STATIC_STRING_IMPL(loads, "loads") +STATIC_STRING_IMPL(result, "result") +STATIC_STRING_IMPL(values, "values") + +// Internal: +STATIC_STRING_IMPL(classmethod, "classmethod") +STATIC_STRING_IMPL(compile, "compile"); +STATIC_STRING_IMPL(function, "function") +STATIC_STRING_IMPL(marshal, "marshal") +STATIC_STRING_IMPL(method, "method") +STATIC_STRING_IMPL(mro, "mro") +STATIC_STRING_IMPL(overload, "overload") +STATIC_STRING_IMPL(staticmethod, "staticmethod") +} // namespace PyName + +namespace PyMagicName { +// exported: +STATIC_STRING_IMPL(class_, "__class__") +STATIC_STRING_IMPL(ecf, "__ecf__") +STATIC_STRING_IMPL(file, "__file__") +STATIC_STRING_IMPL(module, "__module__") +STATIC_STRING_IMPL(name, "__name__") +STATIC_STRING_IMPL(qualname, "__qualname__") + +// Internal: +STATIC_STRING_IMPL(base, "__base__") +STATIC_STRING_IMPL(bases, "__bases__") +STATIC_STRING_IMPL(builtins, "__builtins__") +STATIC_STRING_IMPL(code, "__code__") +STATIC_STRING_IMPL(dictoffset, "__dictoffset__") +STATIC_STRING_IMPL(func, "__func__") +STATIC_STRING_IMPL(func_kind, "__func_kind__") +STATIC_STRING_IMPL(iter, "__iter__") +STATIC_STRING_IMPL(mro, "__mro__") +STATIC_STRING_IMPL(new_, "__new__") +STATIC_STRING_IMPL(objclass, "__objclass__") +STATIC_STRING_IMPL(self, "__self__") +STATIC_STRING_IMPL(signature, "__signature__") +STATIC_STRING_IMPL(weakrefoffset, "__weakrefoffset__") +} // namespace PyMagicName +} // namespace Shiboken diff --git a/sources/shiboken2/libshiboken/sbkstaticstrings.h b/sources/shiboken2/libshiboken/sbkstaticstrings.h new file mode 100644 index 000000000..07d6cc60a --- /dev/null +++ b/sources/shiboken2/libshiboken/sbkstaticstrings.h @@ -0,0 +1,68 @@ +/**************************************************************************** +** +** Copyright (C) 2019 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt for Python. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef SBKSTATICSTRINGS_H +#define SBKSTATICSTRINGS_H + +#include "sbkpython.h" +#include "shibokenmacros.h" + +namespace Shiboken +{ +// Some often-used strings +namespace PyName +{ +LIBSHIBOKEN_API PyObject *dumps(); +LIBSHIBOKEN_API PyObject *loads(); +LIBSHIBOKEN_API PyObject *result(); +LIBSHIBOKEN_API PyObject *values(); +} // namespace PyName + +namespace PyMagicName +{ +LIBSHIBOKEN_API PyObject *class_(); +LIBSHIBOKEN_API PyObject *ecf(); +LIBSHIBOKEN_API PyObject *file(); +LIBSHIBOKEN_API PyObject *module(); +LIBSHIBOKEN_API PyObject *name(); +LIBSHIBOKEN_API PyObject *qualname(); +} // namespace PyMagicName +} // namespace Shiboken + +#endif // SBKSTATICSTRINGS_H diff --git a/sources/shiboken2/libshiboken/sbkstaticstrings_p.h b/sources/shiboken2/libshiboken/sbkstaticstrings_p.h new file mode 100644 index 000000000..12c11376f --- /dev/null +++ b/sources/shiboken2/libshiboken/sbkstaticstrings_p.h @@ -0,0 +1,74 @@ +/**************************************************************************** +** +** Copyright (C) 2019 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt for Python. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "sbkpython.h" +#include "shibokenmacros.h" + +namespace Shiboken +{ +namespace PyName +{ +PyObject *classmethod(); +PyObject *compile(); +PyObject *function(); +PyObject *marshal(); +PyObject *method(); +PyObject *mro(); +PyObject *overload(); +PyObject *staticmethod(); +} // namespace PyName +namespace PyMagicName +{ +PyObject *base(); +PyObject *bases(); +PyObject *builtins(); +PyObject *code(); +PyObject *dictoffset(); +PyObject *func(); +PyObject *func_kind(); +PyObject *iter(); +PyObject *module(); +PyObject *mro(); +PyObject *new_(); +PyObject *objclass(); +PyObject *self(); +PyObject *signature(); +PyObject *weakrefoffset(); +} // namespace PyMagicName +} // namespace Shiboken diff --git a/sources/shiboken2/libshiboken/sbkstring.cpp b/sources/shiboken2/libshiboken/sbkstring.cpp index 9ba5be281..38bb105d0 100644 --- a/sources/shiboken2/libshiboken/sbkstring.cpp +++ b/sources/shiboken2/libshiboken/sbkstring.cpp @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2018 The Qt Company Ltd. +** Copyright (C) 2019 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of Qt for Python. @@ -38,14 +38,23 @@ ****************************************************************************/ #include "sbkstring.h" +#include "sbkstaticstrings_p.h" #include "autodecref.h" +#include <vector> + namespace Shiboken { namespace String { +// PYSIDE-795: Redirecting PySequence to Iterable +bool checkIterable(PyObject *obj) +{ + return PyObject_HasAttr(obj, Shiboken::PyMagicName::iter()); +} + bool checkType(PyTypeObject *type) { return type == &PyUnicode_Type @@ -193,13 +202,71 @@ Py_ssize_t len(PyObject *str) return 0; if (PyUnicode_Check(str)) - return PyUnicode_GET_SIZE(str); + return PepUnicode_GetLength(str); if (PyBytes_Check(str)) return PyBytes_GET_SIZE(str); return 0; } -} // namespace String +/////////////////////////////////////////////////////////////////////// +// +// Implementation of efficient Python strings +// ------------------------------------------ +// +// Instead of repetitively executing +// +// PyObject *attr = PyObject_GetAttrString(obj, "__name__"); +// +// a helper of the form +// +// PyObject *name() +// { +// static PyObject *const s = Shiboken::String::createStaticString("__name__"); +// return result; +// } +// +// can now be implemented, which registers the string into a static set avoiding +// repetitive string creation. The resulting code looks like: +// +// PyObject *attr = PyObject_GetAttr(obj, name()); +// +// Missing: +// There is no finalization for the string structures, yet. +// But this is a global fault in shiboken. We are missing a true +// finalization like in all other modules. + +using StaticStrings = std::vector<PyObject *>; +static StaticStrings &staticStrings() +{ + static StaticStrings result; + return result; +} + +PyObject *createStaticString(const char *str) +{ +#if PY_VERSION_HEX >= 0x03000000 + PyObject *result = PyUnicode_InternFromString(str); +#else + PyObject *result = PyString_InternFromString(str); +#endif + if (result == nullptr) { + // This error is never checked, but also very unlikely. Report and exit. + PyErr_Print(); + Py_FatalError("unexpected error in createStaticString()"); + } + staticStrings().push_back(result); + return result; +} + +void finalizeStaticStrings() // Currently unused +{ + auto &list = staticStrings(); + for (auto s : list) + Py_DECREF(s); + list.clear(); +} + +} // namespace String } // namespace Shiboken diff --git a/sources/shiboken2/libshiboken/sbkstring.h b/sources/shiboken2/libshiboken/sbkstring.h index 7f434e1b9..84d7768c5 100644 --- a/sources/shiboken2/libshiboken/sbkstring.h +++ b/sources/shiboken2/libshiboken/sbkstring.h @@ -1,6 +1,6 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2019 The Qt Company Ltd. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of Qt for Python. @@ -43,17 +43,12 @@ #include "sbkpython.h" #include "shibokenmacros.h" -#if PY_MAJOR_VERSION >= 3 - #define SBK_BYTES_NAME "bytes" -#else - #define SBK_BYTES_NAME "str" -#endif - namespace Shiboken { namespace String { LIBSHIBOKEN_API bool check(PyObject *obj); + LIBSHIBOKEN_API bool checkIterable(PyObject *obj); LIBSHIBOKEN_API bool checkType(PyTypeObject *obj); LIBSHIBOKEN_API bool checkChar(PyObject *obj); LIBSHIBOKEN_API bool isConvertible(PyObject *obj); @@ -65,6 +60,7 @@ namespace String LIBSHIBOKEN_API PyObject *fromStringAndSize(const char *str, Py_ssize_t size); LIBSHIBOKEN_API int compare(PyObject *val1, const char *val2); LIBSHIBOKEN_API Py_ssize_t len(PyObject *str); + LIBSHIBOKEN_API PyObject *createStaticString(const char *str); } // namespace String } // namespace Shiboken diff --git a/sources/shiboken2/libshiboken/shiboken.h b/sources/shiboken2/libshiboken/shiboken.h index 1356670aa..0d2d6b0a6 100644 --- a/sources/shiboken2/libshiboken/shiboken.h +++ b/sources/shiboken2/libshiboken/shiboken.h @@ -52,6 +52,7 @@ #include "sbkenum.h" #include "sbkmodule.h" #include "sbkstring.h" +#include "sbkstaticstrings.h" #include "shibokenmacros.h" #include "shibokenbuffer.h" diff --git a/sources/shiboken2/libshiboken/shibokenbuffer.cpp b/sources/shiboken2/libshiboken/shibokenbuffer.cpp index 330470183..dd6e46320 100644 --- a/sources/shiboken2/libshiboken/shibokenbuffer.cpp +++ b/sources/shiboken2/libshiboken/shibokenbuffer.cpp @@ -43,7 +43,11 @@ bool Shiboken::Buffer::checkType(PyObject *pyObj) { +#ifdef IS_PY3K + return PyObject_CheckBuffer(pyObj) != 0; +#else return PyObject_CheckReadBuffer(pyObj) != 0; +#endif } void *Shiboken::Buffer::getPointer(PyObject *pyObj, Py_ssize_t *size) diff --git a/sources/shiboken2/libshiboken/signature.cpp b/sources/shiboken2/libshiboken/signature.cpp index 543ccac5f..ba1fc119e 100644 --- a/sources/shiboken2/libshiboken/signature.cpp +++ b/sources/shiboken2/libshiboken/signature.cpp @@ -39,6 +39,9 @@ #include "basewrapper.h" #include "autodecref.h" +#include "sbkstring.h" +#include "sbkstaticstrings.h" +#include "sbkstaticstrings_p.h" extern "C" { @@ -51,11 +54,11 @@ extern "C" // These constants were needed in former versions of the module: #define PYTHON_HAS_QUALNAME (PY_VERSION_HEX >= 0x03030000) -#define PYTHON_HAS_UNICODE (PY_VERSION_HEX >= 0x03000000) #define PYTHON_HAS_WEAKREF_PYCFUNCTION (PY_VERSION_HEX >= 0x030500A0) #define PYTHON_IS_PYTHON3 (PY_VERSION_HEX >= 0x03000000) #define PYTHON_HAS_KEYWORDONLY (PYTHON_IS_PYTHON3) #define PYTHON_USES_PERCENT_V_FORMAT (PYTHON_IS_PYTHON3) +#define PYTHON_USES_D_COMMON (PY_VERSION_HEX >= 0x03020000) #define PYTHON_HAS_DESCR_REDUCE (PY_VERSION_HEX >= 0x03040000) #define PYTHON_HAS_METH_REDUCE (PYTHON_HAS_DESCR_REDUCE) #define PYTHON_NEEDS_ITERATOR_FLAG (!PYTHON_IS_PYTHON3) @@ -64,7 +67,7 @@ extern "C" #define PYTHON_HAS_INT_AND_LONG (!PYTHON_IS_PYTHON3) // These constants are still in use: -#define PYTHON_USES_D_COMMON (PY_VERSION_HEX >= 0x03020000) +#define PYTHON_USES_UNICODE (PY_VERSION_HEX >= 0x03000000) typedef struct safe_globals_struc { // init part 1: get arg_dict @@ -84,11 +87,11 @@ static safe_globals pyside_globals = nullptr; static PyObject *GetTypeKey(PyObject *ob); -static PyObject *GetSignature_Function(PyObject *, const char *); -static PyObject *GetSignature_TypeMod(PyObject *, const char *); -static PyObject *GetSignature_Wrapper(PyObject *, const char *); +static PyObject *GetSignature_Function(PyObject *, PyObject *); +static PyObject *GetSignature_TypeMod(PyObject *, PyObject *); +static PyObject *GetSignature_Wrapper(PyObject *, PyObject *); static PyObject *get_signature(PyObject *self, PyObject *args); -static PyObject *get_signature_intern(PyObject *ob, const char *modifier); +static PyObject *get_signature_intern(PyObject *ob, PyObject *modifier); static PyObject *PySide_BuildSignatureProps(PyObject *class_mod); @@ -108,10 +111,10 @@ CreateSignature(PyObject *props, PyObject *key) const_cast<char *>("(OO)"), props, key); } -typedef PyObject *(*signaturefunc)(PyObject *, const char *); +typedef PyObject *(*signaturefunc)(PyObject *, PyObject *); static PyObject * -_get_written_signature(signaturefunc sf, PyObject *ob, const char *modifier) +_get_written_signature(signaturefunc sf, PyObject *ob, PyObject *modifier) { /* * Be a writable Attribute, but have a computed value. @@ -134,19 +137,19 @@ _get_written_signature(signaturefunc sf, PyObject *ob, const char *modifier) } static PyObject * -pyside_cf_get___signature__(PyObject *func, const char *modifier) +pyside_cf_get___signature__(PyObject *func, PyObject *modifier) { init_module_2(); return _get_written_signature(GetSignature_Function, func, modifier); } static PyObject * -pyside_sm_get___signature__(PyObject *sm, const char *modifier) +pyside_sm_get___signature__(PyObject *sm, PyObject *modifier) { init_module_2(); - Shiboken::AutoDecRef func(PyObject_GetAttrString(sm, "__func__")); + Shiboken::AutoDecRef func(PyObject_GetAttr(sm, Shiboken::PyMagicName::func())); if (Py_TYPE(func) == PepFunction_TypePtr) - return PyObject_GetAttrString(func, "__signature__"); + return PyObject_GetAttr(func, Shiboken::PyMagicName::signature()); return _get_written_signature(GetSignature_Function, func, modifier); } @@ -158,7 +161,7 @@ _get_class_of_cf(PyObject *ob_cf) selftype = PyDict_GetItem(pyside_globals->map_dict, ob_cf); if (selftype == nullptr) { // This must be an overloaded function that we handled special. - Shiboken::AutoDecRef special(Py_BuildValue("(Os)", ob_cf, "overload")); + Shiboken::AutoDecRef special(Py_BuildValue("(OO)", ob_cf, Shiboken::PyName::overload())); selftype = PyDict_GetItem(pyside_globals->map_dict, special); if (selftype == nullptr) { // This is probably a module function. We will return type(None). @@ -176,15 +179,14 @@ _get_class_of_cf(PyObject *ob_cf) static PyObject * _get_class_of_sm(PyObject *ob_sm) { - Shiboken::AutoDecRef func(PyObject_GetAttrString(ob_sm, "__func__")); + Shiboken::AutoDecRef func(PyObject_GetAttr(ob_sm, Shiboken::PyMagicName::func())); return _get_class_of_cf(func); } static PyObject * _get_class_of_descr(PyObject *ob) { - Shiboken::AutoDecRef func_name(PyObject_GetAttrString(ob, "__name__")); - return PyObject_GetAttrString(ob, "__objclass__"); + return PyObject_GetAttr(ob, Shiboken::PyMagicName::objclass()); } static PyObject * @@ -216,10 +218,10 @@ get_funcname(PyObject *ob) { PyObject *func = ob; if (Py_TYPE(ob) == PepStaticMethod_TypePtr) - func = PyObject_GetAttrString(ob, "__func__"); + func = PyObject_GetAttr(ob, Shiboken::PyMagicName::func()); else Py_INCREF(func); - PyObject *func_name = PyObject_GetAttrString(func, "__name__"); + PyObject *func_name = PyObject_GetAttr(func, Shiboken::PyMagicName::name()); Py_DECREF(func); if (func_name == nullptr) Py_FatalError("unexpected name problem in compute_name_key"); @@ -287,7 +289,7 @@ name_key_to_func(PyObject *ob) } static PyObject * -pyside_md_get___signature__(PyObject *ob_md, const char *modifier) +pyside_md_get___signature__(PyObject *ob_md, PyObject *modifier) { init_module_2(); Shiboken::AutoDecRef func(name_key_to_func(ob_md)); @@ -299,14 +301,14 @@ pyside_md_get___signature__(PyObject *ob_md, const char *modifier) } static PyObject * -pyside_wd_get___signature__(PyObject *ob, const char *modifier) +pyside_wd_get___signature__(PyObject *ob, PyObject *modifier) { init_module_2(); return _get_written_signature(GetSignature_Wrapper, ob, modifier); } static PyObject * -pyside_tp_get___signature__(PyObject *obtype_mod, const char *modifier) +pyside_tp_get___signature__(PyObject *obtype_mod, PyObject *modifier) { init_module_2(); return _get_written_signature(GetSignature_TypeMod, obtype_mod, modifier); @@ -314,7 +316,21 @@ pyside_tp_get___signature__(PyObject *obtype_mod, const char *modifier) // forward static PyObject * -GetSignature_Cached(PyObject *props, const char *func_kind, const char *modifier); +GetSignature_Cached(PyObject *props, PyObject *func_kind, PyObject *modifier); + +// Helper for __qualname__ which might not always exist in Python 2 (type). +static PyObject * +_get_qualname(PyObject *ob) +{ + // We support __qualname__ for types, only. + assert(PyType_Check(ob)); + PyObject *name = PyObject_GetAttr(ob, Shiboken::PyMagicName::qualname()); + if (name == nullptr) { + PyErr_Clear(); + name = PyObject_GetAttr(ob, Shiboken::PyMagicName::name()); + } + return name; +} static PyObject * GetTypeKey(PyObject *ob) @@ -332,19 +348,20 @@ GetTypeKey(PyObject *ob) * * This is the PyCFunction behavior, as opposed to Python functions. */ - Shiboken::AutoDecRef class_name(PyObject_GetAttrString(ob, "__name__")); - Shiboken::AutoDecRef module_name(PyObject_GetAttrString(ob, "__module__")); - - if (module_name.isNull()) + // PYSIDE-1286: We use correct __module__ and __qualname__, now. + Shiboken::AutoDecRef module_name(PyObject_GetAttr(ob, Shiboken::PyMagicName::module())); + if (module_name.isNull()) { + // We have no module_name because this is a module ;-) PyErr_Clear(); - - // Note: if we have a module, then __module__ is null, and we get - // the module name through __name__ . - if (class_name.isNull()) + module_name.reset(PyObject_GetAttr(ob, Shiboken::PyMagicName::name())); + return Py_BuildValue("O", module_name.object()); + } + Shiboken::AutoDecRef class_name(_get_qualname(ob)); + if (class_name.isNull()) { + Py_FatalError("Signature: missing class name in GetTypeKey"); return nullptr; - if (module_name.object()) - return Py_BuildValue("(OO)", module_name.object(), class_name.object()); - return Py_BuildValue("O", class_name.object()); + } + return Py_BuildValue("(OO)", module_name.object(), class_name.object()); } static PyObject *empty_dict = nullptr; @@ -364,7 +381,7 @@ TypeKey_to_PropsDict(PyObject *type_key, PyObject *obtype) } static PyObject * -GetSignature_Function(PyObject *obfunc, const char *modifier) +GetSignature_Function(PyObject *obfunc, PyObject *modifier) { // make sure that we look into PyCFunction, only... if (Py_TYPE(obfunc) == PepFunction_TypePtr) @@ -376,31 +393,30 @@ GetSignature_Function(PyObject *obfunc, const char *modifier) PyObject *dict = TypeKey_to_PropsDict(type_key, obtype_mod); if (dict == nullptr) return nullptr; - Shiboken::AutoDecRef func_name(PyObject_GetAttrString(obfunc, "__name__")); + Shiboken::AutoDecRef func_name(PyObject_GetAttr(obfunc, Shiboken::PyMagicName::name())); PyObject *props = !func_name.isNull() ? PyDict_GetItem(dict, func_name) : nullptr; if (props == nullptr) Py_RETURN_NONE; int flags = PyCFunction_GET_FLAGS(obfunc); - const char *func_kind; + PyObject *func_kind; if (PyModule_Check(obtype_mod)) - func_kind = "function"; + func_kind = Shiboken::PyName::function(); else if (flags & METH_CLASS) - func_kind = "classmethod"; + func_kind = Shiboken::PyName::classmethod(); else if (flags & METH_STATIC) - func_kind = "staticmethod"; + func_kind = Shiboken::PyName::staticmethod(); else - func_kind = "method"; + func_kind = Shiboken::PyName::method(); return GetSignature_Cached(props, func_kind, modifier); } static PyObject * -GetSignature_Wrapper(PyObject *ob, const char *modifier) +GetSignature_Wrapper(PyObject *ob, PyObject *modifier) { - Shiboken::AutoDecRef func_name(PyObject_GetAttrString(ob, "__name__")); - Shiboken::AutoDecRef objclass(PyObject_GetAttrString(ob, "__objclass__")); + Shiboken::AutoDecRef func_name(PyObject_GetAttr(ob, Shiboken::PyMagicName::name())); + Shiboken::AutoDecRef objclass(PyObject_GetAttr(ob, Shiboken::PyMagicName::objclass())); Shiboken::AutoDecRef class_key(GetTypeKey(objclass)); - if (func_name.isNull() || objclass.isNull() || class_key.isNull()) return nullptr; PyObject *dict = TypeKey_to_PropsDict(class_key, objclass); @@ -409,13 +425,13 @@ GetSignature_Wrapper(PyObject *ob, const char *modifier) PyObject *props = PyDict_GetItem(dict, func_name); if (props == nullptr) Py_RETURN_NONE; - return GetSignature_Cached(props, "method", modifier); + return GetSignature_Cached(props, Shiboken::PyName::method(), modifier); } static PyObject * -GetSignature_TypeMod(PyObject *ob, const char *modifier) +GetSignature_TypeMod(PyObject *ob, PyObject *modifier) { - Shiboken::AutoDecRef ob_name(PyObject_GetAttrString(ob, "__name__")); + Shiboken::AutoDecRef ob_name(PyObject_GetAttr(ob, Shiboken::PyMagicName::name())); Shiboken::AutoDecRef ob_key(GetTypeKey(ob)); PyObject *dict = TypeKey_to_PropsDict(ob_key, ob); @@ -424,19 +440,26 @@ GetSignature_TypeMod(PyObject *ob, const char *modifier) PyObject *props = PyDict_GetItem(dict, ob_name); if (props == nullptr) Py_RETURN_NONE; - return GetSignature_Cached(props, "method", modifier); + return GetSignature_Cached(props, Shiboken::PyName::method(), modifier); } static PyObject * -GetSignature_Cached(PyObject *props, const char *func_kind, const char *modifier) +GetSignature_Cached(PyObject *props, PyObject *func_kind, PyObject *modifier) { // Special case: We want to know the func_kind. - if (modifier && strcmp(modifier, "__func_kind__") == 0) - return Py_BuildValue("s", func_kind); + if (modifier) { +#if PYTHON_USES_UNICODE + PyUnicode_InternInPlace(&modifier); +#else + PyString_InternInPlace(&modifier); +#endif + if (modifier == Shiboken::PyMagicName::func_kind()) + return Py_BuildValue("O", func_kind); + } Shiboken::AutoDecRef key(modifier == nullptr - ? Py_BuildValue("s", func_kind) - : Py_BuildValue("(ss)", func_kind, modifier)); + ? Py_BuildValue("O", func_kind) + : Py_BuildValue("(OO)", func_kind, modifier)); PyObject *value = PyDict_GetItem(props, key); if (value == nullptr) { // we need to compute a signature object @@ -455,11 +478,11 @@ GetSignature_Cached(PyObject *props, const char *func_kind, const char *modifier } static const char *PySide_CompressedSignaturePackage[] = { -#include "embed/signature.inc" +#include "embed/signature_inc.h" }; static const unsigned char PySide_SignatureLoader[] = { -#include "embed/signature_bootstrap.inc" +#include "embed/signature_bootstrap_inc.h" }; static safe_globals_struc * @@ -478,13 +501,10 @@ init_phase_1(void) #ifdef Py_LIMITED_API // We must work for multiple versions, so use source code. #else - Shiboken::AutoDecRef marshal_str(Py_BuildValue("s", "marshal")); - if (marshal_str.isNull()) - goto error; - Shiboken::AutoDecRef marshal_module(PyImport_Import(marshal_str)); + Shiboken::AutoDecRef marshal_module(PyImport_Import(Shiboken::PyName::marshal())); if (marshal_module.isNull()) goto error; - Shiboken::AutoDecRef loads(PyObject_GetAttrString(marshal_module, "loads")); + Shiboken::AutoDecRef loads(PyObject_GetAttr(marshal_module, Shiboken::PyName::loads())); if (loads.isNull()) goto error; #endif @@ -496,7 +516,7 @@ init_phase_1(void) goto error; #ifdef Py_LIMITED_API PyObject *builtins = PyEval_GetBuiltins(); - PyObject *compile = PyDict_GetItemString(builtins, "compile"); + PyObject *compile = PyDict_GetItem(builtins, Shiboken::PyName::compile()); if (compile == nullptr) goto error; Shiboken::AutoDecRef code_obj(PyObject_CallFunction(compile, "Oss", @@ -513,7 +533,7 @@ init_phase_1(void) goto error; // Initialize the module PyObject *mdict = PyModule_GetDict(p->helper_module); - if (PyDict_SetItemString(mdict, "__builtins__", PyEval_GetBuiltins()) < 0) + if (PyDict_SetItem(mdict, Shiboken::PyMagicName::builtins(), PyEval_GetBuiltins()) < 0) goto error; /* * Unpack an embedded ZIP file with more signature modules. @@ -559,7 +579,7 @@ init_phase_1(void) } error: PyErr_Print(); - PyErr_SetString(PyExc_SystemError, "could not initialize part 1"); + Py_FatalError("could not initialize part 1"); return nullptr; } @@ -601,11 +621,10 @@ init_phase_2(safe_globals_struc *p, PyMethodDef *methods) if (p->finish_import_func == nullptr) goto error; return 0; - return 0; } error: PyErr_Print(); - PyErr_SetString(PyExc_SystemError, "could not initialize part 2"); + Py_FatalError("could not initialize part 2"); return -1; } @@ -803,7 +822,7 @@ static PyGetSetDef new_PyWrapperDescr_getsets[] = { // static PyObject * -get_signature_intern(PyObject *ob, const char *modifier) +get_signature_intern(PyObject *ob, PyObject *modifier) { if (PyType_IsSubtype(Py_TYPE(ob), &PyCFunction_Type)) return pyside_cf_get___signature__(ob, modifier); @@ -822,11 +841,11 @@ static PyObject * get_signature(PyObject * /* self */, PyObject *args) { PyObject *ob; - const char *modifier = nullptr; + PyObject *modifier = nullptr; init_module_1(); - if (!PyArg_ParseTuple(args, "O|s", &ob, &modifier)) + if (!PyArg_ParseTuple(args, "O|O", &ob, &modifier)) return nullptr; if (Py_TYPE(ob) == PepFunction_TypePtr) Py_RETURN_NONE; @@ -846,12 +865,15 @@ get_signature(PyObject * /* self */, PyObject *args) //////////////////////////////////////////////////////////////////////////// // a stack trace for linux-like platforms #include <stdio.h> -#include <execinfo.h> +#if defined(__GLIBC__) +# include <execinfo.h> +#endif #include <signal.h> #include <stdlib.h> #include <unistd.h> void handler(int sig) { +#if defined(__GLIBC__) void *array[30]; size_t size; @@ -859,8 +881,11 @@ void handler(int sig) { size = backtrace(array, 30); // print out all the frames to stderr +#endif fprintf(stderr, "Error: signal %d:\n", sig); +#if defined(__GLIBC__) backtrace_symbols_fd(array, size, STDERR_FILENO); +#endif exit(1); } @@ -1110,13 +1135,14 @@ _build_func_to_type(PyObject *obtype) * "{name}.overload". */ PyObject *descr = PyDict_GetItemString(dict, meth->ml_name); - const char *look_attr = meth->ml_flags & METH_STATIC ? "__func__" : "__name__"; + PyObject *look_attr = meth->ml_flags & METH_STATIC + ? Shiboken::PyMagicName::func() : Shiboken::PyMagicName::name(); int check_name = meth->ml_flags & METH_STATIC ? 0 : 1; if (descr == nullptr) return -1; // We first check all methods if one is hidden by something else. - Shiboken::AutoDecRef look(PyObject_GetAttrString(descr, look_attr)); + Shiboken::AutoDecRef look(PyObject_GetAttr(descr, look_attr)); Shiboken::AutoDecRef given(Py_BuildValue("s", meth->ml_name)); if (look.isNull() || (check_name && PyObject_RichCompareBool(look, given, Py_EQ) != 1)) { @@ -1223,7 +1249,7 @@ SetError_Argument(PyObject *args, const char *func_name) */ PyObject * -Sbk_TypeGet___signature__(PyObject *ob, const char *modifier) +Sbk_TypeGet___signature__(PyObject *ob, PyObject *modifier) { return pyside_tp_get___signature__(ob, modifier); } diff --git a/sources/shiboken2/libshiboken/signature.h b/sources/shiboken2/libshiboken/signature.h index 57fd4047a..b22a78497 100644 --- a/sources/shiboken2/libshiboken/signature.h +++ b/sources/shiboken2/libshiboken/signature.h @@ -48,7 +48,7 @@ extern "C" LIBSHIBOKEN_API int SbkSpecial_Type_Ready(PyObject *, PyTypeObject *, const char *[]); LIBSHIBOKEN_API void FinishSignatureInitialization(PyObject *, const char *[]); LIBSHIBOKEN_API void SetError_Argument(PyObject *, const char *); -LIBSHIBOKEN_API PyObject *Sbk_TypeGet___signature__(PyObject *, const char *); +LIBSHIBOKEN_API PyObject *Sbk_TypeGet___signature__(PyObject *, PyObject *); LIBSHIBOKEN_API PyObject *Sbk_TypeGet___doc__(PyObject *); } // extern "C" diff --git a/sources/shiboken2/libshiboken/signature_doc.rst b/sources/shiboken2/libshiboken/signature_doc.rst index a984de4ce..cb9041d0c 100644 --- a/sources/shiboken2/libshiboken/signature_doc.rst +++ b/sources/shiboken2/libshiboken/signature_doc.rst @@ -265,6 +265,17 @@ we can now capture the error output of COIN and check the generated module in. +Explicitly Enforcing Recreation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The former way to regenerate the registry files was to remove the files +and check that in. This has the desired effect, but creates huge deltas. +As a more efficient way, we have prepared a comment in the first line +that contains the word "recreate". +By uncommenting this line, a NameError is triggered, which has the same +effect. + + init_platform.py ~~~~~~~~~~~~~~~~ diff --git a/sources/shiboken2/libshiboken/typespec.cpp b/sources/shiboken2/libshiboken/typespec.cpp index 6dc5b00bc..902ed55af 100644 --- a/sources/shiboken2/libshiboken/typespec.cpp +++ b/sources/shiboken2/libshiboken/typespec.cpp @@ -39,6 +39,7 @@ #include "sbkpython.h" #include "typespec.h" +#include "sbkstaticstrings.h" #include <structmember.h> #if PY_MAJOR_VERSION < 3 @@ -713,7 +714,7 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) if (PyType_Ready(type) < 0) goto fail; - // no ht_hached_keys in Python 2 + // no ht_cached_keys in Python 2 // if (type->tp_dictoffset) { // res->ht_cached_keys = _PyDict_NewKeysForClass(); // } @@ -730,7 +731,7 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) } // no PyId_ things in Python 2 // err = _PyDict_SetItemId(type->tp_dict, &PyId___module__, modname); - err = PyDict_SetItemString(type->tp_dict, "__module__", modname); + err = PyDict_SetItem(type->tp_dict, Shiboken::PyMagicName::module(), modname); Py_DECREF(modname); if (err != 0) goto fail; diff --git a/sources/shiboken2/libshiboken/voidptr.cpp b/sources/shiboken2/libshiboken/voidptr.cpp index d4ce58c87..6ac3c131d 100644 --- a/sources/shiboken2/libshiboken/voidptr.cpp +++ b/sources/shiboken2/libshiboken/voidptr.cpp @@ -335,12 +335,12 @@ static PyType_Slot SbkVoidPtrType_slots[] = { {Py_tp_richcompare, (void *)SbkVoidPtrObject_richcmp}, {Py_tp_init, (void *)SbkVoidPtrObject_init}, {Py_tp_new, (void *)SbkVoidPtrObject_new}, - {Py_tp_dealloc, (void *)object_dealloc}, + {Py_tp_dealloc, (void *)Sbk_object_dealloc}, {Py_tp_methods, (void *)SbkVoidPtrObject_methods}, {0, nullptr} }; static PyType_Spec SbkVoidPtrType_spec = { - "shiboken2.libshiboken.VoidPtr", + "2:shiboken2.shiboken2.VoidPtr", sizeof(SbkVoidPtrObject), 0, Py_TPFLAGS_DEFAULT, @@ -353,16 +353,15 @@ static PyType_Spec SbkVoidPtrType_spec = { PyTypeObject *SbkVoidPtrTypeF(void) { static PyTypeObject *type = nullptr; - if (!type) - type = reinterpret_cast<PyTypeObject *>(PyType_FromSpec(&SbkVoidPtrType_spec)); - + if (!type) { + type = reinterpret_cast<PyTypeObject *>(SbkType_FromSpec(&SbkVoidPtrType_spec)); #if PY_VERSION_HEX < 0x03000000 - type->tp_as_buffer = &SbkVoidPtrObjectBufferProc; - type->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; + type->tp_as_buffer = &SbkVoidPtrObjectBufferProc; + type->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; #else - PepType_AS_BUFFER(type) = &SbkVoidPtrObjectBufferProc; + PepType_AS_BUFFER(type) = &SbkVoidPtrObjectBufferProc; #endif - + } return type; } |