diff options
Diffstat (limited to 'sources/shiboken6/libshiboken')
58 files changed, 3423 insertions, 2176 deletions
diff --git a/sources/shiboken6/libshiboken/CMakeLists.txt b/sources/shiboken6/libshiboken/CMakeLists.txt index 2dd4e86ae..b5bbb498a 100644 --- a/sources/shiboken6/libshiboken/CMakeLists.txt +++ b/sources/shiboken6/libshiboken/CMakeLists.txt @@ -1,3 +1,6 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + project(libshiboken) option(ENABLE_VERSION_SUFFIX "Used to use current version in suffix to generated files. This is used to allow multiples versions installed simultaneous." FALSE) @@ -27,7 +30,7 @@ if(SHIBOKEN_IS_CROSS_BUILD) set(host_python_path "${QFP_PYTHON_HOST_PATH}") set(use_pyc_in_embedding FALSE) else() - set(host_python_path "${PYTHON_EXECUTABLE}") + set(host_python_path "${Python_EXECUTABLE}") if(PYTHON_LIMITED_API) set(use_pyc_in_embedding FALSE) else() @@ -55,38 +58,50 @@ set(libshiboken_VERSION "${libshiboken_MAJOR_VERSION}.${libshiboken_MINOR_VERSIO set(libshiboken_SOVERSION "${shiboken6_library_so_version}") set(libshiboken_SRC -basewrapper.cpp -debugfreehook.cpp -gilstate.cpp -helper.cpp -sbkarrayconverter.cpp -sbkcontainer.cpp -sbkconverter.cpp -sbkenum.cpp -sbkerrors.cpp -sbkfeature_base.cpp -sbkmodule.cpp -sbknumpy.cpp -sbkcppstring.cpp -sbkstring.cpp -sbkstaticstrings.cpp -sbktypefactory.cpp -bindingmanager.cpp -threadstatesaver.cpp -shibokenbuffer.cpp -pep384impl.cpp -voidptr.cpp -bufferprocs_py37.cpp +autodecref.h +basewrapper.cpp basewrapper.h basewrapper_p.h +bindingmanager.cpp bindingmanager.h +bufferprocs_py37.cpp bufferprocs_py37.h +debugfreehook.cpp debugfreehook.h +gilstate.cpp gilstate.h +helper.cpp helper.h +pep384impl.cpp pep384impl.h +pyobjectholder.h +sbkarrayconverter.cpp sbkarrayconverter.h sbkarrayconverter_p.h +sbkcontainer.cpp sbkcontainer.h +sbkconverter.cpp sbkconverter.h sbkconverter_p.h +sbkcppstring.cpp sbkcppstring.h sbkcpptonumpy.h +sbkenum.cpp sbkenum.h +sbkerrors.cpp sbkerrors.h +sbkfeature_base.cpp sbkfeature_base.h +sbkmodule.cpp sbkmodule.h +sbknumpy.cpp sbknumpycheck.h +sbknumpyview.h +sbkpython.h +sbksmartpointer.cpp sbksmartpointer.h +sbkstaticstrings.cpp sbkstaticstrings.h sbkstaticstrings_p.h +sbkstring.cpp sbkstring.h +sbktypefactory.cpp sbktypefactory.h +sbkwindows.h +shiboken.h +shibokenbuffer.cpp shibokenbuffer.h +shibokenmacros.h +threadstatesaver.cpp threadstatesaver.h +voidptr.cpp voidptr.h embed/signature_bootstrap_inc.h embed/signature_inc.h -signature/signature.cpp +signature/signature.cpp signature.h signature_p.h signature/signature_globals.cpp signature/signature_extend.cpp signature/signature_helper.cpp ) +# This is needed to let the header obey a variable in "pep384impl.h". +# Note: This must be set on the cpp file! +set_property(SOURCE "pep384impl.cpp" PROPERTY SKIP_UNITY_BUILD_INCLUSION ON) + add_library(libshiboken SHARED ${libshiboken_SRC}) add_library(Shiboken6::libshiboken ALIAS libshiboken) @@ -144,12 +159,12 @@ install(FILES bindingmanager.h gilstate.h helper.h + pyobjectholder.h sbkarrayconverter.h sbkcontainer.h sbkconverter.h sbkcpptonumpy.h sbkenum.h - sbkenum_p.h sbkerrors.h sbkfeature_base.h sbkmodule.h @@ -157,6 +172,7 @@ install(FILES sbknumpyview.h sbkstring.h sbkcppstring.h + sbksmartpointer.h sbkstaticstrings.h sbktypefactory.h shiboken.h @@ -164,7 +180,9 @@ install(FILES threadstatesaver.h shibokenbuffer.h sbkpython.h + sbkwindows.h pep384impl.h + pep384ext.h voidptr.h bufferprocs_py37.h "${CMAKE_CURRENT_BINARY_DIR}/sbkversion.h" diff --git a/sources/shiboken6/libshiboken/autodecref.h b/sources/shiboken6/libshiboken/autodecref.h index d2b660676..62a8584e1 100644 --- a/sources/shiboken6/libshiboken/autodecref.h +++ b/sources/shiboken6/libshiboken/autodecref.h @@ -5,7 +5,8 @@ #define AUTODECREF_H #include "sbkpython.h" -#include "basewrapper.h" + +#include <utility> struct SbkObject; namespace Shiboken @@ -14,29 +15,27 @@ namespace Shiboken /** * AutoDecRef holds a PyObject pointer and decrement its reference counter when destroyed. */ -struct LIBSHIBOKEN_API AutoDecRef +struct AutoDecRef { public: AutoDecRef(const AutoDecRef &) = delete; - AutoDecRef(AutoDecRef &&) = delete; + AutoDecRef(AutoDecRef &&o) noexcept : m_pyObj{std::exchange(o.m_pyObj, nullptr)} {} AutoDecRef &operator=(const AutoDecRef &) = delete; - AutoDecRef &operator=(AutoDecRef &&) = delete; + AutoDecRef &operator=(AutoDecRef &&o) noexcept + { + m_pyObj = std::exchange(o.m_pyObj, nullptr); + return *this; + } - /** - * AutoDecRef constructor. - * \param pyobj A borrowed reference to a Python object - */ - explicit AutoDecRef(PyObject *pyObj) : m_pyObj(pyObj) {} - /** - * AutoDecRef constructor. - * \param pyobj A borrowed reference to a Python object - */ - explicit AutoDecRef(SbkObject *pyObj) : m_pyObj(reinterpret_cast<PyObject *>(pyObj)) {} - /** - * AutoDecref constructor. - * To be used later with reset(): - */ - AutoDecRef() : m_pyObj(nullptr) {} + /// AutoDecRef constructor. + /// \param pyobj A borrowed reference to a Python object + explicit AutoDecRef(PyObject *pyObj) noexcept : m_pyObj(pyObj) {} + /// AutoDecRef constructor. + /// \param pyobj A borrowed reference to a wrapped Python object + explicit AutoDecRef(SbkObject *pyObj) noexcept : m_pyObj(reinterpret_cast<PyObject *>(pyObj)) {} + /// AutoDecref default constructor. + /// To be used later with reset(): + AutoDecRef() noexcept = default; /// Decref the borrowed python reference ~AutoDecRef() @@ -44,18 +43,19 @@ public: Py_XDECREF(m_pyObj); } - inline bool isNull() const { return m_pyObj == nullptr; } + [[nodiscard]] bool isNull() const { return m_pyObj == nullptr; } /// Returns the pointer of the Python object being held. - inline PyObject *object() { return m_pyObj; } - inline operator PyObject *() { return m_pyObj; } + [[nodiscard]] PyObject *object() const { return m_pyObj; } + [[nodiscard]] operator PyObject *() const { return m_pyObj; } #ifndef Py_LIMITED_API - inline operator PyTupleObject *() { return reinterpret_cast<PyTupleObject *>(m_pyObj); } + [[deprecated]] inline operator PyTupleObject *() + { return reinterpret_cast<PyTupleObject *>(m_pyObj); } #endif inline operator bool() const { return m_pyObj != nullptr; } inline PyObject *operator->() { return m_pyObj; } template<typename T> - T cast() + [[deprecated]] T cast() { return reinterpret_cast<T>(m_pyObj); } @@ -79,10 +79,9 @@ public: } private: - PyObject *m_pyObj; + PyObject *m_pyObj = nullptr; }; } // namespace Shiboken #endif // AUTODECREF_H - diff --git a/sources/shiboken6/libshiboken/basewrapper.cpp b/sources/shiboken6/libshiboken/basewrapper.cpp index 24980239d..0ce80d0c6 100644 --- a/sources/shiboken6/libshiboken/basewrapper.cpp +++ b/sources/shiboken6/libshiboken/basewrapper.cpp @@ -5,9 +5,12 @@ #include "basewrapper_p.h" #include "bindingmanager.h" #include "helper.h" +#include "pep384ext.h" #include "sbkconverter.h" #include "sbkenum.h" +#include "sbkerrors.h" #include "sbkfeature_base.h" +#include "sbkmodule.h" #include "sbkstring.h" #include "sbkstaticstrings.h" #include "sbkstaticstrings_p.h" @@ -24,7 +27,9 @@ #include "signature_p.h" #include "voidptr.h" +#include <string> #include <iostream> +#include <sstream> #if defined(__APPLE__) #include <dlfcn.h> @@ -34,7 +39,73 @@ namespace { void _destroyParentInfo(SbkObject *obj, bool keepReference); } -static void callDestructor(const Shiboken::DtorAccumulatorVisitor::DestructorEntries &dts) +namespace Shiboken +{ +// Walk through the first level of non-user-type Sbk base classes relevant for +// C++ object allocation. Return true from the predicate to terminate. +template <class Predicate> +bool walkThroughBases(PyTypeObject *currentType, Predicate predicate) +{ + PyObject *bases = currentType->tp_bases; + const Py_ssize_t numBases = PyTuple_Size(bases); + bool result = false; + for (Py_ssize_t i = 0; !result && i < numBases; ++i) { + auto type = reinterpret_cast<PyTypeObject *>(PyTuple_GetItem(bases, i)); + if (PyType_IsSubtype(type, SbkObject_TypeF()) != 0) { + result = PepType_SOTP(type)->is_user_type + ? walkThroughBases(type, predicate) : predicate(type); + } + } + return result; +} + +int getTypeIndexOnHierarchy(PyTypeObject *baseType, PyTypeObject *desiredType) +{ + int index = -1; + walkThroughBases(baseType, [&index, desiredType](PyTypeObject *node) { + ++index; + return PyType_IsSubtype(node, desiredType) != 0; + }); + return index; +} + +int getNumberOfCppBaseClasses(PyTypeObject *baseType) +{ + int count = 0; + walkThroughBases(baseType, [&count](PyTypeObject *) { + ++count; + return false; + }); + return count; +} + +std::vector<PyTypeObject *> getCppBaseClasses(PyTypeObject *baseType) +{ + std::vector<PyTypeObject *> cppBaseClasses; + walkThroughBases(baseType, [&cppBaseClasses](PyTypeObject *node) { + cppBaseClasses.push_back(node); + return false; + }); + return cppBaseClasses; +} + +using DestructorEntries = std::vector<DestructorEntry>; + +DestructorEntries getDestructorEntries(SbkObject *o) +{ + DestructorEntries result; + void **cptrs = o->d->cptr; + walkThroughBases(Py_TYPE(o), [&result, cptrs](PyTypeObject *node) { + auto *sotp = PepType_SOTP(node); + auto index = result.size(); + result.push_back(DestructorEntry{sotp->cpp_dtor, + cptrs[index]}); + return false; + }); + return result; +} + +static void callDestructor(const DestructorEntries &dts) { for (const auto &e : dts) { Shiboken::ThreadStateSaver threadSaver; @@ -43,6 +114,8 @@ static void callDestructor(const Shiboken::DtorAccumulatorVisitor::DestructorEnt } } +} // namespace Shiboken + extern "C" { @@ -54,7 +127,7 @@ void Sbk_object_dealloc(PyObject *self) // This was not needed before Python 3.8 (Python issue 35810) Py_DECREF(Py_TYPE(self)); } - Py_TYPE(self)->tp_free(self); + PepExt_TypeCallFree(self); } static void SbkObjectType_tp_dealloc(PyTypeObject *pyType); @@ -71,6 +144,7 @@ void setDestroyQApplication(DestroyQAppHook func) // PYSIDE-535: Use the C API in PyPy instead of `op->ob_dict`, directly LIBSHIBOKEN_API PyObject *SbkObject_GetDict_NoRef(PyObject *op) { + assert(Shiboken::Object::checkType(op)); #ifdef PYPY_VERSION Shiboken::GilState state; auto *ret = PyObject_GenericGetDict(op, nullptr); @@ -109,15 +183,13 @@ type_set_doc(PyTypeObject *type, PyObject *value, void * /* context */) if (!check_set_special_type_attr(type, value, "__doc__")) return -1; PyType_Modified(type); - return PyDict_SetItem(type->tp_dict, Shiboken::PyMagicName::doc(), value); + Shiboken::AutoDecRef tpDict(PepType_GetDict(type)); + return PyDict_SetItem(tpDict.object(), Shiboken::PyMagicName::doc(), value); } // PYSIDE-908: The function PyType_Modified does not work in PySide, so we need to -// explicitly pass __doc__. For __signature__ it _did_ actually work, because -// it was not existing before. We add them both for clarity. +// explicitly pass __doc__. static PyGetSetDef SbkObjectType_tp_getset[] = { - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(Sbk_TypeGet___signature__), - nullptr, nullptr, nullptr}, {const_cast<char *>("__doc__"), reinterpret_cast<getter>(Sbk_TypeGet___doc__), reinterpret_cast<setter>(type_set_doc), nullptr, nullptr}, {const_cast<char *>("__dict__"), reinterpret_cast<getter>(Sbk_TypeGet___dict__), @@ -125,31 +197,53 @@ static PyGetSetDef SbkObjectType_tp_getset[] = { {nullptr, nullptr, nullptr, nullptr, nullptr} // Sentinel }; -static PyType_Slot SbkObjectType_Type_slots[] = { - {Py_tp_dealloc, reinterpret_cast<void *>(SbkObjectType_tp_dealloc)}, - {Py_tp_getattro, reinterpret_cast<void *>(mangled_type_getattro)}, - {Py_tp_base, static_cast<void *>(&PyType_Type)}, - {Py_tp_alloc, reinterpret_cast<void *>(PyType_GenericAlloc)}, - {Py_tp_new, reinterpret_cast<void *>(SbkObjectType_tp_new)}, - {Py_tp_free, reinterpret_cast<void *>(PyObject_GC_Del)}, - {Py_tp_getset, reinterpret_cast<void *>(SbkObjectType_tp_getset)}, - {0, nullptr} -}; - -// PYSIDE-535: The tp_itemsize field is inherited and does not need to be set. -// In PyPy, it _must_ not be set, because it would have the meaning that a -// `__len__` field must be defined. Not doing so creates a hard-to-find crash. -static PyType_Spec SbkObjectType_Type_spec = { - "1:Shiboken.ObjectType", - 0, - 0, // sizeof(PyMemberDef), not for PyPy without a __len__ defined - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, - SbkObjectType_Type_slots, -}; +static PyTypeObject *createObjectTypeType() +{ + PyType_Slot SbkObjectType_Type_slots[] = { + {Py_tp_dealloc, reinterpret_cast<void *>(SbkObjectType_tp_dealloc)}, + {Py_tp_getattro, reinterpret_cast<void *>(mangled_type_getattro)}, + {Py_tp_base, static_cast<void *>(&PyType_Type)}, + {Py_tp_alloc, reinterpret_cast<void *>(PyType_GenericAlloc)}, + {Py_tp_new, reinterpret_cast<void *>(SbkObjectType_tp_new)}, + {Py_tp_free, reinterpret_cast<void *>(PyObject_GC_Del)}, + {Py_tp_getset, reinterpret_cast<void *>(SbkObjectType_tp_getset)}, + {0, nullptr} + }; + + // PYSIDE-535: The tp_itemsize field is inherited and does not need to be set. + // In PyPy, it _must_ not be set, because it would have the meanin + // that a `__len__` field must be defined. Not doing so creates + // a hard-to-find crash. + // + // PYSIDE-2230: In Python < 3.12, the decision which base class should create + // the instance is arbitrarily drawn by the size of the type. + // Ignoring this creates a bug in the new version of bug_825 that + // selects the wrong metatype. + // + PyType_Spec SbkObjectType_Type_spec = { + "1:Shiboken.ObjectType", + static_cast<int>(PyType_Type.tp_basicsize) + 1, // see above + 0, // sizeof(PyMemberDef), not for PyPy without a __len__ defined + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_TYPE_SUBCLASS, + SbkObjectType_Type_slots, + }; + + PyType_Spec SbkObjectType_Type_spec_312 = { + "1:Shiboken.ObjectType", + -long(sizeof(SbkObjectTypePrivate)), + 0, // sizeof(PyMemberDef), not for PyPy without a __len__ defined + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_TYPE_SUBCLASS, + SbkObjectType_Type_slots, + }; + + return SbkType_FromSpec(_PepRuntimeVersion() >= 0x030C00 ? + &SbkObjectType_Type_spec_312 : + &SbkObjectType_Type_spec); +} PyTypeObject *SbkObjectType_TypeF(void) { - static auto *type = SbkType_FromSpec(&SbkObjectType_Type_spec); + static auto *type = createObjectTypeType(); return type; } @@ -186,10 +280,8 @@ static int SbkObject_tp_traverse(PyObject *self, visitproc visit, void *arg) if (sbkSelf->ob_dict) Py_VISIT(sbkSelf->ob_dict); -#if PY_VERSION_HEX >= 0x03090000 // This was not needed before Python 3.9 (Python issue 35810 and 40217) Py_VISIT(Py_TYPE(self)); -#endif return 0; } @@ -209,40 +301,53 @@ static int SbkObject_tp_clear(PyObject *self) return 0; } -static PyType_Slot SbkObject_Type_slots[] = { - {Py_tp_getattro, reinterpret_cast<void *>(SbkObject_GenericGetAttr)}, - {Py_tp_setattro, reinterpret_cast<void *>(SbkObject_GenericSetAttr)}, - {Py_tp_dealloc, reinterpret_cast<void *>(SbkDeallocWrapperWithPrivateDtor)}, - {Py_tp_traverse, reinterpret_cast<void *>(SbkObject_tp_traverse)}, - {Py_tp_clear, reinterpret_cast<void *>(SbkObject_tp_clear)}, - // unsupported: {Py_tp_weaklistoffset, (void *)offsetof(SbkObject, weakreflist)}, - {Py_tp_getset, reinterpret_cast<void *>(SbkObject_tp_getset)}, - // unsupported: {Py_tp_dictoffset, (void *)offsetof(SbkObject, ob_dict)}, - {0, nullptr} -}; -static PyType_Spec SbkObject_Type_spec = { - "1:Shiboken.Object", - sizeof(SbkObject), - 0, - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, - SbkObject_Type_slots, -}; - -static const char *SbkObject_SignatureStrings[] = { - "Shiboken.Object(self)", - nullptr}; // Sentinel +static PyTypeObject *createObjectType() +{ + PyType_Slot SbkObject_Type_slots[] = { + {Py_tp_getattro, reinterpret_cast<void *>(SbkObject_GenericGetAttr)}, + {Py_tp_setattro, reinterpret_cast<void *>(SbkObject_GenericSetAttr)}, + {Py_tp_dealloc, reinterpret_cast<void *>(SbkDeallocWrapperWithPrivateDtor)}, + {Py_tp_traverse, reinterpret_cast<void *>(SbkObject_tp_traverse)}, + {Py_tp_clear, reinterpret_cast<void *>(SbkObject_tp_clear)}, + // unsupported: {Py_tp_weaklistoffset, (void *)offsetof(SbkObject, weakreflist)}, + {Py_tp_getset, reinterpret_cast<void *>(SbkObject_tp_getset)}, + // unsupported: {Py_tp_dictoffset, (void *)offsetof(SbkObject, ob_dict)}, + {0, nullptr} + }; + + PyType_Spec SbkObject_Type_spec = { + "1:Shiboken.Object", + sizeof(SbkObject), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + SbkObject_Type_slots, + }; + + // PYSIDE-2230: When creating this type, we cannot easily handle the metaclass. + // In versions < Python 3.12, the metaclass can only be set + // indirectly by a base which has that metaclass. + // But before 3.12 is the minimum version, we cannot use the new + // function, although we would need this for 3.12 :-D + // We do a special patching here that is triggered through Py_None. + auto *type = SbkType_FromSpec_BMDWB(&SbkObject_Type_spec, + Py_None, // bases, spectial flag! + SbkObjectType_TypeF(), + offsetof(SbkObject, ob_dict), + offsetof(SbkObject, weakreflist), + nullptr); // bufferprocs + return type; +} PyTypeObject *SbkObject_TypeF(void) { - static auto *type = SbkType_FromSpec_BMDWB(&SbkObject_Type_spec, - nullptr, // bases - SbkObjectType_TypeF(), - offsetof(SbkObject, ob_dict), - offsetof(SbkObject, weakreflist), - nullptr); // bufferprocs + static auto *type = createObjectType(); // bufferprocs return type; } +static const char *SbkObject_SignatureStrings[] = { + "Shiboken.Object(self)", + nullptr}; // Sentinel + static int mainThreadDeletionHandler(void *) { if (Py_IsInitialized()) @@ -305,9 +410,8 @@ static void SbkDeallocWrapperCommon(PyObject *pyObj, bool canDelete) if (sotp->delete_in_main_thread && Shiboken::currentThreadId() != Shiboken::mainThreadId()) { auto &bindingManager = Shiboken::BindingManager::instance(); if (sotp->is_multicpp) { - Shiboken::DtorAccumulatorVisitor visitor(sbkObj); - Shiboken::walkThroughClassHierarchy(Py_TYPE(pyObj), &visitor); - for (const auto &e : visitor.entries()) + const auto entries = Shiboken::getDestructorEntries(sbkObj); + for (const auto &e : entries) bindingManager.addToDeletionInMainThread(e); } else { Shiboken::DestructorEntry e{sotp->cpp_dtor, sbkObj->d->cptr[0]}; @@ -325,10 +429,9 @@ static void SbkDeallocWrapperCommon(PyObject *pyObj, bool canDelete) if (canDelete) { if (sotp->is_multicpp) { - Shiboken::DtorAccumulatorVisitor visitor(sbkObj); - Shiboken::walkThroughClassHierarchy(Py_TYPE(pyObj), &visitor); + const auto entries = Shiboken::getDestructorEntries(sbkObj); Shiboken::Object::deallocData(sbkObj, true); - callDestructor(visitor.entries()); + callDestructor(entries); } else { void *cptr = sbkObj->d->cptr[0]; Shiboken::Object::deallocData(sbkObj, true); @@ -388,7 +491,7 @@ void SbkObjectType_tp_dealloc(PyTypeObject *sbkType) auto pyObj = reinterpret_cast<PyObject *>(sbkType); PyObject_GC_UnTrack(pyObj); -#ifndef Py_LIMITED_API +#if !defined(Py_LIMITED_API) && !defined(PYPY_VERSION) # if PY_VERSION_HEX >= 0x030A0000 Py_TRASHCAN_BEGIN(pyObj, 1); # else @@ -406,7 +509,7 @@ void SbkObjectType_tp_dealloc(PyTypeObject *sbkType) Shiboken::Conversions::deleteConverter(sotp->converter); PepType_SOTP_delete(sbkType); } -#ifndef Py_LIMITED_API +#if !defined(Py_LIMITED_API) && !defined(PYPY_VERSION) # if PY_VERSION_HEX >= 0x030A0000 Py_TRASHCAN_END; # else @@ -477,7 +580,7 @@ static PyTypeObject *SbkObjectType_tp_new(PyTypeObject *metatype, PyObject *args PyObject *dict; static const char *kwlist[] = { "name", "bases", "dict", nullptr}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "sO!O!:sbktype", const_cast<char **>(kwlist), + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO!O!:sbktype", const_cast<char **>(kwlist), &name, &PyTuple_Type, &pyBases, &PyDict_Type, &dict)) @@ -485,22 +588,26 @@ static PyTypeObject *SbkObjectType_tp_new(PyTypeObject *metatype, PyObject *args for (int i=0, i_max=PyTuple_GET_SIZE(pyBases); i < i_max; i++) { PyObject *baseType = PyTuple_GET_ITEM(pyBases, i); - if (reinterpret_cast<PyTypeObject *>(baseType)->tp_new == SbkDummyNew) { + if (PepExt_Type_GetNewSlot(reinterpret_cast<PyTypeObject *>(baseType)) == SbkDummyNew) { // PYSIDE-595: A base class does not allow inheritance. return reinterpret_cast<PyTypeObject *>(SbkDummyNew(metatype, args, kwds)); } } - // PYSIDE-939: This is a temporary patch that circumvents the problem - // with Py_TPFLAGS_METHOD_DESCRIPTOR until this is finally solved. - // PyType_Ready uses mro(). We need to temporarily remove the flag from it's type. - // We cannot use PyMethodDescr_Type since it is not exported by Python 2.7 . - static PyTypeObject *PyMethodDescr_TypePtr = Py_TYPE( - PyObject_GetAttr(reinterpret_cast<PyObject *>(&PyType_Type), Shiboken::PyName::mro())); - auto hold = PyMethodDescr_TypePtr->tp_flags; - PyMethodDescr_TypePtr->tp_flags &= ~Py_TPFLAGS_METHOD_DESCRIPTOR; - auto *newType = PepType_Type_tp_new(metatype, args, kwds); - PyMethodDescr_TypePtr->tp_flags = hold; + // PYSIDE-939: This is still a temporary patch that circumvents the problem + // with Py_TPFLAGS_METHOD_DESCRIPTOR. The problem exists in Python 3.8 + // until 3.9.12, only. We check the runtime and hope for this version valishing. + // https://github.com/python/cpython/issues/92112 will not be fixed for 3.8 :/ + PyTypeObject *newType{}; + static auto triplet = _PepRuntimeVersion(); + if (triplet >= (3 << 16 | 8 << 8 | 0) && triplet < (3 << 16 | 9 << 8 | 13)) { + auto hold = PyMethodDescr_Type.tp_flags; + PyMethodDescr_Type.tp_flags &= ~Py_TPFLAGS_METHOD_DESCRIPTOR; + newType = PepType_Type_tp_new(metatype, args, kwds); + PyMethodDescr_Type.tp_flags = hold; + } else { + newType = PepType_Type_tp_new(metatype, args, kwds); + } if (!newType) return nullptr; @@ -598,9 +705,9 @@ PyObject *SbkQApp_tp_new(PyTypeObject *subtype, PyObject *, PyObject *) PyObject *SbkDummyNew(PyTypeObject *type, PyObject *, PyObject *) { // PYSIDE-595: Give the same error as type_call does when tp_new is NULL. + const char regret[] = "¯\\_(ツ)_/¯"; PyErr_Format(PyExc_TypeError, - "cannot create '%.100s' instances ¯\\_(ツ)_/¯", - type->tp_name); + "cannot create '%.100s' instances %s", type->tp_name, regret); return nullptr; } @@ -631,6 +738,12 @@ PyObject *FallbackRichCompare(PyObject *self, PyObject *other, int op) return res; } +bool SbkObjectType_Check(PyTypeObject *type) +{ + static auto *meta = SbkObjectType_TypeF(); + return Py_TYPE(type) == meta || PyType_IsSubtype(Py_TYPE(type), meta); +} + } //extern "C" @@ -655,54 +768,24 @@ void _destroyParentInfo(SbkObject *obj, bool keepReference) namespace Shiboken { -bool walkThroughClassHierarchy(PyTypeObject *currentType, HierarchyVisitor *visitor) -{ - PyObject *bases = currentType->tp_bases; - Py_ssize_t numBases = PyTuple_GET_SIZE(bases); - bool result = false; - for (int i = 0; !result && i < numBases; ++i) { - auto type = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(bases, i)); - if (PyType_IsSubtype(type, reinterpret_cast<PyTypeObject *>(SbkObject_TypeF()))) { - result = PepType_SOTP(type)->is_user_type - ? walkThroughClassHierarchy(type, visitor) : visitor->visit(type); - } - } - return result; -} // Wrapper metatype and base type ---------------------------------------------------------- -HierarchyVisitor::HierarchyVisitor() = default; -HierarchyVisitor::~HierarchyVisitor() = default; - -bool BaseCountVisitor::visit(PyTypeObject *) -{ - m_count++; - return false; -} - -bool BaseAccumulatorVisitor::visit(PyTypeObject *node) -{ - m_bases.push_back(node); - return false; -} - -bool GetIndexVisitor::visit(PyTypeObject *node) -{ - m_index++; - return PyType_IsSubtype(node, m_desiredType); -} +void _initMainThreadId(); // helper.cpp -bool DtorAccumulatorVisitor::visit(PyTypeObject *node) +static std::string msgFailedToInitializeType(const char *description) { - auto *sotp = PepType_SOTP(node); - m_entries.push_back(DestructorEntry{sotp->cpp_dtor, - m_pyObject->d->cptr[m_entries.size()]}); - return false; + std::ostringstream stream; + stream << "[libshiboken] Failed to initialize " << description; + if (auto *error = PepErr_GetRaisedException()) { + if (auto *str = PyObject_Str(error)) + stream << ": " << Shiboken::String::toCString(str); + Py_DECREF(error); + } + stream << '.'; + return stream.str(); } -void _initMainThreadId(); // helper.cpp - namespace Conversions { void init(); } void init() @@ -718,14 +801,13 @@ void init() //Init private data Pep384_Init(); - if (PyType_Ready(SbkEnumType_TypeF()) < 0) - Py_FatalError("[libshiboken] Failed to initialize Shiboken.SbkEnumType metatype."); + auto *type = SbkObjectType_TypeF(); + if (type == nullptr || PyType_Ready(type) < 0) + Py_FatalError(msgFailedToInitializeType("Shiboken.BaseWrapperType metatype").c_str()); - if (PyType_Ready(SbkObjectType_TypeF()) < 0) - Py_FatalError("[libshiboken] Failed to initialize Shiboken.BaseWrapperType metatype."); - - if (PyType_Ready(SbkObject_TypeF()) < 0) - Py_FatalError("[libshiboken] Failed to initialize Shiboken.BaseWrapper type."); + type = SbkObject_TypeF(); + if (type == nullptr || PyType_Ready(type) < 0) + Py_FatalError(msgFailedToInitializeType("Shiboken.BaseWrapper type").c_str()); VoidPtr::init(); @@ -756,6 +838,32 @@ void setErrorAboutWrongArguments(PyObject *args, const char *funcName, PyObject SetError_Argument(args, funcName, info); } +PyObject *returnWrongArguments(PyObject *args, const char *funcName, PyObject *info) +{ + setErrorAboutWrongArguments(args, funcName, info); + return {}; +} + +int returnWrongArguments_Zero(PyObject *args, const char *funcName, PyObject *info) +{ + setErrorAboutWrongArguments(args, funcName, info); + return 0; +} + +int returnWrongArguments_MinusOne(PyObject *args, const char *funcName, PyObject *info) +{ + setErrorAboutWrongArguments(args, funcName, info); + return -1; +} + +PyObject *returnFromRichCompare(PyObject *result) +{ + if (result && !PyErr_Occurred()) + return result; + Shiboken::Errors::setOperatorNotImplemented(); + return {}; +} + PyObject *checkInvalidArgumentCount(Py_ssize_t numArgs, Py_ssize_t minArgs, Py_ssize_t maxArgs) { PyObject *result = nullptr; @@ -773,20 +881,6 @@ PyObject *checkInvalidArgumentCount(Py_ssize_t numArgs, Py_ssize_t minArgs, Py_s return result; } -class FindBaseTypeVisitor : public HierarchyVisitor -{ -public: - explicit FindBaseTypeVisitor(PyTypeObject *typeToFind) : m_typeToFind(typeToFind) {} - - bool visit(PyTypeObject *node) override - { - return node == m_typeToFind; - } - -private: - PyTypeObject *m_typeToFind; -}; - std::vector<SbkObject *> splitPyObject(PyObject *pyObj) { std::vector<SbkObject *> result; @@ -827,8 +921,8 @@ bool isUserType(PyTypeObject *type) bool canCallConstructor(PyTypeObject *myType, PyTypeObject *ctorType) { - FindBaseTypeVisitor visitor(ctorType); - if (!walkThroughClassHierarchy(myType, &visitor)) { + auto findBasePred = [ctorType](PyTypeObject *type) { return type == ctorType; }; + if (!walkThroughBases(myType, findBasePred)) { PyErr_Format(PyExc_TypeError, "%s isn't a direct base class of %s", ctorType->tp_name, myType->tp_name); return false; } @@ -900,31 +994,34 @@ introduceWrapperType(PyObject *enclosingObject, const char *originalName, PyType_Spec *typeSpec, ObjectDestructor cppObjDtor, - PyTypeObject *baseType, - PyObject *baseTypes, + PyObject *bases, unsigned wrapperFlags) { - auto *base = baseType ? baseType : SbkObject_TypeF(); - typeSpec->slots[0].pfunc = reinterpret_cast<void *>(base); - auto *bases = baseTypes ? baseTypes : PyTuple_Pack(1, base); + assert(PySequence_Fast_GET_SIZE(bases) > 0); + typeSpec->slots[0].pfunc = PySequence_Fast_GET_ITEM(bases, 0); auto *type = SbkType_FromSpecBasesMeta(typeSpec, bases, SbkObjectType_TypeF()); - for (int i = 0; i < PySequence_Fast_GET_SIZE(bases); ++i) { - auto *st = reinterpret_cast<PyTypeObject *>(PySequence_Fast_GET_ITEM(bases, i)); - BindingManager::instance().addClassInheritance(st, type); - } - auto sotp = PepType_SOTP(type); if (wrapperFlags & DeleteInMainThread) sotp->delete_in_main_thread = 1; + sotp->type_behaviour = (wrapperFlags & Value) != 0 + ? BEHAVIOUR_VALUETYPE : BEHAVIOUR_OBJECTTYPE; setOriginalName(type, originalName); setDestructorFunction(type, cppObjDtor); auto *ob_type = reinterpret_cast<PyObject *>(type); - if (wrapperFlags & InnerClass) + if (wrapperFlags & InnerClass) { + // PYSIDE-2230: Instead of tp_dict, use the enclosing type. + // This stays interface compatible. + if (PyType_Check(enclosingObject)) { + AutoDecRef tpDict(PepType_GetDict(reinterpret_cast<PyTypeObject *>(enclosingObject))); + return PyDict_SetItemString(tpDict, typeName, ob_type) == 0 ? type : nullptr; + } + assert(PyDict_Check(enclosingObject)); return PyDict_SetItemString(enclosingObject, typeName, ob_type) == 0 ? type : nullptr; + } // PyModule_AddObject steals type's reference. Py_INCREF(ob_type); @@ -939,16 +1036,19 @@ introduceWrapperType(PyObject *enclosingObject, void setSubTypeInitHook(PyTypeObject *type, SubTypeInitHook func) { + assert(SbkObjectType_Check(type)); PepType_SOTP(type)->subtype_init = func; } void *getTypeUserData(PyTypeObject *type) { + assert(SbkObjectType_Check(type)); return PepType_SOTP(type)->user_data; } void setTypeUserData(PyTypeObject *type, void *userData, DeleteUserDataFunc d_func) { + assert(SbkObjectType_Check(type)); auto *sotp = PepType_SOTP(type); sotp->user_data = userData; sotp->d_func = d_func; @@ -971,6 +1071,26 @@ bool hasSpecialCastFunction(PyTypeObject *sbkType) return d != nullptr && d->mi_specialcast != nullptr; } +// Find whether base is a direct single line base class of type +// (no multiple inheritance), that is, a C++ pointer cast can safely be done. +static bool isDirectAncestor(PyTypeObject *type, PyTypeObject *base) +{ + if (type == base) + return true; + if (PyTuple_Size(type->tp_bases) == 0) + return false; + auto *sbkObjectType = SbkObject_TypeF(); + auto *firstBase = reinterpret_cast<PyTypeObject *>(PyTuple_GetItem(type->tp_bases, 0)); + return firstBase != sbkObjectType + && PyType_IsSubtype(type, sbkObjectType) != 0 + && isDirectAncestor(firstBase, base); +} + +bool canDowncastTo(PyTypeObject *baseType, PyTypeObject *targetType) +{ + return isDirectAncestor(targetType, baseType); +} + } // namespace ObjectType @@ -1059,9 +1179,7 @@ void callCppDestructors(SbkObject *pyObj) PyTypeObject *type = Py_TYPE(pyObj); auto *sotp = PepType_SOTP(type); if (sotp->is_multicpp) { - Shiboken::DtorAccumulatorVisitor visitor(pyObj); - Shiboken::walkThroughClassHierarchy(type, &visitor); - callDestructor(visitor.entries()); + callDestructor(getDestructorEntries(pyObj)); } else { Shiboken::ThreadStateSaver threadSaver; threadSaver.save(); @@ -1210,11 +1328,10 @@ void makeValid(SbkObject *self) // If has ref to other objects make all valid again if (self->d->referredObjects) { - RefCountMap &refCountMap = *(self->d->referredObjects); - RefCountMap::iterator iter; - for (auto it = refCountMap.begin(), end = refCountMap.end(); it != end; ++it) { - if (Shiboken::Object::checkType(it->second)) - makeValid(reinterpret_cast<SbkObject *>(it->second)); + const RefCountMap &refCountMap = *(self->d->referredObjects); + for (const auto &p : refCountMap) { + if (Shiboken::Object::checkType(p.second)) + makeValid(reinterpret_cast<SbkObject *>(p.second)); } } } @@ -1250,7 +1367,8 @@ bool setCppPointer(SbkObject *sbkObj, PyTypeObject *desiredType, void *cptr) const bool alreadyInitialized = sbkObj->d->cptr[idx] != nullptr; if (alreadyInitialized) - PyErr_SetString(PyExc_RuntimeError, "You can't initialize an object twice!"); + PyErr_Format(PyExc_RuntimeError, "You can't initialize an %s object in class %s twice!", + desiredType->tp_name, type->tp_name); else sbkObj->d->cptr[idx] = cptr; @@ -1342,20 +1460,67 @@ SbkObject *findColocatedChild(SbkObject *wrapper, return nullptr; } +// Legacy, for compatibility only. PyObject *newObject(PyTypeObject *instanceType, void *cptr, bool hasOwnership, bool isExactType, const char *typeName) { - // Try to find the exact type of cptr. - if (!isExactType) { - if (PyTypeObject *exactType = ObjectType::typeForTypeName(typeName)) - instanceType = exactType; - else - instanceType = BindingManager::instance().resolveType(&cptr, instanceType); + return isExactType + ? newObjectForType(instanceType, cptr, hasOwnership) + : newObjectWithHeuristics(instanceType, cptr, hasOwnership, typeName); +} + +static PyObject *newObjectWithHeuristicsHelper(PyTypeObject *instanceType, + PyTypeObject *exactType, + void *cptr, + bool hasOwnership) +{ + // Try to find the exact type of cptr. For hierarchies with + // non-virtual destructors, typeid() will return the base name. + // Try type discovery in these cases. + if (exactType == nullptr || exactType == instanceType) { + auto resolved = BindingManager::instance().findDerivedType(cptr, instanceType); + if (resolved.first != nullptr + && Shiboken::ObjectType::canDowncastTo(instanceType, resolved.first)) { + exactType = resolved.first; + cptr = resolved.second; + } } + return newObjectForType(exactType != nullptr ? exactType : instanceType, + cptr, hasOwnership); +} + +PyObject *newObjectForPointer(PyTypeObject *instanceType, + void *cptr, + bool hasOwnership, + const char *typeName) +{ + // Try to find the exact type of cptr. + PyTypeObject *exactType = ObjectType::typeForTypeName(typeName); + // PYSIDE-868: In case of multiple inheritance, (for example, + // a function returning a QPaintDevice * from a QWidget *), + // use instance type to avoid pointer offset errors. + return exactType != nullptr && !Shiboken::ObjectType::canDowncastTo(instanceType, exactType) + ? newObjectForType(instanceType, cptr, hasOwnership) + : newObjectWithHeuristicsHelper(instanceType, exactType, cptr, hasOwnership); +} + + +PyObject *newObjectWithHeuristics(PyTypeObject *instanceType, + void *cptr, + bool hasOwnership, + const char *typeName) +{ + return newObjectWithHeuristicsHelper(instanceType, + ObjectType::typeForTypeName(typeName), + cptr, hasOwnership); +} + +PyObject *newObjectForType(PyTypeObject *instanceType, void *cptr, bool hasOwnership) +{ bool shouldCreate = true; bool shouldRegister = true; SbkObject *self = nullptr; @@ -1564,7 +1729,7 @@ void deallocData(SbkObject *self, bool cleanup) } delete self->d; // PYSIDE-205: always delete d. Py_XDECREF(self->ob_dict); - Py_TYPE(self)->tp_free(self); + PepExt_TypeCallFree(reinterpret_cast<PyObject *>(self)); } void setTypeUserData(SbkObject *wrapper, void *userData, DeleteUserDataFunc d_func) @@ -1656,6 +1821,11 @@ static std::vector<PyTypeObject *> getBases(SbkObject *self) : std::vector<PyTypeObject *>(1, Py_TYPE(self)); } +static bool isValueType(SbkObject *self) +{ + return PepType_SOTP(Py_TYPE(self))->type_behaviour == BEHAVIOUR_VALUETYPE; +} + void _debugFormat(std::ostream &s, SbkObject *self) { assert(self); @@ -1679,6 +1849,8 @@ void _debugFormat(std::ostream &s, SbkObject *self) s << " [validCppObject]"; if (d->cppObjectCreated) s << " [wasCreatedByPython]"; + s << (isValueType(self) ? " [value]" : " [object]"); + if (d->parentInfo) { if (auto *parent = d->parentInfo->parent) s << ", parent=" << reinterpret_cast<PyObject *>(parent)->ob_type->tp_name @@ -1709,8 +1881,9 @@ std::string info(SbkObject *self) s << "hasOwnership...... " << bool(self->d->hasOwnership) << "\n" "containsCppWrapper " << self->d->containsCppWrapper << "\n" "validCppObject.... " << self->d->validCppObject << "\n" - "wasCreatedByPython " << self->d->cppObjectCreated << "\n"; - + "wasCreatedByPython " << self->d->cppObjectCreated << "\n" + "value...... " << isValueType(self) << "\n" + "reference count... " << reinterpret_cast<PyObject *>(self)->ob_refcnt << '\n'; if (self->d->parentInfo && self->d->parentInfo->parent) { s << "parent............ "; @@ -1728,17 +1901,17 @@ std::string info(SbkObject *self) } if (self->d->referredObjects && !self->d->referredObjects->empty()) { - Shiboken::RefCountMap &map = *self->d->referredObjects; + const Shiboken::RefCountMap &map = *self->d->referredObjects; s << "referred objects.. "; std::string lastKey; - for (auto it = map.begin(), end = map.end(); it != end; ++it) { - if (it->first != lastKey) { + for (const auto &p : map) { + if (p.first != lastKey) { if (!lastKey.empty()) s << " "; - s << '"' << it->first << "\" => "; - lastKey = it->first; + s << '"' << p.first << "\" => "; + lastKey = p.first; } - Shiboken::AutoDecRef obj(PyObject_Str(it->second)); + Shiboken::AutoDecRef obj(PyObject_Str(p.second)); s << String::toCString(obj) << ' '; } s << '\n'; diff --git a/sources/shiboken6/libshiboken/basewrapper.h b/sources/shiboken6/libshiboken/basewrapper.h index 86806c917..ec5545aea 100644 --- a/sources/shiboken6/libshiboken/basewrapper.h +++ b/sources/shiboken6/libshiboken/basewrapper.h @@ -38,28 +38,30 @@ LIBSHIBOKEN_API void SbkDeallocQAppWrapper(PyObject *pyObj); LIBSHIBOKEN_API void SbkDeallocWrapperWithPrivateDtor(PyObject *self); /// Function signature for the multiple inheritance information initializers that should be provided by classes with multiple inheritance. -typedef int *(*MultipleInheritanceInitFunction)(const void *); +using MultipleInheritanceInitFunction = int *(*)(const void *); /** * Special cast function is used to correctly cast an object when it's * part of a multiple inheritance hierarchy. * The implementation of this function is auto generated by the generator and you don't need to care about it. */ -typedef void *(*SpecialCastFunction)(void *, PyTypeObject *); -typedef PyTypeObject *(*TypeDiscoveryFunc)(void *, PyTypeObject *); -typedef void *(*TypeDiscoveryFuncV2)(void *, PyTypeObject *); +using SpecialCastFunction = void *(*)(void *, PyTypeObject *); +using TypeDiscoveryFunc = PyTypeObject *(*)(void *, PyTypeObject *); +using TypeDiscoveryFuncV2 = void *(*)(void *, PyTypeObject *); // Used in userdata dealloc function -typedef void (*DeleteUserDataFunc)(void *); +using DeleteUserDataFunc = void (*)(void *); -typedef void (*ObjectDestructor)(void *); +using ObjectDestructor = void (*)(void *); -typedef void (*SubTypeInitHook)(PyTypeObject *, PyObject *, PyObject *); +using SubTypeInitHook = void (*)(PyTypeObject *, PyObject *, PyObject *); /// PYSIDE-1019: Set the function to select the current feature. /// Return value is the previous content. -typedef void (*SelectableFeatureHook)(PyTypeObject *); +using SelectableFeatureHook = void (*)(PyTypeObject *); +using SelectableFeatureCallback = void (*)(bool); LIBSHIBOKEN_API SelectableFeatureHook initSelectableFeature(SelectableFeatureHook func); +LIBSHIBOKEN_API void setSelectableFeatureCallback(SelectableFeatureCallback func); /// PYSIDE-1626: Enforcing a context switch without further action. LIBSHIBOKEN_API void SbkObjectType_UpdateFeature(PyTypeObject *type); @@ -72,7 +74,7 @@ LIBSHIBOKEN_API void SbkObjectType_SetPropertyStrings(PyTypeObject *type, const LIBSHIBOKEN_API void SbkObjectType_SetEnumFlagInfo(PyTypeObject *type, const char **strings); /// PYSIDE-1470: Set the function to kill a Q*Application. -typedef void(*DestroyQAppHook)(); +using DestroyQAppHook = void(*)(); LIBSHIBOKEN_API void setDestroyQApplication(DestroyQAppHook func); /// PYSIDE-535: Use the C API in PyPy instead of `op->ob_dict`, directly (borrowed ref) @@ -110,6 +112,9 @@ LIBSHIBOKEN_API PyObject *FallbackRichCompare(PyObject *self, PyObject *other, i /// PYSIDE-1970: Be easily able to see what is happening in the running code. LIBSHIBOKEN_API void disassembleFrame(const char *marker); +/// PYSIDE-2230: Check if an object is an SbkObject. +LIBSHIBOKEN_API bool SbkObjectType_Check(PyTypeObject *type); + } // extern "C" namespace Shiboken @@ -130,11 +135,25 @@ void callCppDestructor(void *cptr) delete reinterpret_cast<T *>(cptr); } -// setErrorAboutWrongArguments now gets overload information from the signature module. -// The extra info argument can contain additional data about the error. +/// setErrorAboutWrongArguments now gets overload information from the signature module. +/// The extra info argument can contain additional data about the error. LIBSHIBOKEN_API void setErrorAboutWrongArguments(PyObject *args, const char *funcName, PyObject *info); +/// Return values for the different retun variants. +/// This is used instead of goto. +LIBSHIBOKEN_API PyObject *returnWrongArguments(PyObject *args, const char *funcName, + PyObject *info); + +LIBSHIBOKEN_API int returnWrongArguments_Zero(PyObject *args, const char *funcName, + PyObject *info); + +LIBSHIBOKEN_API int returnWrongArguments_MinusOne(PyObject *args, const char *funcName, + PyObject *info); + +/// A simple special version for the end of rich comparison. +LIBSHIBOKEN_API PyObject *returnFromRichCompare(PyObject *result); + // Return error information object if the argument count is wrong LIBSHIBOKEN_API PyObject *checkInvalidArgumentCount(Py_ssize_t numArgs, Py_ssize_t minArgs, @@ -181,14 +200,15 @@ LIBSHIBOKEN_API const char *getOriginalName(PyTypeObject *self); LIBSHIBOKEN_API void setTypeDiscoveryFunctionV2(PyTypeObject *self, TypeDiscoveryFuncV2 func); LIBSHIBOKEN_API void copyMultipleInheritance(PyTypeObject *self, PyTypeObject *other); LIBSHIBOKEN_API void setMultipleInheritanceFunction(PyTypeObject *self, MultipleInheritanceInitFunction func); -LIBSHIBOKEN_API MultipleInheritanceInitFunction getMultipleInheritanceFunction(PyTypeObject *self); +LIBSHIBOKEN_API MultipleInheritanceInitFunction getMultipleInheritanceFunction(PyTypeObject *type); LIBSHIBOKEN_API void setDestructorFunction(PyTypeObject *self, ObjectDestructor func); enum WrapperFlags { InnerClass = 0x1, - DeleteInMainThread = 0x2 + DeleteInMainThread = 0x2, + Value = 0x4 }; /** @@ -209,13 +229,12 @@ enum WrapperFlags * \returns true if the initialization went fine, false otherwise. */ LIBSHIBOKEN_API PyTypeObject *introduceWrapperType(PyObject *enclosingObject, - const char *typeName, - const char *originalName, - PyType_Spec *typeSpec, - ObjectDestructor cppObjDtor, - PyTypeObject *baseType, - PyObject *baseTypes, - unsigned wrapperFlags = 0); + const char *typeName, + const char *originalName, + PyType_Spec *typeSpec, + ObjectDestructor cppObjDtor, + PyObject *bases, + unsigned wrapperFlags = 0); /** * Set the subtype init hook for a type. @@ -246,6 +265,14 @@ LIBSHIBOKEN_API PyTypeObject *typeForTypeName(const char *typeName); * \since 5.12 */ LIBSHIBOKEN_API bool hasSpecialCastFunction(PyTypeObject *sbkType); + +/// Returns whether a C++ pointer of \p baseType can be safely downcast +/// to \p targetType (base is a direct, single line base class of targetType). +/// (is a direct, single-line inheritance) +/// \param baseType Python type of base class +/// \param targetType Python type of derived class +/// \since 6.8 +LIBSHIBOKEN_API bool canDowncastTo(PyTypeObject *baseType, PyTypeObject *targetType); } namespace Object { @@ -278,7 +305,8 @@ LIBSHIBOKEN_API SbkObject *findColocatedChild(SbkObject *wrapper, const PyTypeObject *instanceType); /** - * Bind a C++ object to Python. + * Bind a C++ object to Python. Forwards to + * newObjectWithHeuristics(), newObjectForType() depending on \p isExactType. * \param instanceType equivalent Python type for the C++ object. * \param hasOwnership if true, Python will try to delete the underlying C++ object when there's no more refs. * \param isExactType if false, Shiboken will use some heuristics to detect the correct Python type of this C++ @@ -292,6 +320,40 @@ LIBSHIBOKEN_API PyObject *newObject(PyTypeObject *instanceType, bool isExactType = false, const char *typeName = nullptr); +/// Bind a C++ object to Python for polymorphic pointers. Calls +/// newObjectWithHeuristics() with an additional check for multiple +/// inheritance, in which case it will fall back to instanceType. +/// \param instanceType Equivalent Python type for the C++ object. +/// \param hasOwnership if true, Python will try to delete the underlying C++ object +/// when there's no more refs. +/// \param typeName If non-null, this will be used as helper to find the correct +/// Python type for this object (obtained by typeid().name(). +LIBSHIBOKEN_API PyObject *newObjectForPointer(PyTypeObject *instanceType, + void *cptr, + bool hasOwnership = true, + const char *typeName = nullptr); + +/// Bind a C++ object to Python using some heuristics to detect the correct +/// Python type of this C++ object. In any case \p instanceType must be provided; +/// it'll be used as search starting point and as fallback. +/// \param instanceType Equivalent Python type for the C++ object. +/// \param hasOwnership if true, Python will try to delete the underlying C++ object +/// C++ object when there are no more references. +/// when there's no more refs. +/// \param typeName If non-null, this will be used as helper to find the correct +/// Python type for this object (obtained by typeid().name(). +LIBSHIBOKEN_API PyObject *newObjectWithHeuristics(PyTypeObject *instanceType, + void *cptr, + bool hasOwnership = true, + const char *typeName = nullptr); + +/// Bind a C++ object to Python using the given type. +/// \param instanceType Equivalent Python type for the C++ object. +/// \param hasOwnership if true, Python will try to delete the underlying +/// C++ object when there are no more references. +LIBSHIBOKEN_API PyObject *newObjectForType(PyTypeObject *instanceType, + void *cptr, bool hasOwnership = true); + /** * Changes the valid flag of a PyObject, invalid objects will raise an exception when someone tries to access it. */ diff --git a/sources/shiboken6/libshiboken/basewrapper_p.h b/sources/shiboken6/libshiboken/basewrapper_p.h index fc3cc321c..fb9140793 100644 --- a/sources/shiboken6/libshiboken/basewrapper_p.h +++ b/sources/shiboken6/libshiboken/basewrapper_p.h @@ -30,14 +30,12 @@ using ChildrenList = std::set<SbkObject *>; /// Structure used to store information about object parent and children. struct ParentInfo { - /// Default ctor. - ParentInfo() : parent(nullptr), hasWrapperRef(false) {} /// Pointer to parent object. - SbkObject *parent; + SbkObject *parent = nullptr; /// List of object children. ChildrenList children; /// has internal ref - bool hasWrapperRef; + bool hasWrapperRef = false; }; } // namespace Shiboken @@ -51,6 +49,12 @@ extern "C" */ struct SbkObjectPrivate { + SbkObjectPrivate() noexcept = default; + SbkObjectPrivate(const SbkObjectPrivate &) = delete; + SbkObjectPrivate(SbkObjectPrivate &&o) = delete; + SbkObjectPrivate &operator=(const SbkObjectPrivate &) = delete; + SbkObjectPrivate &operator=(SbkObjectPrivate &&o) = delete; + /// Pointer to the C++ class. void ** cptr; /// True when Python is responsible for freeing the used memory. @@ -110,7 +114,8 @@ struct SbkObjectTypePrivate /// True if this type holds two or more C++ instances, e.g.: a Python class which inherits from two C++ classes. unsigned int is_multicpp : 1; - /// True if this type was defined by the user. + /// True if this type was defined by the user (a class written in Python inheriting + /// a class provided by a Shiboken binding). unsigned int is_user_type : 1; /// Tells is the type is a value type or an object-type, see BEHAVIOUR_ *constants. unsigned int type_behaviour : 2; @@ -138,107 +143,7 @@ struct DestructorEntry **/ std::vector<SbkObject *> splitPyObject(PyObject *pyObj); -/** -* Visitor class used by walkOnClassHierarchy function. -*/ -class HierarchyVisitor -{ -public: - HierarchyVisitor(const HierarchyVisitor &) = delete; - HierarchyVisitor(HierarchyVisitor &&) = delete; - HierarchyVisitor &operator=(const HierarchyVisitor &) = delete; - HierarchyVisitor &operator=(HierarchyVisitor &&) = delete; - - HierarchyVisitor(); - virtual ~HierarchyVisitor(); - - virtual bool visit(PyTypeObject *node) = 0; // return true to terminate -}; - -class BaseCountVisitor : public HierarchyVisitor -{ -public: - bool visit(PyTypeObject *) override; - - int count() const { return m_count; } - -private: - int m_count = 0; -}; - -class BaseAccumulatorVisitor : public HierarchyVisitor -{ -public: - using Result = std::vector<PyTypeObject *>; - - bool visit(PyTypeObject *node) override; - - Result bases() const { return m_bases; } - -private: - Result m_bases; -}; - -class GetIndexVisitor : public HierarchyVisitor -{ -public: - explicit GetIndexVisitor(PyTypeObject *desiredType) : m_desiredType(desiredType) {} - - bool visit(PyTypeObject *node) override; - - int index() const { return m_index; } - -private: - int m_index = -1; - PyTypeObject *m_desiredType; -}; - -/// Collect destructors and C++ instances of each C++ object held by a Python -/// object -class DtorAccumulatorVisitor : public HierarchyVisitor -{ -public: - explicit DtorAccumulatorVisitor(SbkObject *pyObj) : m_pyObject(pyObj) {} - - bool visit(PyTypeObject *node) override; - - using DestructorEntries = std::vector<DestructorEntry>; - - const DestructorEntries &entries() const { return m_entries; } - -private: - DestructorEntries m_entries; - SbkObject *m_pyObject; -}; - -/// \internal Internal function used to walk on classes inheritance trees. -/** -* Walk on class hierarchy using a DFS algorithm. -* For each pure Shiboken type found, HierarchyVisitor::visit is called and the algorithm -* considers all children of this type as visited. -*/ -bool walkThroughClassHierarchy(PyTypeObject *currentType, HierarchyVisitor *visitor); - -inline int getTypeIndexOnHierarchy(PyTypeObject *baseType, PyTypeObject *desiredType) -{ - GetIndexVisitor visitor(desiredType); - walkThroughClassHierarchy(baseType, &visitor); - return visitor.index(); -} - -inline int getNumberOfCppBaseClasses(PyTypeObject *baseType) -{ - BaseCountVisitor visitor; - walkThroughClassHierarchy(baseType, &visitor); - return visitor.count(); -} - -inline std::vector<PyTypeObject *> getCppBaseClasses(PyTypeObject *baseType) -{ - BaseAccumulatorVisitor visitor; - walkThroughClassHierarchy(baseType, &visitor); - return visitor.bases(); -} +int getNumberOfCppBaseClasses(PyTypeObject *baseType); namespace Object { diff --git a/sources/shiboken6/libshiboken/bindingmanager.cpp b/sources/shiboken6/libshiboken/bindingmanager.cpp index 9d74e9721..83c927ae5 100644 --- a/sources/shiboken6/libshiboken/bindingmanager.cpp +++ b/sources/shiboken6/libshiboken/bindingmanager.cpp @@ -6,101 +6,162 @@ #include "basewrapper_p.h" #include "bindingmanager.h" #include "gilstate.h" +#include "helper.h" +#include "sbkmodule.h" #include "sbkstring.h" #include "sbkstaticstrings.h" #include "sbkfeature_base.h" #include "debugfreehook.h" #include <cstddef> +#include <cstring> #include <fstream> +#include <iostream> #include <mutex> +#include <string_view> #include <unordered_map> +#include <unordered_set> + +// GraphNode for the dependency graph. It keeps a pointer to +// the TypeInitStruct to be able to lazily create the type and hashes +// by the full type name. +struct GraphNode +{ + explicit GraphNode(Shiboken::Module::TypeInitStruct *i) : name(i->fullName), initStruct(i) {} + explicit GraphNode(const char *n) : name(n), initStruct(nullptr) {} // Only for searching + + std::string_view name; + Shiboken::Module::TypeInitStruct *initStruct; + + friend bool operator==(const GraphNode &n1, const GraphNode &n2) { return n1.name == n2.name; } + friend bool operator!=(const GraphNode &n1, const GraphNode &n2) { return n1.name != n2.name; } +}; + +template <> +struct std::hash<GraphNode> { + size_t operator()(const GraphNode &n) const noexcept + { + return std::hash<std::string_view>{}(n.name); + } +}; namespace Shiboken { using WrapperMap = std::unordered_map<const void *, SbkObject *>; -class Graph +template <class NodeType> +class BaseGraph { public: - using NodeList = std::vector<PyTypeObject *>; - using Edges = std::unordered_map<PyTypeObject *, NodeList>; + using NodeList = std::vector<NodeType>; + using NodeSet = std::unordered_set<NodeType>; + + using Edges = std::unordered_map<NodeType, NodeList>; Edges m_edges; - Graph() = default; + BaseGraph() = default; - void addEdge(PyTypeObject *from, PyTypeObject *to) + void addEdge(NodeType from, NodeType to) { m_edges[from].push_back(to); } -#ifndef NDEBUG - void dumpDotGraph() + NodeSet nodeSet() const { - std::ofstream file("/tmp/shiboken_graph.dot"); - - file << "digraph D {\n"; - - for (auto i = m_edges.begin(), end = m_edges.end(); i != end; ++i) { - auto *node1 = i->first; - const NodeList &nodeList = i->second; - for (const PyTypeObject *o : nodeList) { - auto *node2 = o; - file << '"' << node2->tp_name << "\" -> \"" - << node1->tp_name << "\"\n"; - } + NodeSet result; + for (const auto &p : m_edges) { + result.insert(p.first); + for (const auto node2 : p.second) + result.insert(node2); } - file << "}\n"; + return result; } -#endif +}; + +class Graph : public BaseGraph<GraphNode> +{ +public: + using TypeCptrPair = BindingManager::TypeCptrPair; - PyTypeObject *identifyType(void **cptr, PyTypeObject *type, PyTypeObject *baseType) const + TypeCptrPair identifyType(void *cptr, PyTypeObject *type, PyTypeObject *baseType) const { - auto edgesIt = m_edges.find(type); - if (edgesIt != m_edges.end()) { - const NodeList &adjNodes = m_edges.find(type)->second; - for (PyTypeObject *node : adjNodes) { - PyTypeObject *newType = identifyType(cptr, node, baseType); - if (newType) - return newType; - } - } - void *typeFound = nullptr; - auto *sotp = PepType_SOTP(type); - if (sotp->type_discovery) - typeFound = sotp->type_discovery(*cptr, baseType); - if (typeFound) { - // This "typeFound != type" is needed for backwards compatibility with old modules using a newer version of - // libshiboken because old versions of type_discovery function used to return a PyTypeObject *instead of - // a possible variation of the C++ instance pointer (*cptr). - if (typeFound != type) - *cptr = typeFound; - return type; - } - return nullptr; + return identifyType(cptr, GraphNode(type->tp_name), type, baseType); } -}; + bool dumpTypeGraph(const char *fileName) const; -#ifndef NDEBUG -static void showWrapperMap(const WrapperMap &wrapperMap) +private: + TypeCptrPair identifyType(void *cptr, const GraphNode &typeNode, PyTypeObject *type, + PyTypeObject *baseType) const; +}; + +Graph::TypeCptrPair Graph::identifyType(void *cptr, + const GraphNode &typeNode, PyTypeObject *type, + PyTypeObject *baseType) const { - if (Py_VerboseFlag > 0) { - fprintf(stderr, "-------------------------------\n"); - fprintf(stderr, "WrapperMap: %p (size: %d)\n", &wrapperMap, (int) wrapperMap.size()); - for (auto it = wrapperMap.begin(), end = wrapperMap.end(); it != end; ++it) { - const SbkObject *sbkObj = it->second; - fprintf(stderr, "key: %p, value: %p (%s, refcnt: %d)\n", it->first, - static_cast<const void *>(sbkObj), - (Py_TYPE(sbkObj))->tp_name, - int(reinterpret_cast<const PyObject *>(sbkObj)->ob_refcnt)); + assert(typeNode.initStruct != nullptr || type != nullptr); + auto edgesIt = m_edges.find(typeNode); + if (edgesIt != m_edges.end()) { + const NodeList &adjNodes = edgesIt->second; + for (const auto &node : adjNodes) { + auto newType = identifyType(cptr, node, nullptr, baseType); + if (newType.first != nullptr) + return newType; } - fprintf(stderr, "-------------------------------\n"); } + + if (type == nullptr) { + if (typeNode.initStruct->type == nullptr) // Layzily create type + type = Shiboken::Module::get(*typeNode.initStruct); + else + type = typeNode.initStruct->type; + } + + auto *sotp = PepType_SOTP(type); + if (sotp->type_discovery != nullptr) { + if (void *derivedCPtr = sotp->type_discovery(cptr, baseType)) + return {type, derivedCPtr}; + } + return {nullptr, nullptr}; +} + +static void formatDotNode(std::string_view name, std::ostream &file) +{ + auto lastDot = name.rfind('.'); + file << " \"" << name << "\" [ label="; + if (lastDot != std::string::npos) { + file << '"' << name.substr(lastDot + 1) << "\" tooltip=\"" + << name.substr(0, lastDot) << '"'; + } else { + file << '"' << name << '"'; + } + file << " ]\n"; +} + +bool Graph::dumpTypeGraph(const char *fileName) const +{ + std::ofstream file(fileName); + if (!file.good()) + return false; + + file << "digraph D {\n"; + + // Define nodes with short names + for (const auto &node : nodeSet()) + formatDotNode(node.name, file); + + // Write edges + for (const auto &p : m_edges) { + const auto &node1 = p.first; + const NodeList &nodeList = p.second; + for (const auto &node2 : nodeList) + file << " \"" << node2.name << "\" -> \"" << node1.name << "\"\n"; + } + file << "}\n"; + return true; } -#endif struct BindingManager::BindingManagerPrivate { using DestructorEntries = std::vector<DestructorEntry>; @@ -113,20 +174,19 @@ struct BindingManager::BindingManagerPrivate { std::recursive_mutex wrapperMapLock; Graph classHierarchy; DestructorEntries deleteInMainThread; - bool destroying; - BindingManagerPrivate() : destroying(false) {} - bool releaseWrapper(void *cptr, SbkObject *wrapper); - void assignWrapper(SbkObject *wrapper, const void *cptr); + bool releaseWrapper(void *cptr, SbkObject *wrapper, const int *bases = nullptr); + bool releaseWrapperHelper(void *cptr, SbkObject *wrapper); + void assignWrapper(SbkObject *wrapper, const void *cptr, const int *bases = nullptr); + void assignWrapperHelper(SbkObject *wrapper, const void *cptr); }; -bool BindingManager::BindingManagerPrivate::releaseWrapper(void *cptr, SbkObject *wrapper) +inline bool BindingManager::BindingManagerPrivate::releaseWrapperHelper(void *cptr, SbkObject *wrapper) { // The wrapper argument is checked to ensure that the correct wrapper is released. // Returns true if the correct wrapper is found and released. // If wrapper argument is NULL, no such check is performed. - std::lock_guard<std::recursive_mutex> guard(wrapperMapLock); auto iter = wrapperMapper.find(cptr); if (iter != wrapperMapper.end() && (wrapper == nullptr || iter->second == wrapper)) { wrapperMapper.erase(iter); @@ -135,15 +195,41 @@ bool BindingManager::BindingManagerPrivate::releaseWrapper(void *cptr, SbkObject return false; } -void BindingManager::BindingManagerPrivate::assignWrapper(SbkObject *wrapper, const void *cptr) +bool BindingManager::BindingManagerPrivate::releaseWrapper(void *cptr, SbkObject *wrapper, + const int *bases) { assert(cptr); std::lock_guard<std::recursive_mutex> guard(wrapperMapLock); + const bool result = releaseWrapperHelper(cptr, wrapper); + if (bases != nullptr) { + auto *base = static_cast<uint8_t *>(cptr); + for (const auto *offset = bases; *offset != -1; ++offset) + releaseWrapperHelper(base + *offset, wrapper); + } + return result; +} + +inline void BindingManager::BindingManagerPrivate::assignWrapperHelper(SbkObject *wrapper, + const void *cptr) +{ auto iter = wrapperMapper.find(cptr); if (iter == wrapperMapper.end()) wrapperMapper.insert(std::make_pair(cptr, wrapper)); } +void BindingManager::BindingManagerPrivate::assignWrapper(SbkObject *wrapper, const void *cptr, + const int *bases) +{ + assert(cptr); + std::lock_guard<std::recursive_mutex> guard(wrapperMapLock); + assignWrapperHelper(wrapper, cptr); + if (bases != nullptr) { + const auto *base = static_cast<const uint8_t *>(cptr); + for (const auto *offset = bases; *offset != -1; ++offset) + assignWrapperHelper(wrapper, base + *offset); + } +} + BindingManager::BindingManager() { m_d = new BindingManager::BindingManagerPrivate; @@ -159,7 +245,8 @@ BindingManager::~BindingManager() debugRemoveFreeHook(); #endif #ifndef NDEBUG - showWrapperMap(m_d->wrapperMapper); + if (Shiboken::pyVerbose() > 0) + dumpWrapperMap(); #endif /* Cleanup hanging references. We just invalidate them as when * the BindingManager is being destroyed the interpreter is alredy @@ -195,15 +282,7 @@ void BindingManager::registerWrapper(SbkObject *pyObj, void *cptr) if (d->mi_init && !d->mi_offsets) d->mi_offsets = d->mi_init(cptr); - m_d->assignWrapper(pyObj, cptr); - if (d->mi_offsets) { - int *offset = d->mi_offsets; - while (*offset != -1) { - if (*offset > 0) - m_d->assignWrapper(pyObj, reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(cptr) + *offset)); - offset++; - } - } + m_d->assignWrapper(pyObj, cptr, d->mi_offsets); } void BindingManager::releaseWrapper(SbkObject *sbkObj) @@ -213,17 +292,10 @@ void BindingManager::releaseWrapper(SbkObject *sbkObj) int numBases = ((d && d->is_multicpp) ? getNumberOfCppBaseClasses(Py_TYPE(sbkObj)) : 1); void ** cptrs = reinterpret_cast<SbkObject *>(sbkObj)->d->cptr; + const int *mi_offsets = d != nullptr ? d->mi_offsets : nullptr; for (int i = 0; i < numBases; ++i) { - auto *cptr = reinterpret_cast<unsigned char *>(cptrs[i]); - m_d->releaseWrapper(cptr, sbkObj); - if (d && d->mi_offsets) { - int *offset = d->mi_offsets; - while (*offset != -1) { - if (*offset > 0) - m_d->releaseWrapper(reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(cptr) + *offset), sbkObj); - offset++; - } - } + if (cptrs[i] != nullptr) + m_d->releaseWrapper(cptrs[i], sbkObj, mi_offsets); } sbkObj->d->validCppObject = false; } @@ -256,7 +328,7 @@ PyObject *BindingManager::getOverride(const void *cptr, SbkObject *wrapper = retrieveWrapper(cptr); // The refcount can be 0 if the object is dieing and someone called // a virtual method from the destructor - if (!wrapper || reinterpret_cast<const PyObject *>(wrapper)->ob_refcnt == 0) + if (!wrapper || Py_REFCNT(reinterpret_cast<const PyObject *>(wrapper)) == 0) return nullptr; // PYSIDE-1626: Touch the type to initiate switching early. @@ -276,6 +348,8 @@ PyObject *BindingManager::getOverride(const void *cptr, auto *obWrapper = reinterpret_cast<PyObject *>(wrapper); auto *wrapper_dict = SbkObject_GetDict_NoRef(obWrapper); if (PyObject *method = PyDict_GetItem(wrapper_dict, pyMethodName)) { + // Note: This special case was implemented for duck-punching, which happens + // in the instance dict. It does not work with properties. Py_INCREF(method); return method; } @@ -317,36 +391,53 @@ PyObject *BindingManager::getOverride(const void *cptr, } if (method != nullptr) { - PyObject *defaultMethod; + PyObject *defaultMethod{}; PyObject *mro = Py_TYPE(wrapper)->tp_mro; int size = PyTuple_GET_SIZE(mro); + bool defaultFound = false; // The first class in the mro (index 0) is the class being checked and it should not be tested. // The last class in the mro (size - 1) is the base Python object class which should not be tested also. for (int idx = 1; idx < size - 1; ++idx) { auto *parent = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(mro, idx)); - if (parent->tp_dict) { - defaultMethod = PyDict_GetItem(parent->tp_dict, pyMethodName); - if (defaultMethod && function != defaultMethod) - return method; + AutoDecRef tpDict(PepType_GetDict(parent)); + auto *parentDict = tpDict.object(); + if (parentDict) { + defaultMethod = PyDict_GetItem(parentDict, pyMethodName); + if (defaultMethod) { + defaultFound = true; + if (function != defaultMethod) + return method; + } } } - + // PYSIDE-2255: If no default method was found, use the method. + if (!defaultFound) + return method; Py_DECREF(method); } return nullptr; } -void BindingManager::addClassInheritance(PyTypeObject *parent, PyTypeObject *child) +void BindingManager::addClassInheritance(Module::TypeInitStruct *parent, + Module::TypeInitStruct *child) { - m_d->classHierarchy.addEdge(parent, child); + m_d->classHierarchy.addEdge(GraphNode(parent), GraphNode(child)); } +BindingManager::TypeCptrPair BindingManager::findDerivedType(void *cptr, PyTypeObject *type) const +{ + return m_d->classHierarchy.identifyType(cptr, type, type); +} + +// FIXME PYSIDE7: remove, just for compatibility PyTypeObject *BindingManager::resolveType(void **cptr, PyTypeObject *type) { - PyTypeObject *identifiedType = m_d->classHierarchy.identifyType(cptr, type, type); - return identifiedType ? identifiedType : type; + auto result = findDerivedType(*cptr, type); + if (result.second != nullptr) + *cptr = result.second; + return result.first != nullptr ? result.first : type; } std::set<PyObject *> BindingManager::getAllPyObjects() @@ -364,10 +455,94 @@ std::set<PyObject *> BindingManager::getAllPyObjects() void BindingManager::visitAllPyObjects(ObjectVisitor visitor, void *data) { WrapperMap copy = m_d->wrapperMapper; - for (auto it = copy.begin(); it != copy.end(); ++it) { - if (hasWrapper(it->first)) - visitor(it->second, data); + for (const auto &p : copy) { + if (hasWrapper(p.first)) + visitor(p.second, data); + } +} + +bool BindingManager::dumpTypeGraph(const char *fileName) const +{ + return m_d->classHierarchy.dumpTypeGraph(fileName); +} + +void BindingManager::dumpWrapperMap() +{ + const auto &wrapperMap = m_d->wrapperMapper; + std::cerr << "-------------------------------\n" + << "WrapperMap size: " << wrapperMap.size() << " Types: " + << m_d->classHierarchy.nodeSet().size() << '\n'; + for (auto it = wrapperMap.begin(), end = wrapperMap.end(); it != end; ++it) { + const SbkObject *sbkObj = it->second; + std::cerr << "key: " << it->first << ", value: " + << static_cast<const void *>(sbkObj) << " (" + << (Py_TYPE(sbkObj))->tp_name << ", refcnt: " + << Py_REFCNT(reinterpret_cast<const PyObject *>(sbkObj)) << ")\n"; + } + std::cerr << "-------------------------------\n"; +} + +static bool isPythonType(PyTypeObject *type) +{ + // This is a type which should be called by multiple inheritance. + // It is either a pure Python type or a derived PySide type. + return !ObjectType::checkType(type) || ObjectType::isUserType(type); +} + +bool callInheritedInit(PyObject *self, PyObject *args, PyObject *kwds, + const char *fullName) +{ + using Shiboken::AutoDecRef; + + static PyObject *const _init = String::createStaticString("__init__"); + static PyObject *objectInit = + PyObject_GetAttr(reinterpret_cast<PyObject *>(&PyBaseObject_Type), _init); + + // A native C++ self cannot have multiple inheritance. + if (!Object::isUserType(self)) + return false; + + auto *startType = Py_TYPE(self); + auto *mro = startType->tp_mro; + Py_ssize_t idx, n = PyTuple_GET_SIZE(mro); + auto classNameLen = std::strrchr(fullName, '.') - fullName; + /* No need to check the last one: it's gonna be skipped anyway. */ + for (idx = 0; idx + 1 < n; ++idx) { + auto *lookType = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(mro, idx)); + const char *lookName = lookType->tp_name; + auto lookLen = long(std::strlen(lookName)); + if (std::strncmp(lookName, fullName, classNameLen) == 0 && lookLen == classNameLen) + break; + } + // We are now at the first non-Python class `QObject`. + // mro: ('C', 'A', 'QObject', 'Object', 'B', 'object') + // We want to catch class `B` and call its `__init__`. + for (idx += 1; idx + 1 < n; ++idx) { + auto *t = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(mro, idx)); + if (isPythonType(t)) + break; } + if (idx >= n) + return false; + + auto *obSubType = PyTuple_GET_ITEM(mro, idx); + auto *subType = reinterpret_cast<PyTypeObject *>(obSubType); + if (subType == &PyBaseObject_Type) + return false; + AutoDecRef func(PyObject_GetAttr(obSubType, _init)); + // PYSIDE-2654: If this has no implementation then we get object.__init__ + // but that is the same case like above. + if (func == objectInit) + return false; + // PYSIDE-2294: We need to explicitly ignore positional args in a mixin class. + SBK_UNUSED(args); + AutoDecRef newArgs(PyTuple_New(1)); + auto *newArgsOb = newArgs.object(); + Py_INCREF(self); + PyTuple_SET_ITEM(newArgsOb, 0, self); + // Note: This can fail, so please always check the error status. + AutoDecRef result(PyObject_Call(func, newArgs, kwds)); + return true; } } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/bindingmanager.h b/sources/shiboken6/libshiboken/bindingmanager.h index e299dad96..54c4e486a 100644 --- a/sources/shiboken6/libshiboken/bindingmanager.h +++ b/sources/shiboken6/libshiboken/bindingmanager.h @@ -5,17 +5,23 @@ #define BINDINGMANAGER_H #include "sbkpython.h" -#include <set> #include "shibokenmacros.h" +#include <set> +#include <utility> + struct SbkObject; namespace Shiboken { +namespace Module { +struct TypeInitStruct; +} + struct DestructorEntry; -typedef void (*ObjectVisitor)(SbkObject *, void *); +using ObjectVisitor = void (*)(SbkObject *, void *); class LIBSHIBOKEN_API BindingManager { @@ -38,7 +44,15 @@ public: SbkObject *retrieveWrapper(const void *cptr); PyObject *getOverride(const void *cptr, PyObject *nameCache[], const char *methodName); - void addClassInheritance(PyTypeObject *parent, PyTypeObject *child); + void addClassInheritance(Module::TypeInitStruct *parent, Module::TypeInitStruct *child); + /// Try to find the correct type of cptr via type discovery knowing that it's at least + /// of type \p type. If a derived class is found, it returns a cptr cast to the type + /// (which may be different in case of multiple inheritance. + /// \param cptr a pointer to the instance of type \p type + /// \param type type of cptr + using TypeCptrPair = std::pair<PyTypeObject *, void *>; + TypeCptrPair findDerivedType(void *cptr, PyTypeObject *type) const; + /** * Try to find the correct type of *cptr knowing that it's at least of type \p type. * In case of multiple inheritance this function may change the contents of cptr. @@ -46,7 +60,7 @@ public: * \param type type of *cptr * \warning This function is slow, use it only as last resort. */ - PyTypeObject *resolveType(void **cptr, PyTypeObject *type); + [[deprecated]] PyTypeObject *resolveType(void **cptr, PyTypeObject *type); std::set<PyObject *> getAllPyObjects(); @@ -59,6 +73,9 @@ public: */ void visitAllPyObjects(ObjectVisitor visitor, void *data); + bool dumpTypeGraph(const char *fileName) const; + void dumpWrapperMap(); + private: ~BindingManager(); BindingManager(); @@ -67,6 +84,9 @@ private: BindingManagerPrivate *m_d; }; +LIBSHIBOKEN_API bool callInheritedInit(PyObject *self, PyObject *args, PyObject *kwds, + const char *fullName); + } // namespace Shiboken #endif // BINDINGMANAGER_H diff --git a/sources/shiboken6/libshiboken/bufferprocs_py37.h b/sources/shiboken6/libshiboken/bufferprocs_py37.h index 06b42cabd..e16194e50 100644 --- a/sources/shiboken6/libshiboken/bufferprocs_py37.h +++ b/sources/shiboken6/libshiboken/bufferprocs_py37.h @@ -67,8 +67,8 @@ typedef struct bufferinfo { void *internal; } Pep_buffer; -typedef int (*getbufferproc)(PyObject *, Pep_buffer *, int); -typedef void (*releasebufferproc)(PyObject *, Pep_buffer *); +using getbufferproc =int (*)(PyObject *, Pep_buffer *, int); +using releasebufferproc = void (*)(PyObject *, Pep_buffer *); /* Maximum number of dimensions */ #define PyBUF_MAX_NDIM 64 diff --git a/sources/shiboken6/libshiboken/debugfreehook.cpp b/sources/shiboken6/libshiboken/debugfreehook.cpp index 1a80a2514..13df6bd6c 100644 --- a/sources/shiboken6/libshiboken/debugfreehook.cpp +++ b/sources/shiboken6/libshiboken/debugfreehook.cpp @@ -6,8 +6,8 @@ #include "gilstate.h" #if defined(_WIN32) && defined(_DEBUG) -#include <crtdbg.h> -#include <windows.h> +# include <sbkwindows.h> +# include <crtdbg.h> #endif #ifdef __GLIBC__ diff --git a/sources/shiboken6/libshiboken/embed/embedding_generator.py b/sources/shiboken6/libshiboken/embed/embedding_generator.py index 96f66b949..a058fd372 100644 --- a/sources/shiboken6/libshiboken/embed/embedding_generator.py +++ b/sources/shiboken6/libshiboken/embed/embedding_generator.py @@ -122,22 +122,23 @@ def _embed_file(fin, fout): limit = 50 text = fin.readlines() print(textwrap.dedent(""" - /* - * This is a ZIP archive of all Python files in the directory - * "shiboken6/shibokenmodule/files.dir/shibokensupport/signature" - * There is also a toplevel file "signature_bootstrap.py[c]" that will be - * directly executed from C++ as a bootstrap loader. - */ + // This is a ZIP archive of all Python files in the directory + // "shiboken6/shibokenmodule/files.dir/shibokensupport/signature" + // There is also a toplevel file "signature_bootstrap.py[c]" that will be + // directly executed from C++ as a bootstrap loader. """).strip(), file=fout) block, blocks = 0, len(text) // limit + 1 for idx, line in enumerate(text): if idx % limit == 0: + if block: + fout.write(')"\n') comma = "," if block else "" block += 1 - print(file=fout) - print(f'/* Block {block} of {blocks} */{comma}', file=fout) - print(f'\"{line.strip()}\"', file=fout) - print(f'/* Sentinel */, \"\"', file=fout) + fout.write(f'\n{comma} // Block {block} of {blocks}\nR"(') + else: + fout.write('\n') + fout.write(line.strip()) + fout.write(')"\n\n/* Sentinel */, ""\n') def _embed_bytefile(fin, fout, is_text): diff --git a/sources/shiboken6/libshiboken/embed/signature_bootstrap.py b/sources/shiboken6/libshiboken/embed/signature_bootstrap.py index c11a0367a..37f95cd35 100644 --- a/sources/shiboken6/libshiboken/embed/signature_bootstrap.py +++ b/sources/shiboken6/libshiboken/embed/signature_bootstrap.py @@ -26,6 +26,7 @@ recursion_trap = 0 import base64 import importlib import io +import os import sys import traceback import zipfile @@ -61,21 +62,83 @@ def bootstrap(): import shibokensupport yield except Exception as e: - print("Problem importing shibokensupport:") - print(f"{e.__class__.__name__}: {e}") + f = sys.stderr + print("Problem importing shibokensupport:", file=f) + print(f"{e.__class__.__name__}: {e}", file=f) traceback.print_exc() - print("sys.path:") + print("sys.path:", file=f) for p in sys.path: - print(" " + p) - sys.stdout.flush() + print(" " + p, file=f) + f.flush() sys.exit(-1) target.remove(support_path) - target, support_path = prepare_zipfile() + # Here we decide if we re-incarnate the embedded files or use embedding. + incarnated = find_incarnated_files() + if incarnated: + target, support_path = sys.path, os.fspath(incarnated) + else: + target, support_path = prepare_zipfile() with ensure_shibokensupport(target, support_path): from shibokensupport.signature import loader return loader +# Newer functionality: +# This function checks if the support directory exist and returns it. +# If does not exist, we try to create it and return it. +# Otherwise, we return None. + +def find_incarnated_files(): + import shiboken6 as root + files_dir = Path(root.__file__).resolve().parent / "files.dir" + handle_embedding_switch(files_dir) + if files_dir.exists(): + sys.path.insert(0, os.fspath(files_dir)) + # Note: To avoid recursion problems, we need to preload the loader. + # But that has the side-effect that we need to delay the feature + # initialization until all function pointers are set. + # See `post_init_func` in signature_globals.cpp . + import shibokensupport.signature.loader + del sys.path[0] + return files_dir + return None + + +def handle_embedding_switch(files_dir): + """ + This handles the optional environment variable `SBK_EMBED` + if not set : do nothing + if set to 0, false, no : de-virtualize the Python files + if set to 1, true, yes : virtualize again (delete "files.dir") + """ + env_name = "SBK_EMBED" + env_var = os.environ.get(env_name) + if not env_var: + return + if env_var.lower() in ("1", "t", "true", "y", "yes"): + import shutil + shutil.rmtree(files_dir, ignore_errors=True) + elif env_var.lower() in ("0", "f", "false", "n", "no"): + reincarnate_files(files_dir) + + +def reincarnate_files(files_dir): + target, zip = prepare_zipfile() + names = (_ for _ in zip.zfile.namelist() if _.endswith(".py")) + try: + # First check mkdir to get an error when we cannot write. + files_dir.mkdir(exist_ok=True) + except os.error as e: + print(f"SBK_EMBED=False: Warning: Cannot write into {files_dir}") + return None + try: + # Then check for a real error when unpacking the zip file. + zip.zfile.extractall(path=files_dir, members=names) + return files_dir + except Exception as e: + print(f"{e.__class__.__name__}: {e}", file=sys.stderr) + traceback.print_exc() + raise # New functionality: Loading from a zip archive. # There exists the zip importer, but as it is written, only real zip files are diff --git a/sources/shiboken6/libshiboken/helper.cpp b/sources/shiboken6/libshiboken/helper.cpp index 8f836316e..46af68956 100644 --- a/sources/shiboken6/libshiboken/helper.cpp +++ b/sources/shiboken6/libshiboken/helper.cpp @@ -5,28 +5,60 @@ #include "basewrapper_p.h" #include "sbkstring.h" #include "sbkstaticstrings.h" +#include "sbkstaticstrings.h" +#include "pep384impl.h" + +#include <algorithm> +#include <optional> #include <iomanip> #include <iostream> - +#include <climits> +#include <cstring> #include <cstdarg> +#include <cctype> #ifdef _WIN32 -# ifndef NOMINMAX -# define NOMINMAX -# endif -# include <windows.h> +# include <sbkwindows.h> #else # include <pthread.h> #endif -#include <algorithm> +static std::optional<std::string> getStringAttr(PyObject *obj, const char *what) +{ + if (PyObject_HasAttrString(obj, what) != 0) { // Check first to suppress error. + Shiboken::AutoDecRef result(PyObject_GetAttrString(obj, what)); + if (PyUnicode_Check(result.object()) != 0) + return _PepUnicode_AsString(result.object()); + } + return std::nullopt; +} -static void formatPyTypeObject(const PyTypeObject *obj, std::ostream &str) +static std::optional<int> getIntAttr(PyObject *obj, const char *what) { - if (obj) { - str << '"' << obj->tp_name << "\", 0x" << std::hex - << obj->tp_flags << std::dec; + if (PyObject_HasAttrString(obj, what) != 0) { // Check first to suppress error. + Shiboken::AutoDecRef result(PyObject_GetAttrString(obj, what)); + if (PyLong_Check(result.object()) != 0) + return PyLong_AsLong(result.object()); + } + return std::nullopt; +} + +static bool verbose = false; + +static void formatTypeTuple(PyObject *t, const char *what, std::ostream &str); + +static void formatPyTypeObject(const PyTypeObject *obj, std::ostream &str, bool verbose) +{ + if (obj == nullptr) { + str << '0'; + return; + } + + str << '"' << obj->tp_name << '"'; + if (verbose) { + bool immutableType = false; + str << ", 0x" << std::hex << obj->tp_flags << std::dec; if (obj->tp_flags & Py_TPFLAGS_HEAPTYPE) str << " [heaptype]"; if (obj->tp_flags & Py_TPFLAGS_BASETYPE) @@ -49,30 +81,59 @@ static void formatPyTypeObject(const PyTypeObject *obj, std::ostream &str) str << " [type]"; if (obj->tp_flags & Py_TPFLAGS_IS_ABSTRACT) str << " [abstract]"; -#if PY_VERSION_HEX >= 0x03080000 + if (obj->tp_flags & Py_TPFLAGS_READY) + str << " [ready]"; + if (obj->tp_flags & Py_TPFLAGS_READYING) + str << " [readying]"; if (obj->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) str << " [method_descriptor]"; -# if PY_VERSION_HEX >= 0x03090000 -# ifndef Py_LIMITED_API +# ifndef Py_LIMITED_API if (obj->tp_flags & Py_TPFLAGS_HAVE_VECTORCALL) str << " [vectorcall]"; -# endif // !Py_LIMITED_API -# if PY_VERSION_HEX >= 0x030A0000 - if (obj->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) +# endif // !Py_LIMITED_API +# if PY_VERSION_HEX >= 0x030A0000 + immutableType = (obj->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) != 0; + if (immutableType) str << " [immutabletype]"; if (obj->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION) str << " [disallow_instantiation]"; -# ifndef Py_LIMITED_API +# ifndef Py_LIMITED_API if (obj->tp_flags & Py_TPFLAGS_MAPPING) str << " [mapping]"; if (obj->tp_flags & Py_TPFLAGS_SEQUENCE) str << " [sequence]"; # endif // !Py_LIMITED_API -# endif // 3.10 -# endif // 3.9 -#endif // 3.8 - } else { - str << '0'; +# endif // 3.10 + if (obj->tp_basicsize != 0) + str << ", basicsize=" << obj->tp_basicsize; + if (verbose) { + formatTypeTuple(obj->tp_bases, "bases", str); + formatTypeTuple(obj->tp_mro, "mro", str); + if (!immutableType) { + auto *underlying = reinterpret_cast<const PyObject *>(obj)->ob_type; + if (underlying != nullptr && underlying != obj) { + str << ", underlying=\"" << underlying->tp_name << '"'; + } + } + } + } +} + +static void formatTypeTuple(PyObject *t, const char *what, std::ostream &str) +{ + const Py_ssize_t size = t != nullptr && PyTuple_Check(t) != 0 ? PyTuple_Size(t) : 0; + if (size > 0) { + str << ", " << what << "=[" << size << "]{"; + for (Py_ssize_t i = 0; i < size; ++i) { + if (i != 0) + str << ", "; + Shiboken::AutoDecRef item(PyTuple_GetItem(t, i)); + if (item.isNull()) + str << '0'; // Observed with non-ready types + else + str << '"' << reinterpret_cast<PyTypeObject *>(item.object())->tp_name << '"'; + } + str << '}'; } } @@ -151,13 +212,17 @@ static void formatPyUnicode(PyObject *obj, std::ostream &str) { // Note: The below call create the PyCompactUnicodeObject.utf8 representation str << '"' << _PepUnicode_AsString(obj) << '"'; + if (!verbose) + return; str << " (" << PyUnicode_GetLength(obj) << ')'; const auto kind = _PepUnicode_KIND(obj); switch (kind) { +#if PY_VERSION_HEX < 0x030C0000 case PepUnicode_WCHAR_KIND: str << " [wchar]"; break; +#endif case PepUnicode_1BYTE_KIND: str << " [1byte]"; break; @@ -178,8 +243,10 @@ static void formatPyUnicode(PyObject *obj, std::ostream &str) void *data =_PepUnicode_DATA(obj); str << ", data="; switch (kind) { +#if PY_VERSION_HEX < 0x030C0000 case PepUnicode_WCHAR_KIND: formatCharSequence(reinterpret_cast<const wchar_t *>(data), str); +#endif break; case PepUnicode_1BYTE_KIND: formatCharSequence(reinterpret_cast<const Py_UCS1 *>(data), str); @@ -208,22 +275,92 @@ static void formatPyUnicode(PyObject *obj, std::ostream &str) #endif // !Py_LIMITED_API } +static std::string getQualName(PyObject *obj) +{ + Shiboken::AutoDecRef result(PyObject_GetAttr(obj, Shiboken::PyMagicName::qualname())); + return result.object() != nullptr + ? _PepUnicode_AsString(result.object()) : std::string{}; +} + +static void formatPyFunction(PyObject *obj, std::ostream &str) +{ + str << '"' << getQualName(obj) << "()\""; +} + +static void formatPyMethod(PyObject *obj, std::ostream &str) +{ + if (auto *func = PyMethod_Function(obj)) + formatPyFunction(func, str); + str << ", instance=" << PyMethod_Self(obj); +} + +static void formatPyCodeObject(PyObject *obj, std::ostream &str) +{ + if (auto name = getStringAttr(obj, "co_name")) + str << '"' << name.value() << '"'; + if (auto qualName = getStringAttr(obj, "co_qualname")) + str << ", co_qualname=\"" << qualName.value() << '"'; + if (auto flags = getIntAttr(obj, "co_flags")) + str << ", flags=0x" << std::hex << flags.value() << std::dec; + if (auto c = getIntAttr(obj, "co_argcount")) + str << ", co_argcounts=" << c.value(); + if (auto c = getIntAttr(obj, "co_posonlyargcount")) + str << ", co_posonlyargcount=" << c.value(); + if (auto c = getIntAttr(obj, "co_kwonlyargcount")) + str << ", co_kwonlyargcount=" << c.value(); + if (auto fileName = getStringAttr(obj, "co_filename")) { + str << " @" << fileName.value(); + if (auto l = getIntAttr(obj, "co_firstlineno")) + str << ':'<< l.value(); + } +} + static void formatPyObjectHelper(PyObject *obj, std::ostream &str) { - str << ", refs=" << obj->ob_refcnt << ", "; + str << ", "; + if (obj == Py_None) { + str << "None"; + return; + } + if (obj == Py_True) { + str << "True"; + return; + } + if (obj == Py_False) { + str << "False"; + return; + } + const auto refs = Py_REFCNT(obj); + if (refs == UINT_MAX) // _Py_IMMORTAL_REFCNT + str << "immortal, "; + else + str << "refs=" << refs << ", "; if (PyType_Check(obj)) { str << "type: "; - formatPyTypeObject(reinterpret_cast<PyTypeObject *>(obj), str); + formatPyTypeObject(reinterpret_cast<PyTypeObject *>(obj), str, true); return; } - formatPyTypeObject(obj->ob_type, str); + formatPyTypeObject(obj->ob_type, str, false); str << ", "; - if (PyLong_Check(obj)) - str << PyLong_AsLong(obj); + if (PyLong_Check(obj)) { + const auto llv = PyLong_AsLongLong(obj); + if (PyErr_Occurred() != PyExc_OverflowError) { + str << llv; + } else { + PyErr_Clear(); + str << "0x" << std::hex << PyLong_AsUnsignedLongLong(obj) << std::dec; + } + } else if (PyFloat_Check(obj)) str << PyFloat_AsDouble(obj); else if (PyUnicode_Check(obj)) formatPyUnicode(obj, str); + else if (PyFunction_Check(obj) != 0) + formatPyFunction(obj, str); + else if (PyMethod_Check(obj) != 0) + formatPyMethod(obj, str); + else if (PepCode_Check(obj) != 0) + formatPyCodeObject(obj, str); else if (PySequence_Check(obj)) formatPySequence(obj, str); else if (PyDict_Check(obj)) @@ -263,7 +400,7 @@ debugPyBuffer::debugPyBuffer(const Py_buffer &b) : m_buffer(b) std::ostream &operator<<(std::ostream &str, const debugPyTypeObject &o) { str << "PyTypeObject("; - formatPyTypeObject(o.m_object, str); + formatPyTypeObject(o.m_object, str, true); str << ')'; return str; } @@ -299,6 +436,18 @@ std::ostream &operator<<(std::ostream &str, const debugPyBuffer &b) return str; } +std::ios_base &debugVerbose(std::ios_base &s) +{ + verbose = true; + return s; +} + +std::ios_base &debugBrief(std::ios_base &s) +{ + verbose = false; + return s; +} + #ifdef _WIN32 // Converts a Unicode string to a string encoded in the Windows console's // code page via wchar_t for use with argv (PYSIDE-1425). @@ -442,4 +591,47 @@ ThreadId mainThreadId() return _mainThreadId; } +const char *typeNameOf(const char *typeIdName) +{ + auto size = std::strlen(typeIdName); +#if defined(Q_CC_MSVC) // MSVC: "class QPaintDevice * __ptr64" + if (auto *lastStar = strchr(typeName, '*')) { + // MSVC: "class QPaintDevice * __ptr64" + while (*--lastStar == ' ') { + } + size = lastStar - typeName + 1; + } +#else // g++, Clang: "QPaintDevice *" -> "P12QPaintDevice" + if (size > 2 && typeIdName[0] == 'P' && std::isdigit(typeIdName[1])) { + ++typeIdName; + --size; + } +#endif + char *result = new char[size + 1]; + result[size] = '\0'; + std::memcpy(result, typeIdName, size); + return result; +} + +#if !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030A0000 && !defined(PYPY_VERSION) +static int _getPyVerbose() +{ + PyConfig config; + PyConfig_InitPythonConfig(&config); + return config.verbose; +} +#endif // !Py_LIMITED_API >= 3.10 + +int pyVerbose() +{ +#ifdef Py_LIMITED_API + return Pep_GetVerboseFlag(); +#elif PY_VERSION_HEX >= 0x030A0000 && !defined(PYPY_VERSION) + static const int result = _getPyVerbose(); + return result; +#else + return Py_VerboseFlag; +#endif +} + } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/helper.h b/sources/shiboken6/libshiboken/helper.h index 265bb6581..f226e8c24 100644 --- a/sources/shiboken6/libshiboken/helper.h +++ b/sources/shiboken6/libshiboken/helper.h @@ -36,6 +36,10 @@ LIBSHIBOKEN_API bool listToArgcArgv(PyObject *argList, int *argc, char ***argv, */ LIBSHIBOKEN_API int *sequenceToIntArray(PyObject *obj, bool zeroTerminated = false); +/// Fix a type name returned by typeid(t).name(), depending on compiler. +/// \returns Fixed name (allocated). +LIBSHIBOKEN_API const char *typeNameOf(const char *typeIdName); + /** * Creates and automatically deallocates C++ arrays. */ @@ -61,6 +65,8 @@ using ThreadId = unsigned long long; LIBSHIBOKEN_API ThreadId currentThreadId(); LIBSHIBOKEN_API ThreadId mainThreadId(); +LIBSHIBOKEN_API int pyVerbose(); + /** * An utility function used to call PyErr_WarnEx with a formatted message. */ @@ -106,7 +112,8 @@ LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugSbkObject LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugPyTypeObject &o); LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugPyBuffer &b); LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugPyArrayObject &b); - +LIBSHIBOKEN_API std::ios_base &debugVerbose(std::ios_base &s); +LIBSHIBOKEN_API std::ios_base &debugBrief(std::ios_base &s); } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/pep384_issue33738.cpp b/sources/shiboken6/libshiboken/pep384_issue33738.cpp deleted file mode 100644 index 7f3872a58..000000000 --- a/sources/shiboken6/libshiboken/pep384_issue33738.cpp +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (C) 2018 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only - -// There is a bug in Python 3.6 that turned the Index_Check function -// into a macro without taking care of the limited API. -// This leads to the single problem that we don't have -// access to PyLong_Type's nb_index field which is no heap type. -// We cannot easily create this function by inheritance since it is -// not inherited. -// -// Simple solution: Create the structure and write such a function. -// Long term: Submit a patch to python.org . - -// This structure comes from Python 3.7, but we have checked that -// it also works for Python 3.8 and 3.9. - -typedef struct { - /* Number implementations must check *both* - arguments for proper type and implement the necessary conversions - in the slot functions themselves. */ - - binaryfunc nb_add; - binaryfunc nb_subtract; - binaryfunc nb_multiply; - binaryfunc nb_remainder; - binaryfunc nb_divmod; - ternaryfunc nb_power; - unaryfunc nb_negative; - unaryfunc nb_positive; - unaryfunc nb_absolute; - inquiry nb_bool; - unaryfunc nb_invert; - binaryfunc nb_lshift; - binaryfunc nb_rshift; - binaryfunc nb_and; - binaryfunc nb_xor; - binaryfunc nb_or; - unaryfunc nb_int; - void *nb_reserved; /* the slot formerly known as nb_long */ - unaryfunc nb_float; - - binaryfunc nb_inplace_add; - binaryfunc nb_inplace_subtract; - binaryfunc nb_inplace_multiply; - binaryfunc nb_inplace_remainder; - ternaryfunc nb_inplace_power; - binaryfunc nb_inplace_lshift; - binaryfunc nb_inplace_rshift; - binaryfunc nb_inplace_and; - binaryfunc nb_inplace_xor; - binaryfunc nb_inplace_or; - - binaryfunc nb_floor_divide; - binaryfunc nb_true_divide; - binaryfunc nb_inplace_floor_divide; - binaryfunc nb_inplace_true_divide; - - unaryfunc nb_index; - - binaryfunc nb_matrix_multiply; - binaryfunc nb_inplace_matrix_multiply; -} PyNumberMethods; - -// temporary structure until we have a generator for the offsets -typedef struct _oldtypeobject { - PyVarObject ob_base; - void *X01; // const char *tp_name; - void *X02; // Py_ssize_t tp_basicsize; - void *X03; // Py_ssize_t tp_itemsize; - void *X04; // destructor tp_dealloc; - void *X05; // printfunc tp_print; - void *X06; // getattrfunc tp_getattr; - void *X07; // setattrfunc tp_setattr; - void *X08; // PyAsyncMethods *tp_as_async; - void *X09; // reprfunc tp_repr; - PyNumberMethods *tp_as_number; - -} PyOldTypeObject; - -static bool is_compatible_version() -{ - auto *sysmodule = PyImport_AddModule("sys"); - auto *dic = PyModule_GetDict(sysmodule); - auto *version = PyDict_GetItemString(dic, "version_info"); - auto *major = PyTuple_GetItem(version, 0); - auto *minor = PyTuple_GetItem(version, 1); - auto number = PyLong_AsLong(major) * 1000 + PyLong_AsLong(minor); - return number < 3010; -} - -/////////////////////////////////////////////////////////////////////// -// -// PYSIE-1797: The Solution -// ======================== -// -// Inspecting the data structures of Python 3.6, 3.7, 3.8 and 3.9 -// shows that concerning the here needed offset of nb_index, they -// are all compatible. -// That means: We can use the above definition for all these versions. -// -// From Python 3.10 on, the `PyType_GetSlot` function also works with -// non-heap types. That means this solution will always work. -// -// Note: When we have moved to Python 3.8 as the minimum version, -// this whole nonsense can be trashed. -// There is an automatic warning about this in parser.py . -// - -LIBSHIBOKEN_API int PepIndex_Check(PyObject *obj) -{ - static bool old_python_version = is_compatible_version(); - if (old_python_version) { - auto *type = reinterpret_cast<PyOldTypeObject *>(Py_TYPE(obj)); - return type->tp_as_number != nullptr && - type->tp_as_number->nb_index != nullptr; - } - // From Python 3.10 on, we can use PyType_GetSlot also with normal types! - unaryfunc nb_index = reinterpret_cast<unaryfunc>(PyType_GetSlot(Py_TYPE(obj), Py_nb_index)); - return nb_index != nullptr; -} - diff --git a/sources/shiboken6/libshiboken/pep384ext.h b/sources/shiboken6/libshiboken/pep384ext.h new file mode 100644 index 000000000..021c53d16 --- /dev/null +++ b/sources/shiboken6/libshiboken/pep384ext.h @@ -0,0 +1,89 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef PEP384EXT_H +#define PEP384EXT_H + +#include "pep384impl.h" + +/// Returns the allocator slot of the PyTypeObject. +inline allocfunc PepExt_Type_GetAllocSlot(PyTypeObject *t) +{ + return reinterpret_cast<allocfunc>(PepType_GetSlot(t, Py_tp_alloc)); +} + +/// Invokes the allocator slot of the PyTypeObject. +template <class Type> +inline Type *PepExt_TypeCallAlloc(PyTypeObject *t, Py_ssize_t nitems) +{ + PyObject *result = PepExt_Type_GetAllocSlot(t)(t, nitems); + return reinterpret_cast<Type *>(result); +} + +/// Returns the getattro slot of the PyTypeObject. +inline getattrofunc PepExt_Type_GetGetAttroSlot(PyTypeObject *t) +{ + return reinterpret_cast<getattrofunc>(PepType_GetSlot(t, Py_tp_getattro)); +} + +/// Returns the setattro slot of the PyTypeObject. +inline setattrofunc PepExt_Type_GetSetAttroSlot(PyTypeObject *t) +{ + return reinterpret_cast<setattrofunc>(PepType_GetSlot(t, Py_tp_setattro)); +} + +/// Returns the descr_get slot of the PyTypeObject. +inline descrgetfunc PepExt_Type_GetDescrGetSlot(PyTypeObject *t) +{ + return reinterpret_cast<descrgetfunc>(PepType_GetSlot(t, Py_tp_descr_get)); +} + +/// Invokes the descr_get slot of the PyTypeObject. +inline PyObject *PepExt_Type_CallDescrGet(PyObject *self, PyObject *obj, PyObject *type) +{ + return PepExt_Type_GetDescrGetSlot(Py_TYPE(self))(self, obj, type); +} + +/// Returns the descr_set slot of the PyTypeObject. +inline descrsetfunc PepExt_Type_GetDescrSetSlot(PyTypeObject *t) +{ + return reinterpret_cast<descrsetfunc>(PepType_GetSlot(t, Py_tp_descr_set)); +} + +/// Returns the call slot of the PyTypeObject. +inline ternaryfunc PepExt_Type_GetCallSlot(PyTypeObject *t) +{ + return reinterpret_cast<ternaryfunc>(PepType_GetSlot(t, Py_tp_call)); +} + +/// Returns the new slot of the PyTypeObject. +inline newfunc PepExt_Type_GetNewSlot(PyTypeObject *t) +{ + return reinterpret_cast<newfunc>(PepType_GetSlot(t, Py_tp_new)); +} + +/// Returns the init slot of the PyTypeObject. +inline initproc PepExt_Type_GetInitSlot(PyTypeObject *t) +{ + return reinterpret_cast<initproc>(PepType_GetSlot(t, Py_tp_init)); +} + +/// Returns the free slot of the PyTypeObject. +inline freefunc PepExt_Type_GetFreeSlot(PyTypeObject *t) +{ + return reinterpret_cast<freefunc>(PepType_GetSlot(t, Py_tp_free)); +} + +/// Invokes the free slot of the PyTypeObject. +inline void PepExt_TypeCallFree(PyTypeObject *t, void *object) +{ + PepExt_Type_GetFreeSlot(t)(object); +} + +/// Invokes the free slot of the PyTypeObject. +inline void PepExt_TypeCallFree(PyObject *object) +{ + PepExt_Type_GetFreeSlot(Py_TYPE(object))(object); +} + +#endif // PEP384EXT_H diff --git a/sources/shiboken6/libshiboken/pep384impl.cpp b/sources/shiboken6/libshiboken/pep384impl.cpp index 3a9a60f74..5310207a3 100644 --- a/sources/shiboken6/libshiboken/pep384impl.cpp +++ b/sources/shiboken6/libshiboken/pep384impl.cpp @@ -1,6 +1,8 @@ -// Copyright (C) 2018 The Qt Company Ltd. +// Copyright (C) 2023 The Qt Company Ltd. // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#define PEP384_INTERN + #include "sbkpython.h" #include "autodecref.h" #include "sbkstaticstrings.h" @@ -8,8 +10,6 @@ #include "basewrapper.h" #include "basewrapper_p.h" #include "sbkenum.h" -#include "sbkenum_p.h" -#include "sbkconverter.h" #include "voidptr.h" #include <cstdlib> @@ -19,7 +19,7 @@ extern "C" { /* - * The documentation is located in pep384impl_doc.rst + * The documentation is located in `sources/pyside6/doc/developer/limited_api.rst`. * Here is the verification code for PyTypeObject. * We create a type object and check if its fields @@ -105,18 +105,20 @@ static PyType_Spec typeprobe_spec = { static void check_PyTypeObject_valid() { - auto *obtype = reinterpret_cast<PyObject *>(&PyType_Type); - auto *probe_tp_base = reinterpret_cast<PyTypeObject *>( - PyObject_GetAttr(obtype, Shiboken::PyMagicName::base())); + auto *typetype = &PyType_Type; + auto *obtype = reinterpret_cast<PyObject *>(typetype); + auto *probe_tp_base_obj = PyObject_GetAttr(obtype, Shiboken::PyMagicName::base()); + auto *probe_tp_base = reinterpret_cast<PyTypeObject *>(probe_tp_base_obj); auto *probe_tp_bases = PyObject_GetAttr(obtype, Shiboken::PyMagicName::bases()); - auto *check = reinterpret_cast<PyTypeObject *>( - PyType_FromSpecWithBases(&typeprobe_spec, probe_tp_bases)); - auto *typetype = reinterpret_cast<PyTypeObject *>(obtype); + auto *checkObj = PyType_FromSpecWithBases(&typeprobe_spec, probe_tp_bases); + auto *check = reinterpret_cast<PyTypeObject *>(checkObj); PyObject *w = PyObject_GetAttr(obtype, Shiboken::PyMagicName::weakrefoffset()); long probe_tp_weakrefoffset = PyLong_AsLong(w); PyObject *d = PyObject_GetAttr(obtype, Shiboken::PyMagicName::dictoffset()); long probe_tp_dictoffset = PyLong_AsLong(d); PyObject *probe_tp_mro = PyObject_GetAttr(obtype, Shiboken::PyMagicName::mro()); + Shiboken::AutoDecRef tpDict(PepType_GetDict(check)); + auto *checkDict = tpDict.object(); if (false || strcmp(probe_tp_name, check->tp_name) != 0 || probe_tp_basicsize != check->tp_basicsize @@ -133,8 +135,8 @@ check_PyTypeObject_valid() || probe_tp_methods != check->tp_methods || probe_tp_getset != check->tp_getset || probe_tp_base != typetype->tp_base - || !PyDict_Check(check->tp_dict) - || !PyDict_GetItemString(check->tp_dict, "dummy") + || !PyDict_Check(checkDict) + || !PyDict_GetItemString(checkDict, "dummy") || probe_tp_descr_get != check->tp_descr_get || probe_tp_descr_set != check->tp_descr_set || probe_tp_dictoffset != typetype->tp_dictoffset @@ -147,17 +149,14 @@ check_PyTypeObject_valid() || probe_tp_mro != typetype->tp_mro || Py_TPFLAGS_DEFAULT != (check->tp_flags & Py_TPFLAGS_DEFAULT)) Py_FatalError("The structure of type objects has changed!"); - Py_DECREF(check); - Py_DECREF(probe_tp_base); + Py_DECREF(checkObj); + Py_DECREF(probe_tp_base_obj); Py_DECREF(w); Py_DECREF(d); Py_DECREF(probe_tp_bases); Py_DECREF(probe_tp_mro); } -// PYSIDE-1797: This must be a runtime decision. -#include "pep384_issue33738.cpp" - #endif // Py_LIMITED_API /***************************************************************************** @@ -179,7 +178,7 @@ static PyObject * find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) { Py_ssize_t i, n; - PyObject *mro, *res, *base, *dict; + PyObject *mro, *res, *base; /* Look in tp_dict of types in MRO */ mro = type->tp_mro; @@ -193,9 +192,10 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) for (i = 0; i < n; i++) { base = PyTuple_GET_ITEM(mro, i); assert(PyType_Check(base)); - dict = ((PyTypeObject *)base)->tp_dict; - assert(dict && PyDict_Check(dict)); - res = PyDict_GetItem(dict, name); + auto *type = reinterpret_cast<PyTypeObject *>(base); + Shiboken::AutoDecRef dict(PepType_GetDict(type)); + assert(!dict.isNull() && PyDict_Check(dict.object())); + res = PyDict_GetItem(dict.object(), name); if (res != nullptr) break; if (PyErr_Occurred()) { @@ -251,7 +251,7 @@ _PepType_Lookup(PyTypeObject *type, PyObject *name) // structs and macros modelled after their equivalents in // cpython/Include/cpython/unicodeobject.h -struct PepASCIIObject +struct PepASCIIObject // since 3.12 { PyObject_HEAD Py_ssize_t length; /* Number of code points in the string */ @@ -264,18 +264,29 @@ struct PepASCIIObject unsigned int ready:1; unsigned int :24; } state; +}; + +struct PepASCIIObject_311 : public PepASCIIObject +{ wchar_t *wstr; /* wchar_t representation (null-terminated) */ }; -struct PepCompactUnicodeObject +struct PepCompactUnicodeObject // since 3.12 { PepASCIIObject _base; Py_ssize_t utf8_length; char *utf8; /* UTF-8 representation (null-terminated) */ +}; + +struct PepCompactUnicodeObject_311 // since 3.12 +{ + PepASCIIObject_311 _base; + Py_ssize_t utf8_length; + char *utf8; /* UTF-8 representation (null-terminated) */ Py_ssize_t wstr_length; /* Number of code points in wstr */ }; -struct PepUnicodeObject +struct PepUnicodeObject // since 3.12 { PepCompactUnicodeObject _base; union { @@ -286,6 +297,17 @@ struct PepUnicodeObject } data; /* Canonical, smallest-form Unicode buffer */ }; +struct PepUnicodeObject_311 +{ + PepCompactUnicodeObject_311 _base; + union { + void *any; + Py_UCS1 *latin1; + Py_UCS2 *ucs2; + Py_UCS4 *ucs4; + } data; /* Canonical, smallest-form Unicode buffer */ +}; + int _PepUnicode_KIND(PyObject *str) { return reinterpret_cast<PepASCIIObject *>(str)->state.kind; @@ -303,18 +325,33 @@ int _PepUnicode_IS_COMPACT(PyObject *str) return asciiObj->state.compact; } -static void *_PepUnicode_COMPACT_DATA(PyObject *str) +static void *_PepUnicode_ASCII_DATA(PyObject *str) { + if (_PepRuntimeVersion() < 0x030C00) { + auto *asciiObj_311 = reinterpret_cast<PepASCIIObject_311 *>(str); + return asciiObj_311 + 1; + } auto *asciiObj = reinterpret_cast<PepASCIIObject *>(str); - if (asciiObj->state.ascii) - return asciiObj + 1; + return asciiObj + 1; +} + +static void *_PepUnicode_COMPACT_DATA(PyObject *str) +{ + if (_PepUnicode_IS_ASCII(str) != 0) + return _PepUnicode_ASCII_DATA(str); + if (_PepRuntimeVersion() < 0x030C00) { + auto *compactObj_311 = reinterpret_cast<PepCompactUnicodeObject_311 *>(str); + return compactObj_311 + 1; + } auto *compactObj = reinterpret_cast<PepCompactUnicodeObject *>(str); return compactObj + 1; } static void *_PepUnicode_NONCOMPACT_DATA(PyObject *str) { - return reinterpret_cast<PepUnicodeObject *>(str)->data.any; + return _PepRuntimeVersion() < 0x030C00 + ? reinterpret_cast<PepUnicodeObject_311 *>(str)->data.any + : reinterpret_cast<PepUnicodeObject *>(str)->data.any; } void *_PepUnicode_DATA(PyObject *str) @@ -325,6 +362,23 @@ void *_PepUnicode_DATA(PyObject *str) // Fast path accessing UTF8 data without doing a conversion similar // to _PyUnicode_AsUTF8String +static const char *utf8FastPath_311(PyObject *str) +{ + if (PyUnicode_GetLength(str) == 0) + return ""; + auto *asciiObj = reinterpret_cast<PepASCIIObject_311 *>(str); + if (asciiObj->state.kind != PepUnicode_1BYTE_KIND || asciiObj->state.compact == 0) + return nullptr; // Empirical: PyCompactUnicodeObject.utf8 is only valid for 1 byte + if (asciiObj->state.ascii) { + auto *data = asciiObj + 1; + return reinterpret_cast<const char *>(data); + } + auto *compactObj = reinterpret_cast<PepCompactUnicodeObject_311 *>(str); + if (compactObj->utf8_length) + return compactObj->utf8; + return nullptr; +} + static const char *utf8FastPath(PyObject *str) { if (PyUnicode_GetLength(str) == 0) @@ -356,8 +410,10 @@ const char *_PepUnicode_AsString(PyObject *str) #define TOSTRING(x) STRINGIFY(x) #define AT __FILE__ ":" TOSTRING(__LINE__) - if (const auto *utf8 = utf8FastPath(str)) + if (const auto *utf8 = _PepRuntimeVersion() < 0x030C00 + ? utf8FastPath_311(str) : utf8FastPath(str)) { return utf8; + } static PyObject *cstring_dict = nullptr; if (cstring_dict == nullptr) { @@ -426,6 +482,47 @@ Pep_GetVerboseFlag() } #endif // Py_LIMITED_API +// Support for pyerrors.h + +#if defined(Py_LIMITED_API) || PY_VERSION_HEX < 0x030C0000 +// Emulate PyErr_GetRaisedException() using the deprecated PyErr_Fetch()/PyErr_Store() +PyObject *PepErr_GetRaisedException() +{ + PyObject *type{}; + PyObject *value{}; + PyObject *traceback{}; + PyErr_Fetch(&type, &value, &traceback); + Py_XINCREF(value); + PyErr_Restore(type, value, traceback); + return value; +} + +struct PepException_HEAD +{ + PyObject_HEAD + PyObject *x1; // dict + PyObject *args; +}; + +// PyException_GetArgs/PyException_SetArgs were added to the stable API in 3.12 +PyObject *PepException_GetArgs(PyObject *ex) +{ + auto *h = reinterpret_cast<PepException_HEAD *>(ex); + Py_XINCREF(h->args); + return h->args; +} + +LIBSHIBOKEN_API void PepException_SetArgs(PyObject *ex, PyObject *args) +{ + auto *h = reinterpret_cast<PepException_HEAD *>(ex); + Py_XINCREF(args); + auto *old = h->args; // Py_XSETREF() + h->args = args; + Py_XDECREF(old); + +} +#endif // Limited or < 3.12 + /***************************************************************************** * * Support for code.h @@ -448,8 +545,24 @@ PepCode_Get(PepCodeObject *co, const char *name) } return ret; } + +int PepCode_Check(PyObject *o) +{ + return o != nullptr && std::strcmp(Py_TYPE(o)->tp_name, "code") == 0 ? 1 : 0; +} + #endif // Py_LIMITED_API +#if defined(Py_LIMITED_API) || defined(PYPY_VERSION) +PyObject *PepFunction_GetDefaults(PyObject *function) +{ + auto *ob_ret = PyObject_GetAttrString(function, "__defaults__"); + Py_XDECREF(ob_ret); // returns borrowed ref + return ob_ret != Py_None ? ob_ret : nullptr; +} + +#endif // defined(Py_LIMITED_API) || defined(PYPY_VERSION) + /***************************************************************************** * * Support for datetime.h @@ -650,11 +763,8 @@ PyTypeObject *PepStaticMethod_TypePtr = nullptr; static PyTypeObject * getStaticMethodType(void) { - // this works for Python 3, only - // "StaticMethodType = type(str.__dict__['maketrans'])\n"; static const char prog[] = - "from xxsubtype import spamlist\n" - "result = type(spamlist.__dict__['staticmeth'])\n"; + "result = type(str.__dict__['maketrans'])\n"; return reinterpret_cast<PyTypeObject *>(PepRun_GetResult(prog)); } @@ -719,12 +829,37 @@ PepType_GetNameStr(PyTypeObject *type) return ret; } +// PYSIDE-2264: Find the _functools or functools module and retrieve the +// partial function. This can be tampered with, check carefully. +PyObject * +Pep_GetPartialFunction(void) +{ + static bool initialized = false; + static PyObject *result{}; + if (initialized) { + Py_INCREF(result); + return result; + } + auto *functools = PyImport_ImportModule("_functools"); + if (!functools) { + PyErr_Clear(); + functools = PyImport_ImportModule("functools"); + } + if (!functools) + Py_FatalError("functools cannot be found"); + result = PyObject_GetAttrString(functools, "partial"); + if (!result || !PyCallable_Check(result)) + Py_FatalError("partial not found or not a function"); + initialized = true; + return result; +} + /***************************************************************************** * * Newly introduced convenience functions * */ -#if PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) +#ifdef Py_LIMITED_API PyObject * PyImport_GetModule(PyObject *name) @@ -750,7 +885,7 @@ PyImport_GetModule(PyObject *name) return m; } -#endif // PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) +#endif // Py_LIMITED_API // 2020-06-16: For simplicity of creating arbitrary things, this function // is now made public. @@ -841,13 +976,13 @@ _Pep_PrivateMangle(PyObject *self, PyObject *name) wchar_t bigbuf[big_stack]; wchar_t *resbuf = amount <= big_stack ? bigbuf : (wchar_t *)malloc(sizeof(wchar_t) * amount); if (!resbuf) - return 0; + return nullptr; /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */ resbuf[0] = '_'; if (PyUnicode_AsWideChar(privateobj, resbuf + 1, ipriv + plen) < 0) - return 0; + return nullptr; if (PyUnicode_AsWideChar(name, resbuf + ipriv + plen + 1, nlen) < 0) - return 0; + return nullptr; PyObject *result = PyUnicode_FromWideChar(resbuf + ipriv, 1 + plen + nlen); if (amount > big_stack) free(resbuf); @@ -873,6 +1008,21 @@ init_PepRuntime() PepRuntime_38_flag = 1; } +static long _GetPepRuntimeVersion() +{ + auto *version = PySys_GetObject("version_info"); + const auto major = PyLong_AsLong(PyTuple_GetItem(version, 0)); + const auto minor = PyLong_AsLong(PyTuple_GetItem(version, 1)); + const auto micro = PyLong_AsLong(PyTuple_GetItem(version, 2)); + return major << 16 | minor << 8 | micro; +} + +long _PepRuntimeVersion() +{ + static const auto number = _GetPepRuntimeVersion(); + return number; +} + /***************************************************************************** * * PYSIDE-535: Support for PyPy @@ -882,33 +1032,108 @@ init_PepRuntime() * */ +/////////////////////////////////////////////////////////////////////// +// +// PEP 697: Support for embedded type structures. +// +// According to `https://docs.python.org/3/c-api/object.html?highlight=pyobject_gettypedata#c.PyObject_GetTypeData` +// the function `PyObject_GetTypeData` should belong to the Stable API +// since version 3.12.0, but it does not. We use instead some copies +// from Python source code. + +#if !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030C0000 + +# define PepObject_GetTypeData PyObject_GetTypeData + +SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *type) +{ + // PYSIDE-2676: Use the meta type explicitly. + // A derived type would fail the offset calculation. + static auto *meta = SbkObjectType_TypeF(); + assert(SbkObjectType_Check(type)); + auto *obType = reinterpret_cast<PyObject *>(type); + void *data = PyObject_GetTypeData(obType, meta); + return reinterpret_cast<SbkObjectTypePrivate *>(data); +} + +void PepType_SOTP_delete(PyTypeObject * /*type*/) +{ +} + +#else + +// The following comments are directly copied from Python 3.12 +// + +// Make sure we have maximum alignment, even if the current compiler +// does not support max_align_t. Note that: +// - Autoconf reports alignment of unknown types to 0. +// - 'long double' has maximum alignment on *most* platforms, +// looks like the best we can do for pre-C11 compilers. +// - The value is tested, see test_alignof_max_align_t +# if !defined(ALIGNOF_MAX_ALIGN_T) || ALIGNOF_MAX_ALIGN_T == 0 +# undef ALIGNOF_MAX_ALIGN_T +# define ALIGNOF_MAX_ALIGN_T alignof(long double) +# endif + +/* Align up to the nearest multiple of alignof(max_align_t) + * (like _Py_ALIGN_UP, but for a size rather than pointer) + */ +static Py_ssize_t _align_up(Py_ssize_t size) +{ + return (size + ALIGNOF_MAX_ALIGN_T - 1) & ~(ALIGNOF_MAX_ALIGN_T - 1); +} + +static void *PepObject_GetTypeData(PyObject *obj, PyTypeObject *cls) +{ + assert(PyObject_TypeCheck(obj, cls)); + return reinterpret_cast<char *>(obj) + _align_up(cls->tp_base->tp_basicsize); +} +// +/////////////////////////////////////////////////////////////////////// + /* * PyTypeObject extender */ + static std::unordered_map<PyTypeObject *, SbkObjectTypePrivate > SOTP_extender{}; static thread_local PyTypeObject *SOTP_key{}; static thread_local SbkObjectTypePrivate *SOTP_value{}; -SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *sbkType) +SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *type) { - if (sbkType == SOTP_key) + static auto *meta = SbkObjectType_TypeF(); + static bool use_312 = _PepRuntimeVersion() >= 0x030C00; + assert(SbkObjectType_Check(type)); + if (use_312) { + auto *obType = reinterpret_cast<PyObject *>(type); + void *data = PepObject_GetTypeData(obType, meta); + return reinterpret_cast<SbkObjectTypePrivate *>(data); + } + if (type == SOTP_key) return SOTP_value; - auto it = SOTP_extender.find(sbkType); + auto it = SOTP_extender.find(type); if (it == SOTP_extender.end()) { - it = SOTP_extender.insert({sbkType, {}}).first; + it = SOTP_extender.insert({type, {}}).first; memset(&it->second, 0, sizeof(SbkObjectTypePrivate)); } - SOTP_key = sbkType; + SOTP_key = type; SOTP_value = &it->second; return SOTP_value; } -void PepType_SOTP_delete(PyTypeObject *sbkType) +void PepType_SOTP_delete(PyTypeObject *type) { - SOTP_extender.erase(sbkType); + static bool use_312 = _PepRuntimeVersion() >= 0x030C00; + assert(SbkObjectType_Check(type)); + if (use_312) + return; + SOTP_extender.erase(type); SOTP_key = nullptr; } +#endif // !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030C0000 + /* * SbkEnumType extender */ @@ -918,6 +1143,7 @@ static thread_local SbkEnumTypePrivate *SETP_value{}; SbkEnumTypePrivate *PepType_SETP(SbkEnumType *enumType) { + // PYSIDE-2230: This makes no sense at all for Enum types. if (enumType == SETP_key) return SETP_value; auto it = SETP_extender.find(enumType); @@ -936,38 +1162,76 @@ void PepType_SETP_delete(SbkEnumType *enumType) SETP_key = nullptr; } -/* - * PySideQFlagsType extender - */ -static std::unordered_map<PySideQFlagsType *, PySideQFlagsTypePrivate> PFTP_extender{}; -static thread_local PySideQFlagsType *PFTP_key{}; -static thread_local PySideQFlagsTypePrivate *PFTP_value{}; - -PySideQFlagsTypePrivate *PepType_PFTP(PySideQFlagsType *flagsType) -{ - static PyTypeObject *enumMeta = getPyEnumMeta(); - auto *mappedType = reinterpret_cast<PyTypeObject *>(flagsType); - auto *metaType = Py_TYPE(mappedType); - if (metaType == enumMeta) { - return reinterpret_cast<PySideQFlagsTypePrivate *>( - PepType_SETP(reinterpret_cast<SbkEnumType *>(flagsType))); - } - if (flagsType == PFTP_key) - return PFTP_value; - auto it = PFTP_extender.find(flagsType); - if (it == PFTP_extender.end()) { - it = PFTP_extender.insert({flagsType, {}}).first; - memset(&it->second, 0, sizeof(PySideQFlagsTypePrivate)); +#ifdef Py_LIMITED_API +static PyObject *emulatePyType_GetDict(PyTypeObject *type) +{ + if (_PepRuntimeVersion() < 0x030C00 || type->tp_dict) { + auto *res = type->tp_dict; + Py_XINCREF(res); + return res; } - PFTP_key = flagsType; - PFTP_value = &it->second; - return PFTP_value; + // PYSIDE-2230: Here we are really cheating. We don't know how to + // access an internal dict, and so we simply pretend + // it were an empty dict. This works great for our types. + // This was an unexpectedly simple solution :D + return PyDict_New(); } +#endif -void PepType_PFTP_delete(PySideQFlagsType *flagsType) +// PyType_GetDict: replacement for <static type>.tp_dict, which is +// zero for builtin types since 3.12. +PyObject *PepType_GetDict(PyTypeObject *type) { - PFTP_extender.erase(flagsType); - PFTP_key = nullptr; +#if !defined(Py_LIMITED_API) +# if PY_VERSION_HEX >= 0x030C0000 + return PyType_GetDict(type); +# else + // pre 3.12 fallback code, mimicking the addref-behavior. + Py_XINCREF(type->tp_dict); + return type->tp_dict; +# endif +#else + return emulatePyType_GetDict(type); +#endif // Py_LIMITED_API +} + +int PepType_SetDict(PyTypeObject *type, PyObject *dict) +{ + type->tp_dict = dict; + return 0; +} + +// Pre 3.10, PyType_GetSlot() would only work for heap types. +// FIXME: PyType_GetSlot() can be used unconditionally when the +// minimum limited API version is >= 3.10. +void *PepType_GetSlot(PyTypeObject *type, int aSlot) +{ + static const bool is310 = _PepRuntimeVersion() >= 0x030A00; + if (is310 || (type->tp_flags & Py_TPFLAGS_HEAPTYPE) != 0) + return PyType_GetSlot(type, aSlot); + + switch (aSlot) { + case Py_tp_alloc: + return reinterpret_cast<void *>(type->tp_alloc); + case Py_tp_getattro: + return reinterpret_cast<void *>(type->tp_getattro); + case Py_tp_setattro: + return reinterpret_cast<void *>(type->tp_setattro); + case Py_tp_descr_get: + return reinterpret_cast<void *>(type->tp_descr_get); + case Py_tp_descr_set: + return reinterpret_cast<void *>(type->tp_descr_set); + case Py_tp_call: + return reinterpret_cast<void *>(type->tp_call); + case Py_tp_new: + return reinterpret_cast<void *>(type->tp_new); + case Py_tp_init: + return reinterpret_cast<void *>(type->tp_init); + case Py_tp_free: + return reinterpret_cast<void *>(type->tp_free); + } + assert(false); + return nullptr; } /*************************************************************************** @@ -1001,16 +1265,16 @@ static inline void *PepType_ExTP(PyTypeObject *type, size_t size) static PyTypeObject *alias{}; const char *kind = size == sizeof(SbkObjectTypePrivate) ? "SOTP" : size == sizeof(SbkEnumTypePrivate) ? "SETP" : - size == sizeof(PySideQFlagsTypePrivate) ? "PFTP" : + size == sizeof(SbkQFlagsTypePrivate) ? "PFTP" : "unk."; fprintf(stderr, "%s:%d %p x %s s=%ld\n", __func__, __LINE__, type, kind, size); PyObject *kill{}; if (strlen(env_p) > 0) { - if (size == sizeof(PySideQFlagsTypePrivate)) { + if (size == sizeof(SbkQFlagsTypePrivate)) { if (alias == nullptr) alias = type; } - if (size != sizeof(PySideQFlagsTypePrivate)) { + if (size != sizeof(SbkQFlagsTypePrivate)) { if (type == alias) Py_INCREF(kill); } diff --git a/sources/shiboken6/libshiboken/pep384impl.h b/sources/shiboken6/libshiboken/pep384impl.h index 07cdb3a6f..7188366e2 100644 --- a/sources/shiboken6/libshiboken/pep384impl.h +++ b/sources/shiboken6/libshiboken/pep384impl.h @@ -4,11 +4,6 @@ #ifndef PEP384IMPL_H #define PEP384IMPL_H -// PYSIDE-1436: Adapt to Python 3.10 -#if PY_VERSION_HEX < 0x030900A4 -# define Py_SET_REFCNT(obj, refcnt) ((Py_REFCNT(obj) = (refcnt)), (void)0) -#endif - extern "C" { @@ -55,42 +50,79 @@ typedef struct _typeobject { const char *tp_name; Py_ssize_t tp_basicsize; void *X03; // Py_ssize_t tp_itemsize; +#ifdef PEP384_INTERN destructor tp_dealloc; +#else + destructor X04; +#endif void *X05; // Py_ssize_t tp_vectorcall_offset; void *X06; // getattrfunc tp_getattr; void *X07; // setattrfunc tp_setattr; void *X08; // PyAsyncMethods *tp_as_async; +#ifdef PEP384_INTERN reprfunc tp_repr; +#else + reprfunc X09; +#endif void *X10; // PyNumberMethods *tp_as_number; void *X11; // PySequenceMethods *tp_as_sequence; void *X12; // PyMappingMethods *tp_as_mapping; void *X13; // hashfunc tp_hash; +#ifdef PEP384_INTERN ternaryfunc tp_call; - reprfunc tp_str; +#else + ternaryfunc X14; +#endif + reprfunc tp_str; // Only used for PEP384_INTERN and a shiboken test getattrofunc tp_getattro; setattrofunc tp_setattro; void *X18; // PyBufferProcs *tp_as_buffer; unsigned long tp_flags; void *X20; // const char *tp_doc; +#ifdef PEP384_INTERN traverseproc tp_traverse; inquiry tp_clear; +#else + traverseproc X21; + inquiry X22; +#endif void *X23; // richcmpfunc tp_richcompare; Py_ssize_t tp_weaklistoffset; void *X25; // getiterfunc tp_iter; +#ifdef PEP384_INTERN iternextfunc tp_iternext; +#else + iternextfunc X26; +#endif struct PyMethodDef *tp_methods; struct PyMemberDef *tp_members; struct PyGetSetDef *tp_getset; struct _typeobject *tp_base; +#ifdef PEP384_INTERN PyObject *tp_dict; descrgetfunc tp_descr_get; descrsetfunc tp_descr_set; +#else + void *X31; + descrgetfunc X32; + descrsetfunc X33; +#endif Py_ssize_t tp_dictoffset; +#ifdef PEP384_INTERN initproc tp_init; allocfunc tp_alloc; +#else + initproc X39; + allocfunc X40; +#endif newfunc tp_new; +#ifdef PEP384_INTERN freefunc tp_free; inquiry tp_is_gc; /* For PyObject_IS_GC */ +#else + freefunc X41; + inquiry X42; /* For PyObject_IS_GC */ +#endif PyObject *tp_bases; PyObject *tp_mro; /* method resolution order */ @@ -103,24 +135,16 @@ typedef struct _typeobject { && (Py_TYPE(o)->tp_is_gc == NULL || Py_TYPE(o)->tp_is_gc(o))) #endif -// This was a macro error in the limited API from the beginning. -// It was fixed in Python master, but did make it only into Python 3.8 . - -// PYSIDE-1797: This must be a runtime decision. -// Remove that when the minimum Python version is 3.8, -// because the macro PepIndex_Check bug was fixed then. -/// FIXME: Remove PepIndex_Check and pep384_issue33738.cpp when Python 3.7 is gone. -LIBSHIBOKEN_API int PepIndex_Check(PyObject *obj); - LIBSHIBOKEN_API PyObject *_PepType_Lookup(PyTypeObject *type, PyObject *name); #else // Py_LIMITED_API -#define PepIndex_Check(obj) PyIndex_Check(obj) #define _PepType_Lookup(type, name) _PyType_Lookup(type, name) #endif // Py_LIMITED_API +/// PYSIDE-939: We need the runtime version, given major << 16 + minor << 8 + micro +LIBSHIBOKEN_API long _PepRuntimeVersion(); /***************************************************************************** * * PYSIDE-535: Implement a clean type extension for PyPy @@ -139,16 +163,15 @@ LIBSHIBOKEN_API SbkEnumTypePrivate *PepType_SETP(SbkEnumType *type); LIBSHIBOKEN_API void PepType_SETP_delete(SbkEnumType *enumType); struct PySideQFlagsType; -struct PySideQFlagsTypePrivate; - -LIBSHIBOKEN_API PySideQFlagsTypePrivate *PepType_PFTP(PySideQFlagsType *type); -LIBSHIBOKEN_API void PepType_PFTP_delete(PySideQFlagsType *flagsType); +struct SbkQFlagsTypePrivate; /*****************************************************************************/ // functions used everywhere LIBSHIBOKEN_API const char *PepType_GetNameStr(PyTypeObject *type); +LIBSHIBOKEN_API PyObject *Pep_GetPartialFunction(void); + /***************************************************************************** * * RESOLVED: pydebug.h @@ -163,7 +186,17 @@ LIBSHIBOKEN_API const char *PepType_GetNameStr(PyTypeObject *type); */ LIBSHIBOKEN_API int Pep_GetFlag(const char *name); LIBSHIBOKEN_API int Pep_GetVerboseFlag(void); -#define Py_VerboseFlag Pep_GetVerboseFlag() +#endif + +// pyerrors.h +#if defined(Py_LIMITED_API) || PY_VERSION_HEX < 0x030C0000 +LIBSHIBOKEN_API PyObject *PepErr_GetRaisedException(); +LIBSHIBOKEN_API PyObject *PepException_GetArgs(PyObject *ex); +LIBSHIBOKEN_API void PepException_SetArgs(PyObject *ex, PyObject *args); +#else +# define PepErr_GetRaisedException PyErr_GetRaisedException +# define PepException_GetArgs PyException_GetArgs +# define PepException_SetArgs PyException_SetArgs #endif /***************************************************************************** @@ -201,7 +234,9 @@ LIBSHIBOKEN_API int Pep_GetVerboseFlag(void); LIBSHIBOKEN_API const char *_PepUnicode_AsString(PyObject *); enum PepUnicode_Kind { +#if PY_VERSION_HEX < 0x030C0000 PepUnicode_WCHAR_KIND = 0, +#endif PepUnicode_1BYTE_KIND = 1, PepUnicode_2BYTE_KIND = 2, PepUnicode_4BYTE_KIND = 4 @@ -216,7 +251,9 @@ LIBSHIBOKEN_API void *_PepUnicode_DATA(PyObject *str); #else enum PepUnicode_Kind { +#if PY_VERSION_HEX < 0x030C0000 PepUnicode_WCHAR_KIND = PyUnicode_WCHAR_KIND, +#endif PepUnicode_1BYTE_KIND = PyUnicode_1BYTE_KIND, PepUnicode_2BYTE_KIND = PyUnicode_2BYTE_KIND, PepUnicode_4BYTE_KIND = PyUnicode_4BYTE_KIND @@ -278,7 +315,7 @@ enum PepUnicode_Kind { #ifdef Py_LIMITED_API -typedef struct _pycfunc PyCFunctionObject; +using PyCFunctionObject = struct _pycfunc; #define PyCFunction_GET_FUNCTION(func) PyCFunction_GetFunction((PyObject *)func) #define PyCFunction_GET_SELF(func) PyCFunction_GetSelf((PyObject *)func) #define PyCFunction_GET_FLAGS(func) PyCFunction_GetFlags((PyObject *)func) @@ -393,10 +430,13 @@ LIBSHIBOKEN_API PyObject *PyMethod_Self(PyObject *); typedef struct _code PepCodeObject; LIBSHIBOKEN_API int PepCode_Get(PepCodeObject *co, const char *name); +LIBSHIBOKEN_API int PepCode_Check(PyObject *o); # define PepCode_GET_FLAGS(o) PepCode_Get(o, "co_flags") # define PepCode_GET_ARGCOUNT(o) PepCode_Get(o, "co_argcount") +LIBSHIBOKEN_API PyObject *PepFunction_GetDefaults(PyObject *function); + /* Masks for co_flags above */ # define CO_OPTIMIZED 0x0001 # define CO_NEWLOCALS 0x0002 @@ -410,7 +450,15 @@ LIBSHIBOKEN_API int PepCode_Get(PepCodeObject *co, const char *name); # define PepCodeObject PyCodeObject # define PepCode_GET_FLAGS(o) ((o)->co_flags) # define PepCode_GET_ARGCOUNT(o) ((o)->co_argcount) +# define PepCode_Check PyCode_Check + +# ifdef PYPY_VERSION + +LIBSHIBOKEN_API PyObject *PepFunction_GetDefaults(PyObject *function); +# else +# define PepFunction_GetDefaults PyFunction_GetDefaults +# endif #endif /***************************************************************************** @@ -511,9 +559,9 @@ extern LIBSHIBOKEN_API PyTypeObject *PepBuiltinMethod_TypePtr; * * This is not defined if Py_LIMITED_API is defined. */ -#if PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) +#ifdef Py_LIMITED_API LIBSHIBOKEN_API PyObject *PyImport_GetModule(PyObject *name); -#endif // PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) +#endif // Py_LIMITED_API // Evaluate a script and return the variable `result` LIBSHIBOKEN_API PyObject *PepRun_GetResult(const char *command); @@ -538,6 +586,20 @@ extern LIBSHIBOKEN_API int PepRuntime_38_flag; /***************************************************************************** * + * Runtime support for Python 3.12 incompatibility + * + */ + +LIBSHIBOKEN_API PyObject *PepType_GetDict(PyTypeObject *type); + +// This function does not exist as PyType_SetDict. But because tp_dict +// is no longer considered to be accessible, we treat it as such. +LIBSHIBOKEN_API int PepType_SetDict(PyTypeObject *type, PyObject *dict); + +LIBSHIBOKEN_API void *PepType_GetSlot(PyTypeObject *type, int aSlot); + +/***************************************************************************** + * * Module Initialization * */ diff --git a/sources/shiboken6/libshiboken/pyobjectholder.h b/sources/shiboken6/libshiboken/pyobjectholder.h new file mode 100644 index 000000000..857748c2f --- /dev/null +++ b/sources/shiboken6/libshiboken/pyobjectholder.h @@ -0,0 +1,86 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef PYOBJECTHOLDER_H +#define PYOBJECTHOLDER_H + +#include "sbkpython.h" + +#include <cassert> +#include <utility> + +namespace Shiboken +{ + +/// PyObjectHolder holds a PyObject pointer, keeping a reference decrementing +/// its reference counter when destroyed. It makes sure to hold the GIL when +/// releasing. It implements copy/move semantics and is mainly intended as a +/// base class for functors holding a callable which can be passed around and +/// stored in containers or moved from freely. +/// For one-shot functors, release() can be invoked after the call. +class PyObjectHolder +{ +public: + PyObjectHolder() noexcept = default; + + /// PyObjectHolder constructor. + /// \param pyobj A reference to a Python object + explicit PyObjectHolder(PyObject *pyObj) noexcept : m_pyObj(pyObj) + { + assert(pyObj != nullptr); + Py_INCREF(m_pyObj); + } + + PyObjectHolder(const PyObjectHolder &o) noexcept : m_pyObj(o.m_pyObj) + { + Py_XINCREF(m_pyObj); + } + + PyObjectHolder &operator=(const PyObjectHolder &o) noexcept + { + if (this != &o) { + m_pyObj = o.m_pyObj; + Py_XINCREF(m_pyObj); + } + return *this; + } + + PyObjectHolder(PyObjectHolder &&o) noexcept : m_pyObj{std::exchange(o.m_pyObj, nullptr)} {} + + PyObjectHolder &operator=(PyObjectHolder &&o) noexcept + { + m_pyObj = std::exchange(o.m_pyObj, nullptr); + return *this; + } + + /// Decref the python reference + ~PyObjectHolder() { release(); } + + [[nodiscard]] bool isNull() const { return m_pyObj == nullptr; } + [[nodiscard]] operator bool() const { return m_pyObj != nullptr; } + + /// Returns the pointer of the Python object being held. + [[nodiscard]] PyObject *object() const { return m_pyObj; } + [[nodiscard]] operator PyObject *() const { return m_pyObj; } + + [[nodiscard]] PyObject *operator->() { return m_pyObj; } + +protected: + void release() + { + if (m_pyObj != nullptr) { + assert(Py_IsInitialized()); + auto gstate = PyGILState_Ensure(); + Py_DECREF(m_pyObj); + PyGILState_Release(gstate); + m_pyObj = nullptr; + } + } + +private: + PyObject *m_pyObj = nullptr; +}; + +} // namespace Shiboken + +#endif // PYOBJECTHOLDER_H diff --git a/sources/shiboken6/libshiboken/sbkarrayconverter.cpp b/sources/shiboken6/libshiboken/sbkarrayconverter.cpp index 8af310a53..bcc3fb767 100644 --- a/sources/shiboken6/libshiboken/sbkarrayconverter.cpp +++ b/sources/shiboken6/libshiboken/sbkarrayconverter.cpp @@ -14,8 +14,7 @@ static SbkArrayConverter *ArrayTypeConverters[Shiboken::Conversions::SBK_ARRAY_IDX_SIZE] [2] = {}; -namespace Shiboken { -namespace Conversions { +namespace Shiboken::Conversions { // Check whether Predicate is true for all elements of a sequence template <class Predicate> @@ -244,5 +243,4 @@ void setArrayTypeConverter(int index, int dimension, SbkArrayConverter *c) ArrayTypeConverters[index][dimension - 1] = c; } -} // namespace Conversions -} // namespace Shiboken +} // namespace Shiboken::Conversions diff --git a/sources/shiboken6/libshiboken/sbkarrayconverter.h b/sources/shiboken6/libshiboken/sbkarrayconverter.h index 97bd8ac6f..f07cb1d70 100644 --- a/sources/shiboken6/libshiboken/sbkarrayconverter.h +++ b/sources/shiboken6/libshiboken/sbkarrayconverter.h @@ -11,8 +11,7 @@ extern "C" { struct SbkArrayConverter; } -namespace Shiboken { -namespace Conversions { +namespace Shiboken::Conversions { enum : int { SBK_UNIMPLEMENTED_ARRAY_IDX, @@ -132,7 +131,6 @@ void ArrayHandle<T>::destroy() m_owned = false; } -} // namespace Conversions -} // namespace Shiboken +} // namespace Shiboken::Conversions #endif // SBKARRAYCONVERTERS_H diff --git a/sources/shiboken6/libshiboken/sbkarrayconverter_p.h b/sources/shiboken6/libshiboken/sbkarrayconverter_p.h index db92e56af..63d03fb12 100644 --- a/sources/shiboken6/libshiboken/sbkarrayconverter_p.h +++ b/sources/shiboken6/libshiboken/sbkarrayconverter_p.h @@ -10,7 +10,7 @@ extern "C" { -typedef PythonToCppFunc (*IsArrayConvertibleToCppFunc)(PyObject *, int dim1, int dim2); +using IsArrayConvertibleToCppFunc = PythonToCppFunc (*)(PyObject *, int dim1, int dim2); /** * \internal * Private structure of SbkArrayConverter. diff --git a/sources/shiboken6/libshiboken/sbkcontainer.cpp b/sources/shiboken6/libshiboken/sbkcontainer.cpp index 13c9f1a29..7de1f03e6 100644 --- a/sources/shiboken6/libshiboken/sbkcontainer.cpp +++ b/sources/shiboken6/libshiboken/sbkcontainer.cpp @@ -3,14 +3,17 @@ #include "sbkcontainer.h" #include "sbkstaticstrings.h" +#include "autodecref.h" namespace Shiboken { bool isOpaqueContainer(PyObject *o) { + if (!o) + return false; + Shiboken::AutoDecRef tpDict(PepType_GetDict(o->ob_type)); return o != nullptr && o != Py_None - && PyDict_Contains(o->ob_type->tp_dict, - Shiboken::PyMagicName::opaque_container()) == 1; + && PyDict_Contains(tpDict.object(), Shiboken::PyMagicName::opaque_container()) == 1; } } // Shiboken diff --git a/sources/shiboken6/libshiboken/sbkcontainer.h b/sources/shiboken6/libshiboken/sbkcontainer.h index 22513d3a0..8ad5aadc6 100644 --- a/sources/shiboken6/libshiboken/sbkcontainer.h +++ b/sources/shiboken6/libshiboken/sbkcontainer.h @@ -63,7 +63,8 @@ public: static PyObject *tpNew(PyTypeObject *subtype, PyObject * /* args */, PyObject * /* kwds */) { - auto *me = reinterpret_cast<ShibokenContainer *>(subtype->tp_alloc(subtype, 0)); + allocfunc allocFunc = reinterpret_cast<allocfunc>(PepType_GetSlot(subtype, Py_tp_alloc)); + auto *me = reinterpret_cast<ShibokenContainer *>(allocFunc(subtype, 0)); auto *d = new ShibokenSequenceContainerPrivate; d->m_list = new SequenceContainer; d->m_ownsList = true; @@ -71,6 +72,13 @@ public: return reinterpret_cast<PyObject *>(me); } + static PyObject *tpNewInvalid(PyTypeObject * /* subtype */, PyObject * /* args */, PyObject * /* kwds */) + { + return PyErr_Format(PyExc_NotImplementedError, + "Opaque containers of type '%s' cannot be instantiated.", + typeid(SequenceContainer).name()); + } + static int tpInit(PyObject * /* self */, PyObject * /* args */, PyObject * /* kwds */) { return 0; @@ -83,7 +91,9 @@ public: if (d->m_ownsList) delete d->m_list; delete d; - Py_TYPE(pySelf)->tp_base->tp_free(self); + auto freeFunc = reinterpret_cast<freefunc>(PepType_GetSlot(Py_TYPE(pySelf)->tp_base, + Py_tp_free)); + freeFunc(self); } static Py_ssize_t sqLen(PyObject *self) @@ -94,11 +104,9 @@ public: static PyObject *sqGetItem(PyObject *self, Py_ssize_t i) { auto *d = get(self); - if (i < 0 || i >= Py_ssize_t(d->m_list->size())) { - PyErr_SetString(PyExc_IndexError, "index out of bounds"); - return nullptr; - } - auto it = d->m_list->cbegin(); + if (i < 0 || i >= Py_ssize_t(d->m_list->size())) + return PyErr_Format(PyExc_IndexError, "index out of bounds"); + auto it = std::cbegin(*d->m_list); std::advance(it, i); return ShibokenContainerValueConverter<value_type>::convertValueToPython(*it); } @@ -110,7 +118,7 @@ public: PyErr_SetString(PyExc_IndexError, "index out of bounds"); return -1; } - auto it = d->m_list->begin(); + auto it = std::begin(*d->m_list); std::advance(it, i); OptionalValue value = ShibokenContainerValueConverter<value_type>::convertValueToCpp(pyArg); if (!value.has_value()) @@ -122,14 +130,10 @@ public: static PyObject *push_back(PyObject *self, PyObject *pyArg) { auto *d = get(self); - if (!ShibokenContainerValueConverter<value_type>::checkValue(pyArg)) { - PyErr_SetString(PyExc_TypeError, "wrong type passed to append."); - return nullptr; - } - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (!ShibokenContainerValueConverter<value_type>::checkValue(pyArg)) + return PyErr_Format(PyExc_TypeError, "wrong type passed to append."); + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); OptionalValue value = ShibokenContainerValueConverter<value_type>::convertValueToCpp(pyArg); if (!value.has_value()) @@ -141,14 +145,10 @@ public: static PyObject *push_front(PyObject *self, PyObject *pyArg) { auto *d = get(self); - if (!ShibokenContainerValueConverter<value_type>::checkValue(pyArg)) { - PyErr_SetString(PyExc_TypeError, "wrong type passed to append."); - return nullptr; - } - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (!ShibokenContainerValueConverter<value_type>::checkValue(pyArg)) + return PyErr_Format(PyExc_TypeError, "wrong type passed to append."); + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); OptionalValue value = ShibokenContainerValueConverter<value_type>::convertValueToCpp(pyArg); if (!value.has_value()) @@ -160,10 +160,8 @@ public: static PyObject *clear(PyObject *self) { auto *d = get(self); - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); d->m_list->clear(); Py_RETURN_NONE; @@ -172,10 +170,8 @@ public: static PyObject *pop_back(PyObject *self) { auto *d = get(self); - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); d->m_list->pop_back(); Py_RETURN_NONE; @@ -184,10 +180,8 @@ public: static PyObject *pop_front(PyObject *self) { auto *d = get(self); - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); d->m_list->pop_front(); Py_RETURN_NONE; @@ -197,21 +191,16 @@ public: static PyObject *reserve(PyObject *self, PyObject *pyArg) { auto *d = get(self); - if (PyLong_Check(pyArg) == 0) { - PyErr_SetString(PyExc_TypeError, "wrong type passed to reserve()."); - return nullptr; - } - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (PyLong_Check(pyArg) == 0) + return PyErr_Format(PyExc_TypeError, "wrong type passed to reserve()."); + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); if constexpr (ShibokenContainerHasReserve<SequenceContainer>::value) { const Py_ssize_t size = PyLong_AsSsize_t(pyArg); d->m_list->reserve(size); } else { - PyErr_SetString(PyExc_TypeError, "Container does not support reserve()."); - return nullptr; + return PyErr_Format(PyExc_TypeError, "Container does not support reserve()."); } Py_RETURN_NONE; diff --git a/sources/shiboken6/libshiboken/sbkconverter.cpp b/sources/shiboken6/libshiboken/sbkconverter.cpp index 309810290..9ab674415 100644 --- a/sources/shiboken6/libshiboken/sbkconverter.cpp +++ b/sources/shiboken6/libshiboken/sbkconverter.cpp @@ -4,6 +4,7 @@ #include "sbkconverter.h" #include "sbkconverter_p.h" #include "sbkarrayconverter_p.h" +#include "sbkmodule.h" #include "basewrapper_p.h" #include "bindingmanager.h" #include "autodecref.h" @@ -11,15 +12,19 @@ #include "voidptr.h" #include <string> +#include <cstring> +#include <iostream> #include <unordered_map> +#include <unordered_set> +#include <map> +#include <set> static SbkConverter **PrimitiveTypeConverters; using ConvertersMap = std::unordered_map<std::string, SbkConverter *>; static ConvertersMap converters; -namespace Shiboken { -namespace Conversions { +namespace Shiboken::Conversions { void initArrayConverters(); @@ -72,6 +77,103 @@ void init() initArrayConverters(); } +static void dumpPyTypeObject(std::ostream &str, PyTypeObject *t) +{ + str << "\nPython type "; + if (t == nullptr) { + str << "<None>"; + return; + } + str << '"' << t->tp_name << '"'; + if (t->tp_base != nullptr && t->tp_base != &PyBaseObject_Type) + str << '(' << t->tp_base->tp_name << ')'; +} + +static void dumpSbkConverter(std::ostream &str, const SbkConverter *c) +{ + str << "SbkConverter " << static_cast<const void *>(c) << ": "; + if (c->pointerToPython != nullptr) + str << ", C++ pointer->Python"; + if (c->copyToPython != nullptr) + str << ", copy->Python"; + if (c->toCppPointerConversion.second != nullptr) + str << ", Python->C++ pointer"; + if (!c->toCppConversions.empty()) + str << ", " << c->toCppConversions.size() << " Python->C++ conversions"; +} + +// Less than operator for a PyTypeObject for dumping the converter map +static bool pyTypeObjectLessThan(const PyTypeObject *t1, const PyTypeObject *t2) +{ + const bool isNull1 = t1 == nullptr; + const bool isNull2 = t2 == nullptr; + if (isNull1 || isNull2) + return isNull1 && !isNull2; + // Internal types (lower case) first + const bool isInternal1 = std::islower(t1->tp_name[0]); + const bool isInternal2 = std::islower(t2->tp_name[0]); + if (isInternal1 != isInternal2) + return !isInternal2; + return std::strcmp(t1->tp_name, t2->tp_name) < 0; +} + +void dumpConverters() +{ + struct PyTypeObjectLess { + + bool operator()(const PyTypeObject *t1, const PyTypeObject *t2) const { + return pyTypeObjectLessThan(t1, t2); + } + }; + + using StringSet = std::set<std::string>; + using SbkConverterNamesMap = std::unordered_map<SbkConverter *, StringSet>; + using PyTypeObjectConverterMap = std::map<PyTypeObject *, SbkConverterNamesMap, + PyTypeObjectLess>; + + auto &str = std::cerr; + + // Sort the entries by the associated PyTypeObjects and converters + PyTypeObjectConverterMap pyTypeObjectConverterMap; + for (const auto &converter : converters) { + auto *sbkConverter = converter.second; + if (sbkConverter == nullptr) { + str << "Non-existent: \"" << converter.first << "\"\n"; + continue; + } + auto *typeObject = sbkConverter->pythonType; + auto typeIt = pyTypeObjectConverterMap.find(typeObject); + if (typeIt == pyTypeObjectConverterMap.end()) + typeIt = pyTypeObjectConverterMap.insert(std::make_pair(typeObject, + SbkConverterNamesMap{})).first; + SbkConverterNamesMap &sbkConverterMap = typeIt->second; + auto convIt = sbkConverterMap.find(sbkConverter); + if (convIt == sbkConverterMap.end()) + convIt = sbkConverterMap.insert(std::make_pair(sbkConverter, + StringSet{})).first; + convIt->second.insert(converter.first); + } + + for (const auto &tc : pyTypeObjectConverterMap) { + dumpPyTypeObject(str, tc.first); + str << ", " << tc.second.size() << " converter(s):\n"; + for (const auto &cn : tc.second) { + str << " "; + dumpSbkConverter(str, cn.first); + str << ", " << cn.second.size() << " alias(es):"; + int i = 0; + for (const auto &name : cn.second) { + if ((i++ % 5) == 0) + str << "\n "; + str << " \"" << name << '"'; + } + str << '\n'; + } + } + + str << '\n'; +} + SbkConverter *createConverterObject(PyTypeObject *type, PythonToCppFunc toCppPointerConvFunc, IsConvertibleToCppFunc toCppPointerCheckFunc, @@ -147,6 +249,13 @@ void addPythonToCppValueConversion(PyTypeObject *type, addPythonToCppValueConversion(sotp->converter, pythonToCppFunc, isConvertibleToCppFunc); } +void addPythonToCppValueConversion(Shiboken::Module::TypeInitStruct typeStruct, + PythonToCppFunc pythonToCppFunc, + IsConvertibleToCppFunc isConvertibleToCppFunc) +{ + addPythonToCppValueConversion(typeStruct.type, pythonToCppFunc, isConvertibleToCppFunc); +} + PyObject *pointerToPython(PyTypeObject *type, const void *cppIn) { auto *sotp = PepType_SOTP(type); @@ -228,6 +337,11 @@ PythonToCppConversion pythonToCppPointerConversion(PyTypeObject *type, PyObject return {}; } +PythonToCppConversion pythonToCppPointerConversion(Module::TypeInitStruct typeStruct, PyObject *pyIn) +{ + return pythonToCppPointerConversion(typeStruct.type, pyIn); +} + static inline PythonToCppFunc IsPythonToCppConvertible(const SbkConverter *converter, PyObject *pyIn) { assert(pyIn); @@ -403,19 +517,55 @@ bool isImplicitConversion(PyTypeObject *type, PythonToCppFunc toCppFunc) return toCppFunc != (*conv).second; } -void registerConverterName(SbkConverter *converter , const char *typeName) +void registerConverterName(SbkConverter *converter, const char *typeName) { auto iter = converters.find(typeName); if (iter == converters.end()) converters.insert(std::make_pair(typeName, converter)); } -SbkConverter *getConverter(const char *typeName) +static std::string getRealTypeName(const std::string &typeName) +{ + auto size = typeName.size(); + if (std::isalnum(typeName[size - 1]) == 0) + return typeName.substr(0, size - 1); + return typeName; +} + +// PYSIDE-2404: Build a negative cache of already failed lookups. +// The resulting list must be reset after each new import, +// because that can change results. Also clear the cache after +// reaching some threashold. +static std::unordered_set<std::string> nonExistingTypeNames{}; + +// Arbitrary size limit to prevent random name overflows. +static constexpr std::size_t negativeCacheLimit = 50; + +static void rememberAsNonexistent(const std::string &typeName) +{ + if (nonExistingTypeNames.size() > negativeCacheLimit) + clearNegativeLazyCache(); + converters.insert(std::make_pair(typeName, nullptr)); + nonExistingTypeNames.insert(typeName); +} + +SbkConverter *getConverter(const char *typeNameC) { - ConvertersMap::const_iterator it = converters.find(typeName); + std::string typeName = typeNameC; + auto it = converters.find(typeName); + // PYSIDE-2404: This can also contain explicit nullptr as a negative cache. + if (it != converters.end()) + return it->second; + // PYSIDE-2404: Did not find the name. Load the lazy classes + // which have this name and try again. + Shiboken::Module::loadLazyClassesWithName(getRealTypeName(typeName).c_str()); + it = converters.find(typeName); if (it != converters.end()) return it->second; - if (Py_VerboseFlag > 0) { + // Cache the negative result. Don't forget to clear the cache for new modules. + rememberAsNonexistent(typeName); + + if (Shiboken::pyVerbose() > 0) { const std::string message = std::string("Can't find type resolver for type '") + typeName + "'."; PyErr_WarnEx(PyExc_RuntimeWarning, message.c_str(), 0); @@ -423,6 +573,15 @@ SbkConverter *getConverter(const char *typeName) return nullptr; } +void clearNegativeLazyCache() +{ + for (const auto &typeName : nonExistingTypeNames) { + auto it = converters.find(typeName); + converters.erase(it); + } + nonExistingTypeNames.clear(); +} + SbkConverter *primitiveTypeConverter(int index) { return PrimitiveTypeConverters[index]; @@ -748,4 +907,4 @@ void SpecificConverter::toCpp(PyObject *pyIn, void *cppOut) } } -} } // namespace Shiboken::Conversions +} // namespace Shiboken::Conversions diff --git a/sources/shiboken6/libshiboken/sbkconverter.h b/sources/shiboken6/libshiboken/sbkconverter.h index 1428b90e2..0d68f3faf 100644 --- a/sources/shiboken6/libshiboken/sbkconverter.h +++ b/sources/shiboken6/libshiboken/sbkconverter.h @@ -5,9 +5,9 @@ #define SBK_CONVERTER_H #include "sbkpython.h" +#include "sbkmodule.h" #include "shibokenmacros.h" #include "sbkenum.h" -#include "sbkenum_p.h" #include "basewrapper_p.h" #include <limits> @@ -43,7 +43,7 @@ struct SbkArrayConverter; * * C++ -> Python */ -typedef PyObject *(*CppToPythonFunc)(const void *); +using CppToPythonFunc = PyObject *(*)(const void *); /** * This function converts a Python object to a C++ value, it may be @@ -56,7 +56,7 @@ typedef PyObject *(*CppToPythonFunc)(const void *); * * Python -> C++ */ -typedef void (*PythonToCppFunc)(PyObject *,void *); +using PythonToCppFunc = void (*)(PyObject *,void *); /** * Checks if the Python object passed in the argument is convertible to a @@ -67,7 +67,7 @@ typedef void (*PythonToCppFunc)(PyObject *,void *); * * Python -> C++ ? */ -typedef PythonToCppFunc (*IsConvertibleToCppFunc)(PyObject *); +using IsConvertibleToCppFunc = PythonToCppFunc (*)(PyObject *); } // extern "C" @@ -147,6 +147,9 @@ LIBSHIBOKEN_API void addPythonToCppValueConversion(SbkConverter *converter, LIBSHIBOKEN_API void addPythonToCppValueConversion(PyTypeObject *type, PythonToCppFunc pythonToCppFunc, IsConvertibleToCppFunc isConvertibleToCppFunc); +LIBSHIBOKEN_API void addPythonToCppValueConversion(Shiboken::Module::TypeInitStruct typeStruct, + PythonToCppFunc pythonToCppFunc, + IsConvertibleToCppFunc isConvertibleToCppFunc); // C++ -> Python --------------------------------------------------------------------------- @@ -204,6 +207,7 @@ struct PythonToCppConversion */ LIBSHIBOKEN_API PythonToCppFunc isPythonToCppPointerConvertible(PyTypeObject *type, PyObject *pyIn); LIBSHIBOKEN_API PythonToCppConversion pythonToCppPointerConversion(PyTypeObject *type, PyObject *pyIn); +LIBSHIBOKEN_API PythonToCppConversion pythonToCppPointerConversion(Module::TypeInitStruct typeStruct, PyObject *pyIn); /** * Returns a Python to C++ conversion function if the Python object is convertible to a C++ value. @@ -410,7 +414,7 @@ template<> inline PyTypeObject *SbkType<std::nullptr_t>() { return Py_TYPE(&_Py_ #define SbkChar_Check(X) (PyNumber_Check(X) || Shiboken::String::checkChar(X)) struct PySideQFlagsType; -struct PySideQFlagsTypePrivate +struct SbkQFlagsTypePrivate { SbkConverter *converter; }; diff --git a/sources/shiboken6/libshiboken/sbkconverter_p.h b/sources/shiboken6/libshiboken/sbkconverter_p.h index 27126fbb1..08fc4c8e1 100644 --- a/sources/shiboken6/libshiboken/sbkconverter_p.h +++ b/sources/shiboken6/libshiboken/sbkconverter_p.h @@ -278,7 +278,7 @@ struct Primitive<PY_LONG_LONG> : OnePrimitive<PY_LONG_LONG> { PY_LONG_LONG result = PyLong_AsLongLong(pyIn); if (OverFlowChecker<PY_LONG_LONG>::check(result, pyIn)) - PyErr_SetObject(PyExc_OverflowError, 0); + PyErr_SetObject(PyExc_OverflowError, nullptr); *reinterpret_cast<PY_LONG_LONG * >(cppOut) = result; } static PythonToCppFunc isConvertible(PyObject *pyIn) @@ -327,7 +327,7 @@ struct FloatPrimitive : TwoPrimitive<FLOAT> } static void toCpp(PyObject *pyIn, void *cppOut) { - *reinterpret_cast<FLOAT *>(cppOut) = FLOAT(PyLong_AsLong(pyIn)); + *reinterpret_cast<FLOAT *>(cppOut) = FLOAT(PyLong_AsDouble(pyIn)); } static PythonToCppFunc isConvertible(PyObject *pyIn) { @@ -524,13 +524,19 @@ struct Primitive<std::nullptr_t> : OnePrimitive<std::nullptr_t> } }; -namespace Shiboken { -namespace Conversions { +namespace Shiboken::Conversions { + SbkConverter *createConverterObject(PyTypeObject *type, PythonToCppFunc toCppPointerConvFunc, IsConvertibleToCppFunc toCppPointerCheckFunc, CppToPythonFunc pointerToPythonFunc, CppToPythonFunc copyToPythonFunc); -} // namespace Conversions -} // namespace Shiboken + +LIBSHIBOKEN_API void dumpConverters(); + +/// Interface for sbkmodule which must reset cache when new module is loaded. +LIBSHIBOKEN_API void clearNegativeLazyCache(); + +} // namespace Shiboken::Conversions + #endif // SBK_CONVERTER_P_H diff --git a/sources/shiboken6/libshiboken/sbkcppstring.cpp b/sources/shiboken6/libshiboken/sbkcppstring.cpp index 42b09111c..8e8324f5e 100644 --- a/sources/shiboken6/libshiboken/sbkcppstring.cpp +++ b/sources/shiboken6/libshiboken/sbkcppstring.cpp @@ -12,6 +12,11 @@ PyObject *fromCppString(const std::string &value) return PyUnicode_FromStringAndSize(value.data(), value.size()); } +PyObject *fromCppStringView(std::string_view value) +{ + return PyUnicode_FromStringAndSize(value.data(), value.size()); +} + PyObject *fromCppWString(const std::wstring &value) { return PyUnicode_FromWideChar(value.data(), value.size()); diff --git a/sources/shiboken6/libshiboken/sbkcppstring.h b/sources/shiboken6/libshiboken/sbkcppstring.h index f418ea8dd..7ffe11c75 100644 --- a/sources/shiboken6/libshiboken/sbkcppstring.h +++ b/sources/shiboken6/libshiboken/sbkcppstring.h @@ -8,10 +8,12 @@ #include "shibokenmacros.h" #include <string> +#include <string_view> namespace Shiboken::String { LIBSHIBOKEN_API PyObject *fromCppString(const std::string &value); + LIBSHIBOKEN_API PyObject *fromCppStringView(std::string_view value); LIBSHIBOKEN_API PyObject *fromCppWString(const std::wstring &value); LIBSHIBOKEN_API void toCppString(PyObject *str, std::string *value); LIBSHIBOKEN_API void toCppWString(PyObject *str, std::wstring *value); diff --git a/sources/shiboken6/libshiboken/sbkcpptonumpy.cpp b/sources/shiboken6/libshiboken/sbkcpptonumpy.cpp index 44e900f01..7637efa70 100644 --- a/sources/shiboken6/libshiboken/sbkcpptonumpy.cpp +++ b/sources/shiboken6/libshiboken/sbkcpptonumpy.cpp @@ -49,17 +49,17 @@ PyObject *createByteArray1(Py_ssize_t, const uint8_t *) PyObject *createDoubleArray1(Py_ssize_t, const double *) { - return Py_None; + Py_RETURN_NONE; } PyObject *createFloatArray1(Py_ssize_t, const float *) { - return Py_None; + Py_RETURN_NONE; } PyObject *createIntArray1(Py_ssize_t, const int *) { - return Py_None; + Py_RETURN_NONE; } #endif // !HAVE_NUMPY diff --git a/sources/shiboken6/libshiboken/sbkenum.cpp b/sources/shiboken6/libshiboken/sbkenum.cpp index c5eae2fbb..d39369979 100644 --- a/sources/shiboken6/libshiboken/sbkenum.cpp +++ b/sources/shiboken6/libshiboken/sbkenum.cpp @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "sbkenum.h" -#include "sbkenum_p.h" #include "sbkstring.h" #include "sbkstaticstrings.h" #include "sbkstaticstrings_p.h" @@ -11,393 +10,28 @@ #include "autodecref.h" #include "sbkpython.h" #include "signature.h" -#include "helper.h" #include <cstring> #include <vector> #include <sstream> -#include <iostream> - -#define SbkEnumType_Check(o) (Py_TYPE(Py_TYPE(o)) == SbkEnumType_TypeF()) -using enum_func = PyObject *(*)(PyObject *, PyObject *); using namespace Shiboken; extern "C" { -// forward -struct lastEnumCreated; - -// forward -static PyTypeObject *recordCurrentEnum(PyObject *scopeOrModule, - const char *name, - PyTypeObject *enumType, - PyTypeObject *flagsType); - struct SbkEnumType { PyTypeObject type; }; -static void cleanupEnumTypes(); - -struct SbkEnumObject -{ - PyObject_HEAD - Enum::EnumValueType ob_value; - PyObject *ob_name; -}; - -static PyTypeObject *SbkEnum_TypeF(); // forward - -static PyObject *SbkEnumObject_repr(PyObject *self) -{ - const SbkEnumObject *enumObj = reinterpret_cast<SbkEnumObject *>(self); - auto name = Py_TYPE(self)->tp_name; - if (enumObj->ob_name) { - return String::fromFormat("%s.%s", name, PyBytes_AS_STRING(enumObj->ob_name)); - } - return String::fromFormat("%s(%ld)", name, enumObj->ob_value); -} - -static PyObject *SbkEnumObject_name(PyObject *self, void *) -{ - auto *enum_self = reinterpret_cast<SbkEnumObject *>(self); - - if (enum_self->ob_name == nullptr) - Py_RETURN_NONE; - - Py_INCREF(enum_self->ob_name); - return enum_self->ob_name; -} - -static PyObject *SbkEnum_tp_new(PyTypeObject *type, PyObject *args, PyObject *) -{ - long itemValue = 0; - if (!PyArg_ParseTuple(args, "|l:__new__", &itemValue)) - return nullptr; - - if (type == SbkEnum_TypeF()) { - PyErr_Format(PyExc_TypeError, "You cannot use %s directly", type->tp_name); - return nullptr; - } - - SbkEnumObject *self = PyObject_New(SbkEnumObject, type); - if (!self) - return nullptr; - self->ob_value = itemValue; - AutoDecRef item(Enum::getEnumItemFromValue(type, itemValue)); - self->ob_name = item.object() ? SbkEnumObject_name(item, nullptr) : nullptr; - return reinterpret_cast<PyObject *>(self); -} - -static const char *SbkEnum_SignatureStrings[] = { - "Shiboken.Enum(self,itemValue:int=0)", - nullptr}; // Sentinel - -static void enum_object_dealloc(PyObject *ob) -{ - auto *self = reinterpret_cast<SbkEnumObject *>(ob); - Py_XDECREF(self->ob_name); - Sbk_object_dealloc(ob); -} - -static PyObject *_enum_op(enum_func f, PyObject *a, PyObject *b) { - PyObject *valA = a; - PyObject *valB = b; - PyObject *result = nullptr; - bool enumA = false; - bool enumB = false; - - // We are not allowing floats - if (!PyFloat_Check(valA) && !PyFloat_Check(valB)) { - // Check if both variables are SbkEnumObject - if (SbkEnumType_Check(valA)) { - valA = PyLong_FromLong(reinterpret_cast<SbkEnumObject *>(valA)->ob_value); - enumA = true; - } - if (SbkEnumType_Check(valB)) { - valB = PyLong_FromLong(reinterpret_cast<SbkEnumObject *>(valB)->ob_value); - enumB = true; - } - } - - // Without an enum we are not supporting the operation - if (!(enumA || enumB)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - - result = f(valA, valB); - - // Decreasing the reference of the used variables a and b. - if (enumA) - Py_DECREF(valA); - if (enumB) - Py_DECREF(valB); - return result; -} - -/* Notes: - * On Py3k land we use long type when using integer numbers. However, on older - * versions of Python (version 2) we need to convert it to int type, - * respectively. - * - * Thus calling PyLong_FromLong() will result in calling PyLong_FromLong in - * Py3k. - */ -static PyObject *enum_int(PyObject *v) -{ - return PyLong_FromLong(reinterpret_cast<SbkEnumObject *>(v)->ob_value); -} - -static PyObject *enum_and(PyObject *self, PyObject *b) -{ - return _enum_op(PyNumber_And, self, b); -} - -static PyObject *enum_or(PyObject *self, PyObject *b) -{ - return _enum_op(PyNumber_Or, self, b); -} - -static PyObject *enum_xor(PyObject *self, PyObject *b) -{ - return _enum_op(PyNumber_Xor, self, b); -} - -static int enum_bool(PyObject *v) -{ - return (reinterpret_cast<SbkEnumObject *>(v)->ob_value > 0); -} - -static PyObject *enum_add(PyObject *self, PyObject *v) -{ - return _enum_op(PyNumber_Add, self, v); -} - -static PyObject *enum_subtract(PyObject *self, PyObject *v) -{ - return _enum_op(PyNumber_Subtract, self, v); -} - -static PyObject *enum_multiply(PyObject *self, PyObject *v) -{ - return _enum_op(PyNumber_Multiply, self, v); -} - -static PyObject *enum_richcompare(PyObject *self, PyObject *other, int op) -{ - PyObject *valA = self; - PyObject *valB = other; - PyObject *result = nullptr; - bool enumA = false; - bool enumB = false; - - // We are not allowing floats - if (!PyFloat_Check(valA) && !PyFloat_Check(valB)) { - - // Check if both variables are SbkEnumObject - if (SbkEnumType_Check(valA)) { - valA = PyLong_FromLong(reinterpret_cast<SbkEnumObject *>(valA)->ob_value); - enumA = true; - } - if (SbkEnumType_Check(valB)) { - valB = PyLong_FromLong(reinterpret_cast<SbkEnumObject *>(valB)->ob_value); - enumB =true; - } - } - - // Without an enum we are not supporting the operation - if (!(enumA || enumB)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - result = PyObject_RichCompare(valA, valB, op); - - // Decreasing the reference of the used variables a and b. - if (enumA) - Py_DECREF(valA); - if (enumB) - Py_DECREF(valB); - - return result; -} - -static Py_hash_t enum_hash(PyObject *pyObj) -{ - Py_hash_t val = reinterpret_cast<SbkEnumObject *>(pyObj)->ob_value; - if (val == -1) - val = -2; - return val; -} - -static PyGetSetDef SbkEnumGetSetList[] = { - {const_cast<char *>("name"), SbkEnumObject_name, nullptr, nullptr, nullptr}, - {nullptr, nullptr, nullptr, nullptr, nullptr} // Sentinel -}; - -static void SbkEnumTypeDealloc(PyObject *pyObj); -static PyTypeObject *SbkEnumTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds); - -static PyGetSetDef SbkEnumType_getsetlist[] = { - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(Sbk_TypeGet___signature__), - nullptr, nullptr, nullptr}, - {nullptr, nullptr, nullptr, nullptr, nullptr} // Sentinel -}; - -static PyType_Slot SbkEnumType_Type_slots[] = { - {Py_tp_dealloc, reinterpret_cast<void *>(SbkEnumTypeDealloc)}, - {Py_tp_base, reinterpret_cast<void *>(&PyType_Type)}, - {Py_tp_alloc, reinterpret_cast<void *>(PyType_GenericAlloc)}, - {Py_tp_new, reinterpret_cast<void *>(SbkEnumTypeTpNew)}, - {Py_tp_free, reinterpret_cast<void *>(PyObject_GC_Del)}, - {Py_tp_getset, reinterpret_cast<void *>(SbkEnumType_getsetlist)}, - {0, nullptr} -}; - -// PYSIDE-535: The tp_itemsize field is inherited and does not need to be set. -// In PyPy, it _must_ not be set, because it would have the meaning that a -// `__len__` field must be defined. Not doing so creates a hard-to-find crash. -static PyType_Spec SbkEnumType_Type_spec = { - "1:Shiboken.EnumMeta", - 0, - 0, // sizeof(PyMemberDef), not for PyPy without a __len__ defined - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, - SbkEnumType_Type_slots, -}; - -PyTypeObject *SbkEnumType_TypeF(void) -{ - static auto *type = SbkType_FromSpec(&SbkEnumType_Type_spec); - return type; -} - -static void SbkEnumTypeDealloc(PyObject *pyObj) -{ - auto *enumType = reinterpret_cast<SbkEnumType *>(pyObj); - auto *setp = PepType_SETP(enumType); - - PyObject_GC_UnTrack(pyObj); -#ifndef Py_LIMITED_API -# if PY_VERSION_HEX >= 0x030A0000 - Py_TRASHCAN_BEGIN(pyObj, 1); -# else - Py_TRASHCAN_SAFE_BEGIN(pyObj); -# endif -#endif - if (setp->converter) - Conversions::deleteConverter(setp->converter); - PepType_SETP_delete(enumType); -#ifndef Py_LIMITED_API -# if PY_VERSION_HEX >= 0x030A0000 - Py_TRASHCAN_END; -# else - Py_TRASHCAN_SAFE_END(pyObj); -# endif -#endif - if (PepRuntime_38_flag) { - // PYSIDE-939: Handling references correctly. - // This was not needed before Python 3.8 (Python issue 35810) - Py_DECREF(Py_TYPE(pyObj)); - } -} - -PyTypeObject *SbkEnumTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds) -{ - init_enum(); - return PepType_Type_tp_new(metatype, args, kwds); -} - -} // extern "C" - -/////////////////////////////////////////////////////////////// -// -// PYSIDE-15: Pickling Support for Qt Enum objects -// This works very well and fixes the issue. -// -extern "C" { - -static PyObject *enum_unpickler = nullptr; - -// Pickling: reduce the Qt Enum object -static PyObject *enum___reduce__(PyObject *obj) -{ - init_enum(); - return Py_BuildValue("O(Ni)", - enum_unpickler, - Py_BuildValue("s", Py_TYPE(obj)->tp_name), - PyLong_AS_LONG(obj)); -} - -} // extern "C" - -namespace Shiboken { namespace Enum { - -// Unpickling: rebuild the Qt Enum object -PyObject *unpickleEnum(PyObject *enum_class_name, PyObject *value) -{ - AutoDecRef parts(PyObject_CallMethod(enum_class_name, - "split", "s", ".")); - if (parts.isNull()) - return nullptr; - PyObject *top_name = PyList_GetItem(parts, 0); // borrowed ref - if (top_name == nullptr) - return nullptr; - PyObject *module = PyImport_GetModule(top_name); - if (module == nullptr) { - PyErr_Format(PyExc_ImportError, "could not import module %.200s", - String::toCString(top_name)); - return nullptr; - } - AutoDecRef cur_thing(module); - int len = PyList_Size(parts); - for (int idx = 1; idx < len; ++idx) { - PyObject *name = PyList_GetItem(parts, idx); // borrowed ref - PyObject *thing = PyObject_GetAttr(cur_thing, name); - if (thing == nullptr) { - PyErr_Format(PyExc_ImportError, "could not import Qt Enum type %.200s", - String::toCString(enum_class_name)); - return nullptr; - } - cur_thing.reset(thing); - } - PyObject *klass = cur_thing; - return PyObject_CallFunctionObjArgs(klass, value, nullptr); -} - -int enumOption{}; - -} // namespace Enum -} // namespace Shiboken - -extern "C" { - // Initialization static bool _init_enum() { AutoDecRef shibo(PyImport_ImportModule("shiboken6.Shiboken")); - auto mod = shibo.object(); - // publish Shiboken.Enum so that the signature gets initialized - if (PyObject_SetAttrString(mod, "Enum", reinterpret_cast<PyObject *>(SbkEnum_TypeF())) < 0) - return false; - if (InitSignatureStrings(SbkEnum_TypeF(), SbkEnum_SignatureStrings) < 0) - return false; - enum_unpickler = PyObject_GetAttrString(mod, "_unpickle_enum"); - if (enum_unpickler == nullptr) - return false; - return true; + return !shibo.isNull(); } -static int useOldEnum = -1; - -static PyMethodDef SbkEnumObject_Methods[] = { - {"__reduce__", reinterpret_cast<PyCFunction>(enum___reduce__), - METH_NOARGS, nullptr}, - {nullptr, nullptr, 0, nullptr} // Sentinel -}; - static PyObject *PyEnumModule{}; static PyObject *PyEnumMeta{}; static PyObject *PyEnum{}; @@ -408,8 +42,7 @@ static PyObject *PyFlag_KEEP{}; bool PyEnumMeta_Check(PyObject *ob) { - return Py_TYPE(ob) == (useOldEnum ? SbkEnumType_TypeF() - : reinterpret_cast<PyTypeObject *>(PyEnumMeta)); + return Py_TYPE(ob) == reinterpret_cast<PyTypeObject *>(PyEnumMeta); } PyTypeObject *getPyEnumMeta() @@ -445,20 +78,17 @@ void init_enum() static bool isInitialized = false; if (isInitialized) return; - if (!(isInitialized || enum_unpickler || _init_enum())) - Py_FatalError("could not load enum pickling helper function"); - Py_AtExit(cleanupEnumTypes); + if (!(isInitialized || _init_enum())) + Py_FatalError("could not init enum"); // PYSIDE-1735: Determine whether we should use the old or the new enum implementation. - static PyObject *sysmodule = PyImport_AddModule("sys"); - static PyObject *option = PyObject_GetAttrString(sysmodule, "pyside63_option_python_enum"); + static PyObject *option = PySys_GetObject("pyside6_option_python_enum"); if (!option || !PyLong_Check(option)) { PyErr_Clear(); - option = PyLong_FromLong(0); + option = PyLong_FromLong(1); } int ignoreOver{}; Enum::enumOption = PyLong_AsLongAndOverflow(option, &ignoreOver); - useOldEnum = Enum::enumOption == Enum::ENOPT_OLD_ENUM; getPyEnumMeta(); isInitialized = true; } @@ -472,8 +102,8 @@ int enumIsFlag(PyObject *ob_type) if (metatype != reinterpret_cast<PyTypeObject *>(PyEnumMeta)) return -1; auto *mro = reinterpret_cast<PyTypeObject *>(ob_type)->tp_mro; - Py_ssize_t idx, n = PyTuple_GET_SIZE(mro); - for (idx = 0; idx < n; idx++) { + const Py_ssize_t n = PyTuple_GET_SIZE(mro); + for (Py_ssize_t idx = 0; idx < n; ++idx) { auto *sub_type = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(mro, idx)); if (sub_type == reinterpret_cast<PyTypeObject *>(PyFlag)) return 1; @@ -481,519 +111,6 @@ int enumIsFlag(PyObject *ob_type) return 0; } -// PYSIDE-1735: Helper function to ask what enum we are using -bool usingNewEnum() -{ - init_enum(); - return !useOldEnum; -} - -} // extern "C" - -// -/////////////////////////////////////////////////////////////// - -namespace Shiboken { - -class DeclaredEnumTypes -{ -public: - struct EnumEntry - { - char *name; // full name as allocated. type->tp_name might be a substring. - PyTypeObject *type; - }; - - DeclaredEnumTypes(const DeclaredEnumTypes &) = delete; - DeclaredEnumTypes(DeclaredEnumTypes &&) = delete; - DeclaredEnumTypes &operator=(const DeclaredEnumTypes &) = delete; - DeclaredEnumTypes &operator=(DeclaredEnumTypes &&) = delete; - - DeclaredEnumTypes(); - ~DeclaredEnumTypes(); - static DeclaredEnumTypes &instance(); - void addEnumType(const EnumEntry &e) { m_enumTypes.push_back(e); } - - void cleanup(); - -private: - std::vector<EnumEntry> m_enumTypes; -}; - -namespace Enum { - -// forward -static PyObject *newItemOld(PyTypeObject *enumType, EnumValueType itemValue, - const char *itemName); - -// forward -static PyTypeObject * newTypeWithNameOld(const char *name, - const char *cppName, - PyTypeObject *numbers_fromFlag); - -bool check(PyObject *pyObj) -{ - init_enum(); - - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return Py_TYPE(Py_TYPE(pyObj)) == SbkEnumType_TypeF(); - - static PyTypeObject *meta = getPyEnumMeta(); - return Py_TYPE(Py_TYPE(pyObj)) == reinterpret_cast<PyTypeObject *>(meta); -} - -static PyObject *getEnumItemFromValueOld(PyTypeObject *enumType, - EnumValueType itemValue) -{ - PyObject *key, *value; - Py_ssize_t pos = 0; - PyObject *values = PyDict_GetItem(enumType->tp_dict, PyName::values()); - if (values == nullptr) - return nullptr; - - while (PyDict_Next(values, &pos, &key, &value)) { - auto *obj = reinterpret_cast<SbkEnumObject *>(value); - if (obj->ob_value == itemValue) { - Py_INCREF(value); - return value; - } - } - return nullptr; -} - -PyObject *getEnumItemFromValue(PyTypeObject *enumType, EnumValueType itemValue) -{ - init_enum(); - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return getEnumItemFromValueOld(enumType, itemValue); - - auto *obEnumType = reinterpret_cast<PyObject *>(enumType); - AutoDecRef val2members(PyObject_GetAttrString(obEnumType, "_value2member_map_")); - if (val2members.isNull()) { - PyErr_Clear(); - return nullptr; - } - AutoDecRef ob_value(PyLong_FromLongLong(itemValue)); - auto *result = PyDict_GetItem(val2members, ob_value); - Py_XINCREF(result); - return result; -} - -static PyTypeObject *createEnum(const char *fullName, const char *cppName, - PyTypeObject *flagsType) -{ - init_enum(); - PyTypeObject *enumType = newTypeWithNameOld(fullName, cppName, flagsType); - if (PyType_Ready(enumType) < 0) { - Py_XDECREF(enumType); - return nullptr; - } - return enumType; -} - -PyTypeObject *createGlobalEnum(PyObject *module, const char *name, const char *fullName, - const char *cppName, PyTypeObject *flagsType) -{ - PyTypeObject *enumType = createEnum(fullName, cppName, flagsType); - if (enumType && PyModule_AddObject(module, name, reinterpret_cast<PyObject *>(enumType)) < 0) { - Py_DECREF(enumType); - return nullptr; - } - flagsType = recordCurrentEnum(module, name, enumType, flagsType); - if (flagsType && PyModule_AddObject(module, PepType_GetNameStr(flagsType), - reinterpret_cast<PyObject *>(flagsType)) < 0) { - Py_DECREF(enumType); - return nullptr; - } - return enumType; -} - -PyTypeObject *createScopedEnum(PyTypeObject *scope, const char *name, const char *fullName, - const char *cppName, PyTypeObject *flagsType) -{ - PyTypeObject *enumType = createEnum(fullName, cppName, flagsType); - if (enumType && PyDict_SetItemString(scope->tp_dict, name, - reinterpret_cast<PyObject *>(enumType)) < 0) { - Py_DECREF(enumType); - return nullptr; - } - auto *obScope = reinterpret_cast<PyObject *>(scope); - flagsType = recordCurrentEnum(obScope, name, enumType, flagsType); - if (flagsType && PyDict_SetItemString(scope->tp_dict, - PepType_GetNameStr(flagsType), - reinterpret_cast<PyObject *>(flagsType)) < 0) { - Py_DECREF(enumType); - return nullptr; - } - return enumType; -} - -static PyObject *createEnumItem(PyTypeObject *enumType, const char *itemName, - EnumValueType itemValue) -{ - init_enum(); - PyObject *enumItem = newItemOld(enumType, itemValue, itemName); - if (PyDict_SetItemString(enumType->tp_dict, itemName, enumItem) < 0) { - Py_DECREF(enumItem); - return nullptr; - } - return enumItem; -} - -bool createGlobalEnumItem(PyTypeObject *enumType, PyObject *module, - const char *itemName, EnumValueType itemValue) -{ - PyObject *enumItem = createEnumItem(enumType, itemName, itemValue); - if (!enumItem) - return false; - int ok = useOldEnum ? PyModule_AddObject(module, itemName, enumItem) : true; - Py_DECREF(enumItem); - return ok >= 0; -} - -bool createScopedEnumItem(PyTypeObject *enumType, PyTypeObject *scope, - const char *itemName, EnumValueType itemValue) -{ - PyObject *enumItem = createEnumItem(enumType, itemName, itemValue); - if (!enumItem) - return false; - int ok = useOldEnum ? PyDict_SetItemString(scope->tp_dict, itemName, enumItem) : true; - Py_DECREF(enumItem); - return ok >= 0; -} - -// This exists temporary as the old way to create an enum item. -// For the public interface, we use a new function -static PyObject *newItemOld(PyTypeObject *enumType, - EnumValueType itemValue, const char *itemName) -{ - bool newValue = true; - SbkEnumObject *enumObj; - if (!itemName) { - enumObj = reinterpret_cast<SbkEnumObject *>( - getEnumItemFromValue(enumType, itemValue)); - if (enumObj) - return reinterpret_cast<PyObject *>(enumObj); - - newValue = false; - } - - enumObj = PyObject_New(SbkEnumObject, enumType); - if (!enumObj) - return nullptr; - - enumObj->ob_name = itemName ? PyBytes_FromString(itemName) : nullptr; - enumObj->ob_value = itemValue; - - if (newValue) { - auto dict = enumType->tp_dict; // Note: 'values' is borrowed - PyObject *values = PyDict_GetItemWithError(dict, PyName::values()); - if (values == nullptr) { - if (PyErr_Occurred()) - return nullptr; - AutoDecRef new_values(values = PyDict_New()); - if (values == nullptr) - return nullptr; - if (PyDict_SetItem(dict, PyName::values(), values) < 0) - return nullptr; - } - PyDict_SetItemString(values, itemName, reinterpret_cast<PyObject *>(enumObj)); - } - - return reinterpret_cast<PyObject *>(enumObj); -} - -PyObject *newItem(PyTypeObject *enumType, EnumValueType itemValue, - const char *itemName) -{ - init_enum(); - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return newItemOld(enumType, itemValue, itemName); - - auto *obEnumType = reinterpret_cast<PyObject *>(enumType); - if (!itemName) - return PyObject_CallFunction(obEnumType, "L", itemValue); - - static PyObject *const _member_map_ = String::createStaticString("_member_map_"); - auto *member_map = PyDict_GetItem(enumType->tp_dict, _member_map_); - if (!(member_map && PyDict_Check(member_map))) - return nullptr; - auto *result = PyDict_GetItemString(member_map, itemName); - Py_XINCREF(result); - return result; -} - -} // namespace Shiboken -} // namespace Enum - -static PyType_Slot SbkNewEnum_slots[] = { - {Py_tp_repr, reinterpret_cast<void *>(SbkEnumObject_repr)}, - {Py_tp_str, reinterpret_cast<void *>(SbkEnumObject_repr)}, - {Py_tp_getset, reinterpret_cast<void *>(SbkEnumGetSetList)}, - {Py_tp_methods, reinterpret_cast<void *>(SbkEnumObject_Methods)}, - {Py_tp_new, reinterpret_cast<void *>(SbkEnum_tp_new)}, - {Py_nb_add, reinterpret_cast<void *>(enum_add)}, - {Py_nb_subtract, reinterpret_cast<void *>(enum_subtract)}, - {Py_nb_multiply, reinterpret_cast<void *>(enum_multiply)}, - {Py_nb_positive, reinterpret_cast<void *>(enum_int)}, - {Py_nb_bool, reinterpret_cast<void *>(enum_bool)}, - {Py_nb_and, reinterpret_cast<void *>(enum_and)}, - {Py_nb_xor, reinterpret_cast<void *>(enum_xor)}, - {Py_nb_or, reinterpret_cast<void *>(enum_or)}, - {Py_nb_int, reinterpret_cast<void *>(enum_int)}, - {Py_nb_index, reinterpret_cast<void *>(enum_int)}, - {Py_tp_richcompare, reinterpret_cast<void *>(enum_richcompare)}, - {Py_tp_hash, reinterpret_cast<void *>(enum_hash)}, - {Py_tp_dealloc, reinterpret_cast<void *>(enum_object_dealloc)}, - {0, nullptr} -}; -static PyType_Spec SbkNewEnum_spec = { - "1:Shiboken.Enum", - sizeof(SbkEnumObject), - 0, - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, - SbkNewEnum_slots, -}; - -static PyTypeObject *SbkEnum_TypeF() -{ - static auto type = SbkType_FromSpecWithMeta(&SbkNewEnum_spec, SbkEnumType_TypeF()); - return type; -} - -namespace Shiboken { namespace Enum { - -static void -copyNumberMethods(PyTypeObject *flagsType, - PyType_Slot number_slots[], - int *pidx) -{ - int idx = *pidx; -#define PUT_SLOT(name) \ - number_slots[idx].slot = (name); \ - number_slots[idx].pfunc = PyType_GetSlot(flagsType, (name)); \ - ++idx; - - PUT_SLOT(Py_nb_absolute); - PUT_SLOT(Py_nb_add); - PUT_SLOT(Py_nb_and); - PUT_SLOT(Py_nb_bool); - PUT_SLOT(Py_nb_divmod); - PUT_SLOT(Py_nb_float); - PUT_SLOT(Py_nb_floor_divide); - PUT_SLOT(Py_nb_index); - PUT_SLOT(Py_nb_inplace_add); - PUT_SLOT(Py_nb_inplace_and); - PUT_SLOT(Py_nb_inplace_floor_divide); - PUT_SLOT(Py_nb_inplace_lshift); - PUT_SLOT(Py_nb_inplace_multiply); - PUT_SLOT(Py_nb_inplace_or); - PUT_SLOT(Py_nb_inplace_power); - PUT_SLOT(Py_nb_inplace_remainder); - PUT_SLOT(Py_nb_inplace_rshift); - PUT_SLOT(Py_nb_inplace_subtract); - PUT_SLOT(Py_nb_inplace_true_divide); - PUT_SLOT(Py_nb_inplace_xor); - PUT_SLOT(Py_nb_int); - PUT_SLOT(Py_nb_invert); - PUT_SLOT(Py_nb_lshift); - PUT_SLOT(Py_nb_multiply); - PUT_SLOT(Py_nb_negative); - PUT_SLOT(Py_nb_or); - PUT_SLOT(Py_nb_positive); - PUT_SLOT(Py_nb_power); - PUT_SLOT(Py_nb_remainder); - PUT_SLOT(Py_nb_rshift); - PUT_SLOT(Py_nb_subtract); - PUT_SLOT(Py_nb_true_divide); - PUT_SLOT(Py_nb_xor); -#undef PUT_SLOT - *pidx = idx; -} - -static PyTypeObject * newTypeWithNameOld(const char *name, - const char *cppName, - PyTypeObject *numbers_fromFlag) -{ - // Careful: SbkType_FromSpec does not allocate the string. - PyType_Slot newslots[99] = {}; // enough but not too big for the stack - PyType_Spec newspec; - DeclaredEnumTypes::EnumEntry entry{strdup(name), nullptr}; - newspec.name = entry.name; // Note that SbkType_FromSpec might use a substring. - newspec.basicsize = SbkNewEnum_spec.basicsize; - newspec.itemsize = SbkNewEnum_spec.itemsize; - newspec.flags = SbkNewEnum_spec.flags; - // we must append all the number methods, so rebuild everything: - int idx = 0; - while (SbkNewEnum_slots[idx].slot) { - newslots[idx].slot = SbkNewEnum_slots[idx].slot; - newslots[idx].pfunc = SbkNewEnum_slots[idx].pfunc; - ++idx; - } - if (numbers_fromFlag) - copyNumberMethods(numbers_fromFlag, newslots, &idx); - newspec.slots = newslots; - AutoDecRef bases(PyTuple_New(1)); - static auto basetype = reinterpret_cast<PyObject *>(SbkEnum_TypeF()); - Py_INCREF(basetype); - PyTuple_SetItem(bases, 0, basetype); - auto *type = SbkType_FromSpecBasesMeta(&newspec, bases, SbkEnumType_TypeF()); - entry.type = type; - - auto *enumType = reinterpret_cast<SbkEnumType *>(type); - auto *setp = PepType_SETP(enumType); - setp->cppName = cppName; - DeclaredEnumTypes::instance().addEnumType(entry); - return entry.type; -} - -// PySIDE-1735: This function is in the API and should be removed in 6.4 . -// Python enums are created differently. -PyTypeObject *newTypeWithName(const char *name, - const char *cppName, - PyTypeObject *numbers_fromFlag) -{ - if (!useOldEnum) - PyErr_Format(PyExc_RuntimeError, "function `%s` can no longer be used when the Python " - "Enum's have been selected", __FUNCTION__); - return newTypeWithNameOld(name, cppName, numbers_fromFlag); -} - -const char *getCppName(PyTypeObject *enumType) -{ - assert(Py_TYPE(enumType) == SbkEnumType_TypeF()); - auto *type = reinterpret_cast<SbkEnumType *>(enumType); - auto *setp = PepType_SETP(type); - return setp->cppName; -} - -EnumValueType getValue(PyObject *enumItem) -{ - init_enum(); - - assert(Enum::check(enumItem)); - - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return reinterpret_cast<SbkEnumObject *>(enumItem)->ob_value; - - std::cerr << __FUNCTION__ << ' ' << Shiboken::debugPyObject(enumItem) - << " err=" << Shiboken::debugPyObject(PyErr_Occurred()) << '\n'; - - AutoDecRef pyValue(PyObject_GetAttrString(enumItem, "value")); - return PyLong_AsLongLong(pyValue); -} - -void setTypeConverter(PyTypeObject *type, SbkConverter *converter, bool isFlag) -{ - if (isFlag) { - auto *flagsType = reinterpret_cast<PySideQFlagsType *>(type); - PepType_PFTP(flagsType)->converter = converter; - } - else { - auto *enumType = reinterpret_cast<SbkEnumType *>(type); - PepType_SETP(enumType)->converter = converter; - } -} - -} // namespace Enum - -DeclaredEnumTypes &DeclaredEnumTypes::instance() -{ - static DeclaredEnumTypes me; - return me; -} - -DeclaredEnumTypes::DeclaredEnumTypes() = default; - -DeclaredEnumTypes::~DeclaredEnumTypes() -{ - cleanup(); -} - -void DeclaredEnumTypes::cleanup() -{ - static bool was_called = false; - if (was_called) - return; - - for (const auto &e : m_enumTypes) { - std::free(e.name); - } - m_enumTypes.clear(); - was_called = true; -} - -} // namespace Shiboken - -static void cleanupEnumTypes() -{ - DeclaredEnumTypes::instance().cleanup(); -} - -/////////////////////////////////////////////////////////////////////// -// -// PYSIDE-1735: Re-implementation of Enums using Python -// ==================================================== -// -// This is a very simple, first implementation of a replacement -// for the Qt-like Enums using the Python Enum module. -// -// The basic idea: -// --------------- -// * We create the Enums as always -// * After creation of each enum, a special function is called that -// * grabs the last generated enum -// * reads all Enum items -// * generates a class statement for the Python Enum -// * creates a new Python Enum class -// * replaces the already inserted Enum with the new one. -// -// There are lots of ways to optimize that. Will be added later. -// -extern "C" { - -struct lastEnumCreated { - PyObject *scopeOrModule; - const char *name; - PyTypeObject *enumType; - PyTypeObject *flagsType; -}; - -static lastEnumCreated lec{}; - -static PyTypeObject *recordCurrentEnum(PyObject *scopeOrModule, - const char *name, - PyTypeObject *enumType, - PyTypeObject *flagsType) -{ - lec.scopeOrModule = scopeOrModule; - lec.name = name; - lec.enumType = enumType; - lec.flagsType = flagsType; - - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return flagsType; - - // We return nullptr as flagsType to disable flag creation. - return nullptr; -} - -static bool is_old_version() -{ - auto *sysmodule = PyImport_AddModule("sys"); - auto *dic = PyModule_GetDict(sysmodule); - auto *version = PyDict_GetItemString(dic, "version_info"); - auto *major = PyTuple_GetItem(version, 0); - auto *minor = PyTuple_GetItem(version, 1); - auto number = PyLong_AsLong(major) * 1000 + PyLong_AsLong(minor); - return number <= 3008; -} - /////////////////////////////////////////////////////////////////////// // // Support for Missing Values @@ -1015,6 +132,7 @@ static bool is_old_version() // We create each constant only once and keep the result in a dict // "_sbk_missing_". This is similar to a competitor's "_sip_missing_". // + static PyObject *missing_func(PyObject * /* self */ , PyObject *args) { // In order to relax matters to be more compatible with C++, we need @@ -1030,10 +148,11 @@ static PyObject *missing_func(PyObject * /* self */ , PyObject *args) if (!PyLong_Check(value)) Py_RETURN_NONE; auto *type = reinterpret_cast<PyTypeObject *>(klass); - auto *sbk_missing = PyDict_GetItem(type->tp_dict, _sbk_missing); + AutoDecRef tpDict(PepType_GetDict(type)); + auto *sbk_missing = PyDict_GetItem(tpDict.object(), _sbk_missing); if (!sbk_missing) { sbk_missing = PyDict_New(); - PyDict_SetItem(type->tp_dict, _sbk_missing, sbk_missing); + PyDict_SetItem(tpDict.object(), _sbk_missing, sbk_missing); } // See if the value is already in the dict. AutoDecRef val_str(PyObject_CallMethod(value, "__str__", nullptr)); @@ -1084,35 +203,99 @@ static PyObject *create_missing_func(PyObject *klass) static auto *const obType = reinterpret_cast<PyObject *>(type); static auto *const _missing = Shiboken::String::createStaticString("_missing_"); static auto *const func = PyObject_GetAttr(obType, _missing); - static auto *const functools = PyImport_ImportModule("_functools"); // builtin - static auto *const _partial = Shiboken::String::createStaticString("partial"); - static auto *const partial = PyObject_GetAttr(functools, _partial); + static auto *const partial = Pep_GetPartialFunction(); return PyObject_CallFunctionObjArgs(partial, func, klass, nullptr); } // //////////////////////////////////////////////////////////////////////// -PyTypeObject *morphLastEnumToPython() +} // extern "C" + +namespace Shiboken::Enum { + +int enumOption{}; + +bool check(PyObject *pyObj) +{ + init_enum(); + + static PyTypeObject *meta = getPyEnumMeta(); + return Py_TYPE(Py_TYPE(pyObj)) == reinterpret_cast<PyTypeObject *>(meta); +} + +PyObject *getEnumItemFromValue(PyTypeObject *enumType, EnumValueType itemValue) +{ + init_enum(); + + auto *obEnumType = reinterpret_cast<PyObject *>(enumType); + AutoDecRef val2members(PyObject_GetAttrString(obEnumType, "_value2member_map_")); + if (val2members.isNull()) { + PyErr_Clear(); + return nullptr; + } + AutoDecRef ob_value(PyLong_FromLongLong(itemValue)); + auto *result = PyDict_GetItem(val2members, ob_value); + Py_XINCREF(result); + return result; +} + +PyObject *newItem(PyTypeObject *enumType, EnumValueType itemValue, + const char *itemName) +{ + init_enum(); + + auto *obEnumType = reinterpret_cast<PyObject *>(enumType); + if (!itemName) + return PyObject_CallFunction(obEnumType, "L", itemValue); + + static PyObject *const _member_map_ = String::createStaticString("_member_map_"); + AutoDecRef tpDict(PepType_GetDict(enumType)); + auto *member_map = PyDict_GetItem(tpDict.object(), _member_map_); + if (!(member_map && PyDict_Check(member_map))) + return nullptr; + auto *result = PyDict_GetItemString(member_map, itemName); + Py_XINCREF(result); + return result; +} + +EnumValueType getValue(PyObject *enumItem) { - /// The Python Enum internal structure is way too complicated. - /// It is much easier to generate Python code and execute it. + init_enum(); + + assert(Enum::check(enumItem)); + + AutoDecRef pyValue(PyObject_GetAttrString(enumItem, "value")); + return PyLong_AsLongLong(pyValue); +} - // Pick up the last generated Enum and convert it into a PyEnum - auto *enumType = lec.enumType; - // This is temporary; SbkEnumType will be removed, soon. +void setTypeConverter(PyTypeObject *type, SbkConverter *converter) +{ + auto *enumType = reinterpret_cast<SbkEnumType *>(type); + PepType_SETP(enumType)->converter = converter; +} - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return enumType; +static PyTypeObject *createEnumForPython(PyObject *scopeOrModule, + const char *fullName, + PyObject *pyEnumItems) +{ + const char *colon = strchr(fullName, ':'); + assert(colon); + int package_level = atoi(fullName); + const char *mod = colon + 1; - auto *setp = PepType_SETP(reinterpret_cast<SbkEnumType *>(enumType)); - if (setp->replacementType) { - // For some (yet to fix) reason, initialization of the enums can happen twice. - // If that happens, use the existing new type to keep type checks correct. - return setp->replacementType; + const char *qual = mod; + for (int idx = package_level; idx > 0; --idx) { + const char *dot = strchr(qual, '.'); + if (!dot) + break; + qual = dot + 1; } + int mlen = qual - mod - 1; + AutoDecRef module(Shiboken::String::fromCString(mod, mlen)); + AutoDecRef qualname(Shiboken::String::fromCString(qual)); + const char *dot = strrchr(qual, '.'); + AutoDecRef name(Shiboken::String::fromCString(dot ? dot + 1 : qual)); - auto *scopeOrModule = lec.scopeOrModule; static PyObject *enumName = String::createStaticString("IntEnum"); if (PyType_Check(scopeOrModule)) { // For global objects, we have no good solution, yet where to put the int info. @@ -1120,15 +303,9 @@ PyTypeObject *morphLastEnumToPython() auto *sotp = PepType_SOTP(type); if (!sotp->enumFlagsDict) initEnumFlagsDict(type); - enumName = PyDict_GetItem(sotp->enumTypeDict, String::fromCString(lec.name)); + enumName = PyDict_GetItem(sotp->enumTypeDict, name); } - PyObject *key, *value; - Py_ssize_t pos = 0; - PyObject *values = PyDict_GetItem(enumType->tp_dict, PyName::values()); - if (!values) - return nullptr; - AutoDecRef PyEnumType(PyObject_GetAttr(PyEnumModule, enumName)); assert(PyEnumType.object()); bool isFlag = PyObject_IsSubclass(PyEnumType, PyFlag); @@ -1141,24 +318,13 @@ PyTypeObject *morphLastEnumToPython() PyEnumType.reset(surrogate); } - // Walk the values dict and create a Python enum type. - AutoDecRef name(PyUnicode_FromString(lec.name)); - AutoDecRef args(PyList_New(0)); + // Walk the enumItemStrings and create a Python enum type. auto *pyName = name.object(); - auto *pyArgs = args.object(); - while (PyDict_Next(values, &pos, &key, &value)) { - auto *key_value = PyTuple_New(2); - PyTuple_SET_ITEM(key_value, 0, key); - Py_INCREF(key); - auto *obj = reinterpret_cast<SbkEnumObject *>(value); - auto *num = PyLong_FromLongLong(obj->ob_value); - PyTuple_SET_ITEM(key_value, 1, num); - PyList_Append(pyArgs, key_value); - } + // We now create the new type. Since Python 3.11, we need to pass in // `boundary=KEEP` because the default STRICT crashes on us. // See QDir.Filter.Drives | QDir.Filter.Files - AutoDecRef callArgs(Py_BuildValue("(OO)", pyName, pyArgs)); + AutoDecRef callArgs(Py_BuildValue("(OO)", pyName, pyEnumItems)); AutoDecRef callDict(PyDict_New()); static PyObject *boundary = String::createStaticString("boundary"); if (PyFlag_KEEP) @@ -1177,40 +343,110 @@ PyTypeObject *morphLastEnumToPython() } auto *newType = reinterpret_cast<PyTypeObject *>(obNewType); - auto *obEnumType = reinterpret_cast<PyObject *>(enumType); - AutoDecRef qual_name(PyObject_GetAttr(obEnumType, PyMagicName::qualname())); - PyObject_SetAttr(obNewType, PyMagicName::qualname(), qual_name); - AutoDecRef module(PyObject_GetAttr(obEnumType, PyMagicName::module())); + PyObject_SetAttr(obNewType, PyMagicName::qualname(), qualname); PyObject_SetAttr(obNewType, PyMagicName::module(), module); // See if we should re-introduce shortcuts in the enclosing object. const bool useGlobalShortcut = (Enum::enumOption & Enum::ENOPT_GLOBAL_SHORTCUT) != 0; const bool useScopedShortcut = (Enum::enumOption & Enum::ENOPT_SCOPED_SHORTCUT) != 0; if (useGlobalShortcut || useScopedShortcut) { + // We have to use the iterator protokol because the values dict is a mappingproxy. + AutoDecRef values(PyObject_GetAttr(obNewType, PyMagicName::members())); + AutoDecRef mapIterator(PyObject_GetIter(values)); + AutoDecRef mapKey{}; bool isModule = PyModule_Check(scopeOrModule); - pos = 0; - while (PyDict_Next(values, &pos, &key, &value)) { - AutoDecRef entry(PyObject_GetAttr(obNewType, key)); - if ((useGlobalShortcut && isModule) || (useScopedShortcut && !isModule)) - if (PyObject_SetAttr(scopeOrModule, key, entry) < 0) + while ((mapKey.reset(PyIter_Next(mapIterator))), mapKey.object()) { + if ((useGlobalShortcut && isModule) || (useScopedShortcut && !isModule)) { + AutoDecRef value(PyObject_GetItem(values, mapKey)); + if (PyObject_SetAttr(scopeOrModule, mapKey, value) < 0) return nullptr; + } } } - // Protect against double initialization - setp->replacementType = newType; - - // PYSIDE-1735: Old Python versions can't stand the early enum deallocation. - static bool old_python_version = is_old_version(); - if (old_python_version) - Py_INCREF(obEnumType); return newType; } -PyTypeObject *mapFlagsToSameEnum(PyTypeObject *FType, PyTypeObject *EType) +template <typename IntT> +static PyObject *toPyObject(IntT v) { - // this will be switchable... - return useOldEnum ? FType : EType; + if constexpr (sizeof(IntT) == 8) { + if constexpr (std::is_unsigned_v<IntT>) + return PyLong_FromUnsignedLongLong(v); + return PyLong_FromLongLong(v); + } + if constexpr (std::is_unsigned_v<IntT>) + return PyLong_FromUnsignedLong(v); + return PyLong_FromLong(v); } -} // extern "C" +template <typename IntT> +static PyTypeObject *createPythonEnumHelper(PyObject *module, + const char *fullName, const char *enumItemStrings[], const IntT enumValues[]) +{ + AutoDecRef args(PyList_New(0)); + auto *pyEnumItems = args.object(); + for (size_t idx = 0; enumItemStrings[idx] != nullptr; ++idx) { + const char *kv = enumItemStrings[idx]; + auto *key = PyUnicode_FromString(kv); + auto *value = toPyObject(enumValues[idx]); + auto *key_value = PyTuple_New(2); + PyTuple_SET_ITEM(key_value, 0, key); + PyTuple_SET_ITEM(key_value, 1, value); + PyList_Append(pyEnumItems, key_value); + } + return createEnumForPython(module, fullName, pyEnumItems); +} + +// Now we have to concretize these functions explicitly, +// otherwise templates will not work across modules. + +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int64_t enumValues[]) +{ + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); +} + +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint64_t enumValues[]) +{ + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); +} + +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int32_t enumValues[]) +{ + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); +} + +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint32_t enumValues[]) +{ + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); +} + +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int16_t enumValues[]) +{ + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); +} + +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint16_t enumValues[]) +{ + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); +} + +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int8_t enumValues[]) +{ + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); +} + +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint8_t enumValues[]) +{ + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); +} + +} // namespace Shiboken::Enum diff --git a/sources/shiboken6/libshiboken/sbkenum.h b/sources/shiboken6/libshiboken/sbkenum.h index a39519189..e19ca4b4c 100644 --- a/sources/shiboken6/libshiboken/sbkenum.h +++ b/sources/shiboken6/libshiboken/sbkenum.h @@ -15,81 +15,88 @@ LIBSHIBOKEN_API bool PyEnumMeta_Check(PyObject *ob); /// exposed for the signature module LIBSHIBOKEN_API void init_enum(); -extern LIBSHIBOKEN_API PyTypeObject *SbkEnumType_TypeF(void); struct SbkConverter; struct SbkEnumType; -struct SbkEnumTypePrivate; -} // extern "C" - -namespace Shiboken +struct SbkEnumTypePrivate { + SbkConverter *converter; +}; + +/// PYSIDE-1735: Pass on the Python enum/flag information. +LIBSHIBOKEN_API void initEnumFlagsDict(PyTypeObject *type); + +/// PYSIDE-1735: Make sure that we can import the Python enum implementation. +LIBSHIBOKEN_API PyTypeObject *getPyEnumMeta(); +/// PYSIDE-1735: Helper function supporting QEnum +LIBSHIBOKEN_API int enumIsFlag(PyObject *ob_enum); -inline bool isShibokenEnum(PyObject *pyObj) -{ - return Py_TYPE(Py_TYPE(pyObj)) == SbkEnumType_TypeF(); } -namespace Enum +namespace Shiboken::Enum { + +enum : int { + ENOPT_OLD_ENUM = 0x00, // PySide 6.6: no longer supported + ENOPT_NEW_ENUM = 0x01, + ENOPT_INHERIT_INT = 0x02, + ENOPT_GLOBAL_SHORTCUT = 0x04, + ENOPT_SCOPED_SHORTCUT = 0x08, + ENOPT_NO_FAKESHORTCUT = 0x10, + ENOPT_NO_FAKERENAMES = 0x20, + ENOPT_NO_ZERODEFAULT = 0x40, + ENOPT_NO_MISSING = 0x80, +}; + +LIBSHIBOKEN_API extern int enumOption; + +using EnumValueType = long long; + +LIBSHIBOKEN_API bool check(PyObject *obj); + +LIBSHIBOKEN_API PyObject *newItem(PyTypeObject *enumType, EnumValueType itemValue, + const char *itemName = nullptr); + +LIBSHIBOKEN_API EnumValueType getValue(PyObject *enumItem); +LIBSHIBOKEN_API PyObject *getEnumItemFromValue(PyTypeObject *enumType, + EnumValueType itemValue); + +/// Sets the enum/flag's type converter. +LIBSHIBOKEN_API void setTypeConverter(PyTypeObject *type, SbkConverter *converter); + +/// Creating Python enums for different types. +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int64_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint64_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int32_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint32_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int16_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint16_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int8_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint8_t enumValues[]); + +/// This template removes duplication by inlining necessary type casts. +template <typename IntT> +inline PyTypeObject *createPythonEnum(PyTypeObject *scope, + const char *fullName, const char *enumItemStrings[], const IntT enumValues[]) { - using EnumValueType = long long; - - LIBSHIBOKEN_API bool check(PyObject *obj); - /** - * Creates a new enum type (and its flags type, if any is given) - * and registers it to Python and adds it to \p module. - * \param module Module to where the new enum type will be added. - * \param name Name of the enum. - * \param fullName Name of the enum that includes all scope information (e.g.: "module.Enum"). - * \param cppName Full qualified C++ name of the enum. - * \param flagsType Optional Python type for the flags associated with the enum. - * \return The new enum type or NULL if it fails. - */ - LIBSHIBOKEN_API PyTypeObject *createGlobalEnum(PyObject *module, - const char *name, - const char *fullName, - const char *cppName, - PyTypeObject *flagsType = nullptr); - /// This function does the same as createGlobalEnum, but adds the enum to a Shiboken type or namespace. - LIBSHIBOKEN_API PyTypeObject *createScopedEnum(PyTypeObject *scope, - const char *name, - const char *fullName, - const char *cppName, - PyTypeObject *flagsType = nullptr); - - /** - * Creates a new enum item for a given enum type and adds it to \p module. - * \param enumType Enum type to where the new enum item will be added. - * \param module Module to where the enum type of the new enum item belongs. - * \param itemName Name of the enum item. - * \param itemValue Numerical value of the enum item. - * \return true if everything goes fine, false if it fails. - */ - LIBSHIBOKEN_API bool createGlobalEnumItem(PyTypeObject *enumType, PyObject *module, - const char *itemName, - EnumValueType itemValue); - /// This function does the same as createGlobalEnumItem, but adds the enum to a Shiboken type or namespace. - LIBSHIBOKEN_API bool createScopedEnumItem(PyTypeObject *enumType, PyTypeObject *scope, - const char *itemName, EnumValueType itemValue); - - LIBSHIBOKEN_API PyObject *newItem(PyTypeObject *enumType, EnumValueType itemValue, - const char *itemName = nullptr); - - LIBSHIBOKEN_API PyTypeObject *newTypeWithName(const char *name, const char *cppName, - PyTypeObject *numbers_fromFlag=nullptr); - LIBSHIBOKEN_API const char *getCppName(PyTypeObject *type); - LIBSHIBOKEN_API PyObject *getCppNameNew(PyTypeObject *type); - - LIBSHIBOKEN_API EnumValueType getValue(PyObject *enumItem); - LIBSHIBOKEN_API PyObject *getEnumItemFromValue(PyTypeObject *enumType, - EnumValueType itemValue); - - /// Sets the enum/flag's type converter. - LIBSHIBOKEN_API void setTypeConverter(PyTypeObject *type, SbkConverter *converter, bool isFlag); - - LIBSHIBOKEN_API PyObject *unpickleEnum(PyObject *, PyObject *); + auto *obScope = reinterpret_cast<PyObject *>(scope); + return createPythonEnum(obScope, fullName, enumItemStrings, enumValues); } -} // namespace Shiboken +} // namespace Shiboken::Enum #endif // SKB_PYENUM_H diff --git a/sources/shiboken6/libshiboken/sbkenum_p.h b/sources/shiboken6/libshiboken/sbkenum_p.h deleted file mode 100644 index d8477b4b3..000000000 --- a/sources/shiboken6/libshiboken/sbkenum_p.h +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (C) 2021 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only - -#ifndef SBKENUM_P_H -#define SBKENUM_P_H - -#include "sbkpython.h" -#include "shibokenmacros.h" - -struct SbkEnumTypePrivate -{ - SbkConverter *converter; - const char *cppName; - PyTypeObject *replacementType; -}; - -extern "C" { - -/// PYSIDE-1735: Pass on the Python enum/flag information. -LIBSHIBOKEN_API void initEnumFlagsDict(PyTypeObject *type); - -/// PYSIDE-1735: Patching the Enum / Flags implementation. Remove in 6.4 -LIBSHIBOKEN_API PyTypeObject *morphLastEnumToPython(); -LIBSHIBOKEN_API PyTypeObject *mapFlagsToSameEnum(PyTypeObject *FType, PyTypeObject *EType); - -/// PYSIDE-1735: Make sure that we can import the Python enum implementation. -LIBSHIBOKEN_API PyTypeObject *getPyEnumMeta(); -/// PYSIDE-1735: Helper function supporting QEnum -LIBSHIBOKEN_API int enumIsFlag(PyObject *ob_enum); -/// PYSIDE-1735: Helper function to ask what enum we are using -LIBSHIBOKEN_API bool usingNewEnum(); - -} - -namespace Shiboken { namespace Enum { - -enum : int { - ENOPT_OLD_ENUM = 0x00, - ENOPT_NEW_ENUM = 0x01, - ENOPT_INHERIT_INT = 0x02, - ENOPT_GLOBAL_SHORTCUT = 0x04, - ENOPT_SCOPED_SHORTCUT = 0x08, - ENOPT_NO_FAKESHORTCUT = 0x10, - ENOPT_NO_FAKERENAMES = 0x20, - ENOPT_NO_ZERODEFAULT = 0x40, - ENOPT_NO_MISSING = 0x80, -}; - -LIBSHIBOKEN_API extern int enumOption; - -}} - -#endif // SBKENUM_P_H diff --git a/sources/shiboken6/libshiboken/sbkerrors.cpp b/sources/shiboken6/libshiboken/sbkerrors.cpp index c8f8a989f..84c080f8d 100644 --- a/sources/shiboken6/libshiboken/sbkerrors.cpp +++ b/sources/shiboken6/libshiboken/sbkerrors.cpp @@ -2,10 +2,42 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "sbkerrors.h" +#include "sbkstring.h" #include "helper.h" +#include "gilstate.h" + +#include <cstdio> +#include <string> + +using namespace std::literals::string_literals; namespace Shiboken { + +// PYSIDE-2335: Track down if we can reach a Python error handler. +// _pythonContextStack has always the current state of handler status +// in its lowest bit. +// Blocking calls like exec or run need to use `setBlocking`. +static thread_local std::size_t _pythonContextStack{}; + +PythonContextMarker::PythonContextMarker() +{ + // Shift history up and set lowest bit. + _pythonContextStack = (_pythonContextStack * 2) + 1; +} + +PythonContextMarker::~PythonContextMarker() +{ + // Shift history down. + _pythonContextStack /= 2; +} + +void PythonContextMarker::setBlocking() +{ + // Clear lowest bit. + _pythonContextStack = _pythonContextStack / 2 * 2; +} + namespace Errors { @@ -66,6 +98,74 @@ void setWrongContainerType() PyErr_SetString(PyExc_TypeError, "Wrong type passed to container conversion."); } +// Prepend something to an exception message provided it is a single string +// argument. +static bool prependToExceptionMessage(PyObject *exc, const char *context) +{ + Shiboken::AutoDecRef args(PepException_GetArgs(exc)); + if (args.isNull() || PyTuple_Check(args.object()) == 0 || PyTuple_Size(args) != 1) + return false; + auto *oldMessage = PyTuple_GetItem(args, 0); + if (oldMessage == nullptr || PyUnicode_CheckExact(oldMessage) == 0) + return false; + auto *newMessage = PyUnicode_FromFormat("%s%U", context, oldMessage); + PepException_SetArgs(exc, PyTuple_Pack(1, newMessage)); + return true; +} + +struct ErrorStore { + PyObject *type; + PyObject *exc; + PyObject *traceback; +}; + +static thread_local ErrorStore savedError{}; + +static bool hasPythonContext() +{ + return _pythonContextStack & 1; +} + +void storeErrorOrPrint() +{ + // This error happened in a function with no way to return an error state. + // Therefore, we handle the error when we are error checking, anyway. + // But we do that only when we know that an error handler can pick it up. + if (hasPythonContext()) + PyErr_Fetch(&savedError.type, &savedError.exc, &savedError.traceback); + else + PyErr_Print(); +} + +// Like storeErrorOrPrint() with additional context info that is prepended +// to the exception message or printed. +static void storeErrorOrPrintWithContext(const char *context) +{ + if (hasPythonContext()) { + PyErr_Fetch(&savedError.type, &savedError.exc, &savedError.traceback); + prependToExceptionMessage(savedError.exc, context); + } else { + std::fputs(context, stderr); + PyErr_Print(); + } +} + +void storePythonOverrideErrorOrPrint(const char *className, const char *funcName) +{ + const std::string context = "Error calling Python override of "s + + className + "::"s + funcName + "(): "s; + storeErrorOrPrintWithContext(context.c_str()); +} + +PyObject *occurred() +{ + if (savedError.type) { + PyErr_Restore(savedError.type, savedError.exc, savedError.traceback); + savedError.type = nullptr; + } + return PyErr_Occurred(); +} + } // namespace Errors namespace Warnings diff --git a/sources/shiboken6/libshiboken/sbkerrors.h b/sources/shiboken6/libshiboken/sbkerrors.h index d894d8e7a..18ce701e7 100644 --- a/sources/shiboken6/libshiboken/sbkerrors.h +++ b/sources/shiboken6/libshiboken/sbkerrors.h @@ -7,8 +7,31 @@ #include "sbkpython.h" #include "shibokenmacros.h" +/// Craving for C++20 and std::source_location::current() +#if defined(_MSC_VER) +# define SBK_FUNC_INFO __FUNCSIG__ +#elif defined(__GNUC__) +# define SBK_FUNC_INFO __PRETTY_FUNCTION__ +#else +# define SBK_FUNC_INFO __FUNCTION__ +#endif + namespace Shiboken { + +struct LIBSHIBOKEN_API PythonContextMarker +{ +public: + PythonContextMarker(const PythonContextMarker &) = delete; + PythonContextMarker(PythonContextMarker &&) = delete; + PythonContextMarker &operator=(const PythonContextMarker &) = delete; + PythonContextMarker &operator=(PythonContextMarker &&) = delete; + + explicit PythonContextMarker(); + ~PythonContextMarker(); + void setBlocking(); +}; + namespace Errors { @@ -23,6 +46,20 @@ LIBSHIBOKEN_API void setSequenceTypeError(const char *expectedType); LIBSHIBOKEN_API void setSetterTypeError(const char *name, const char *expectedType); LIBSHIBOKEN_API void setWrongContainerType(); +/// Report an error ASAP: Instead of printing, store for later re-raise. +/// This replaces `PyErr_Print`, which cannot report errors as exception. +/// To be used in contexts where raising errors is impossible. +LIBSHIBOKEN_API void storeErrorOrPrint(); + +/// Call storeErrorOrPrint() and print the context to report +/// errors when calling Python overrides of virtual functions. +LIBSHIBOKEN_API void storePythonOverrideErrorOrPrint(const char *className, const char *funcName); + +/// Handle an error as in PyErr_Occurred(), but also check for errors which +/// were captured by `storeErrorOrPrint`. +/// To be used in normal error checks. +LIBSHIBOKEN_API PyObject *occurred(); + } // namespace Errors namespace Warnings diff --git a/sources/shiboken6/libshiboken/sbkfeature_base.cpp b/sources/shiboken6/libshiboken/sbkfeature_base.cpp index 2931a8abd..f31b8f4f7 100644 --- a/sources/shiboken6/libshiboken/sbkfeature_base.cpp +++ b/sources/shiboken6/libshiboken/sbkfeature_base.cpp @@ -4,7 +4,8 @@ #include "basewrapper.h" #include "basewrapper_p.h" #include "autodecref.h" -#include "sbkenum_p.h" +#include "pep384ext.h" +#include "sbkenum.h" #include "sbkstring.h" #include "sbkstaticstrings.h" #include "sbkstaticstrings_p.h" @@ -12,6 +13,8 @@ #include "sbkfeature_base.h" #include "gilstate.h" +#include <cctype> + using namespace Shiboken; extern "C" @@ -23,7 +26,8 @@ extern "C" // int currentSelectId(PyTypeObject *type) { - PyObject *PyId = PyObject_GetAttr(type->tp_dict, PyName::select_id()); + AutoDecRef tpDict(PepType_GetDict(type)); + PyObject *PyId = PyObject_GetAttr(tpDict.object(), PyName::select_id()); if (PyId == nullptr) { PyErr_Clear(); return 0x00; @@ -34,11 +38,19 @@ int currentSelectId(PyTypeObject *type) } static SelectableFeatureHook SelectFeatureSet = nullptr; +static SelectableFeatureCallback featureCb = nullptr; + +void setSelectableFeatureCallback(SelectableFeatureCallback func) +{ + featureCb = func; +} SelectableFeatureHook initSelectableFeature(SelectableFeatureHook func) { auto ret = SelectFeatureSet; SelectFeatureSet = func; + if (featureCb) + featureCb(SelectFeatureSet != nullptr); return ret; } // @@ -53,47 +65,62 @@ void disassembleFrame(const char *marker) static PyObject *dismodule = PyImport_ImportModule("dis"); static PyObject *disco = PyObject_GetAttrString(dismodule, "disco"); static PyObject *const _f_lasti = Shiboken::String::createStaticString("f_lasti"); + static PyObject *const _f_lineno = Shiboken::String::createStaticString("f_lineno"); static PyObject *const _f_code = Shiboken::String::createStaticString("f_code"); + static PyObject *const _co_filename = Shiboken::String::createStaticString("co_filename"); + AutoDecRef ignore{}; auto *frame = reinterpret_cast<PyObject *>(PyEval_GetFrame()); - AutoDecRef f_lasti(PyObject_GetAttr(frame, _f_lasti)); - AutoDecRef f_code(PyObject_GetAttr(frame, _f_code)); - fprintf(stdout, "\n%s BEGIN\n", marker); - PyObject_CallFunctionObjArgs(disco, f_code.object(), f_lasti.object(), nullptr); - fprintf(stdout, "%s END\n\n", marker); - static PyObject *sysmodule = PyImport_ImportModule("sys"); - static PyObject *stdout_file = PyObject_GetAttrString(sysmodule, "stdout"); - PyObject_CallMethod(stdout_file, "flush", nullptr); + if (frame == nullptr) { + fprintf(stdout, "\n%s BEGIN no frame END\n\n", marker); + } else { + AutoDecRef f_lasti(PyObject_GetAttr(frame, _f_lasti)); + AutoDecRef f_lineno(PyObject_GetAttr(frame, _f_lineno)); + AutoDecRef f_code(PyObject_GetAttr(frame, _f_code)); + AutoDecRef co_filename(PyObject_GetAttr(f_code, _co_filename)); + long line = PyLong_AsLong(f_lineno); + const char *fname = String::toCString(co_filename); + fprintf(stdout, "\n%s BEGIN line=%ld %s\n", marker, line, fname); + ignore.reset(PyObject_CallFunctionObjArgs(disco, f_code.object(), f_lasti.object(), nullptr)); + fprintf(stdout, "%s END line=%ld %s\n\n", marker, line, fname); + } +#if PY_VERSION_HEX >= 0x030C0000 && !Py_LIMITED_API + if (error_type) + PyErr_DisplayException(error_value); +#endif + static PyObject *stdout_file = PySys_GetObject("stdout"); + ignore.reset(PyObject_CallMethod(stdout_file, "flush", nullptr)); PyErr_Restore(error_type, error_value, error_traceback); } +// python 3.12 +static int const CALL = 171; // Python 3.11 static int const PRECALL = 166; // we have "big instructions" with gaps after them -static int const LOAD_ATTR_GAP = 4 * 2; -static int const LOAD_METHOD_GAP = 10 * 2; +static int const LOAD_METHOD_GAP_311 = 10 * 2; +static int const LOAD_ATTR_GAP_311 = 4 * 2; +static int const LOAD_ATTR_GAP = 9 * 2; // Python 3.7 - 3.10 static int const LOAD_METHOD = 160; static int const CALL_METHOD = 161; // Python 3.6 static int const CALL_FUNCTION = 131; static int const LOAD_ATTR = 106; - -static int _getVersion() -{ - static PyObject *const sysmodule = PyImport_AddModule("sys"); - static PyObject *const version = PyObject_GetAttrString(sysmodule, "version_info"); - static PyObject *const major = PyTuple_GetItem(version, 0); - static PyObject *const minor = PyTuple_GetItem(version, 1); - static auto number = PyLong_AsLong(major) * 1000 + PyLong_AsLong(minor); - return number; -} +// NoGil (how long will this exist in this form?) +static int const LOAD_METHOD_NOGIL = 55; +static int const CALL_METHOD_NOGIL = 72; static bool currentOpcode_Is_CallMethNoArgs() { + // PYSIDE-2221: Special case for the NoGil version: + // Find out if we have such a version. + // We could also ask the variable `Py_NOGIL`. + static PyObject *flags = PySys_GetObject("flags"); + static bool isNoGil = PyObject_HasAttrString(flags, "nogil"); // We look into the currently active operation if we are going to call // a method with zero arguments. auto *frame = PyEval_GetFrame(); -#if PY_VERSION_HEX >= 0x03090000 && !Py_LIMITED_API && !defined(PYPY_VERSION) +#if !Py_LIMITED_API && !defined(PYPY_VERSION) auto *f_code = PyFrame_GetCode(frame); #else static PyObject *const _f_code = Shiboken::String::createStaticString("f_code"); @@ -114,20 +141,37 @@ static bool currentOpcode_Is_CallMethNoArgs() char *co_code{}; PyBytes_AsStringAndSize(dec_co_code, &co_code, &code_len); uint8_t opcode1 = co_code[f_lasti]; + if (isNoGil) { + uint8_t opcode2 = co_code[f_lasti + 4]; + uint8_t oparg2 = co_code[f_lasti + 6]; + return opcode1 == LOAD_METHOD_NOGIL && opcode2 == CALL_METHOD_NOGIL && oparg2 == 1; + } uint8_t opcode2 = co_code[f_lasti + 2]; uint8_t oparg2 = co_code[f_lasti + 3]; - static auto number = _getVersion(); - if (number < 3007) - return opcode1 == LOAD_ATTR && opcode2 == CALL_FUNCTION && oparg2 == 0; - if (number < 3011) + static auto number = _PepRuntimeVersion(); + if (number < 0x030B00) return opcode1 == LOAD_METHOD && opcode2 == CALL_METHOD && oparg2 == 0; - // With Python 3.11, the opcodes get bigger and change a bit. + if (number < 0x030C00) { + // With Python 3.11, the opcodes get bigger and change a bit. + // Note: The new adaptive opcodes are elegantly hidden and we + // don't need to take care of them. + if (opcode1 == LOAD_METHOD) + f_lasti += LOAD_METHOD_GAP_311; + else if (opcode1 == LOAD_ATTR) + f_lasti += LOAD_ATTR_GAP_311; + else + return false; + + opcode2 = co_code[f_lasti + 2]; + oparg2 = co_code[f_lasti + 3]; + + return opcode2 == PRECALL && oparg2 == 0; + } + // With Python 3.12, the opcodes get again bigger and change a bit. // Note: The new adaptive opcodes are elegantly hidden and we // don't need to take care of them. - if (opcode1 == LOAD_METHOD) - f_lasti += LOAD_METHOD_GAP; - else if (opcode1 == LOAD_ATTR) + if (opcode1 == LOAD_ATTR) f_lasti += LOAD_ATTR_GAP; else return false; @@ -135,7 +179,7 @@ static bool currentOpcode_Is_CallMethNoArgs() opcode2 = co_code[f_lasti + 2]; oparg2 = co_code[f_lasti + 3]; - return opcode2 == PRECALL && oparg2 == 0; + return opcode2 == CALL && oparg2 == 0; } void initEnumFlagsDict(PyTypeObject *type) @@ -166,12 +210,107 @@ void initEnumFlagsDict(PyTypeObject *type) static PyObject *replaceNoArgWithZero(PyObject *callable) { - static auto *functools = PyImport_ImportModule("_functools"); // builtin - static auto *partial = PyObject_GetAttrString(functools, "partial"); + static auto *partial = Pep_GetPartialFunction(); static auto *zero = PyLong_FromLong(0); return PyObject_CallFunctionObjArgs(partial, callable, zero, nullptr); } +static PyObject *lookupUnqualifiedOrOldEnum(PyTypeObject *type, PyObject *name) +{ + // MRO has been observed to be 0 in case of errors with QML decorators + if (type == nullptr || type->tp_mro == nullptr) + return nullptr; + // Quick Check: Avoid "__..", "_slots", etc. + if (std::isalpha(Shiboken::String::toCString(name)[0]) == 0) + return nullptr; + static PyTypeObject *const EnumMeta = getPyEnumMeta(); + static PyObject *const _member_map_ = String::createStaticString("_member_map_"); + // This is similar to `find_name_in_mro`, but instead of looking directly into + // tp_dict, we also search for the attribute in local classes of that dict (Part 2). + PyObject *mro = type->tp_mro; + PyObject *result{}; + assert(PyTuple_Check(mro)); + Py_ssize_t idx, n = PyTuple_GET_SIZE(mro); + for (idx = 0; idx < n; ++idx) { + auto *base = PyTuple_GET_ITEM(mro, idx); + auto *type_base = reinterpret_cast<PyTypeObject *>(base); + if (!SbkObjectType_Check(type_base)) + continue; + auto sotp = PepType_SOTP(type_base); + // The EnumFlagInfo structure tells us if there are Enums at all. + const char **enumFlagInfo = sotp->enumFlagInfo; + if (!(enumFlagInfo)) + continue; + if (!sotp->enumFlagsDict) + initEnumFlagsDict(type_base); + bool useFakeRenames = !(Enum::enumOption & Enum::ENOPT_NO_FAKERENAMES); + if (useFakeRenames) { + auto *rename = PyDict_GetItem(sotp->enumFlagsDict, name); + if (rename) { + /* + * Part 1: Look into the enumFlagsDict if we have an old flags name. + * ------------------------------------------------------------- + * We need to replace the parameterless + + QtCore.Qt.Alignment() + + * by the one-parameter call + + QtCore.Qt.AlignmentFlag(0) + + * That means: We need to bind the zero as default into a wrapper and + * return that to be called. + * + * Addendum: + * --------- + * We first need to look into the current opcode of the bytecode to find + * out if we have a call like above or just a type lookup. + */ + AutoDecRef tpDict(PepType_GetDict(type_base)); + auto *flagType = PyDict_GetItem(tpDict.object(), rename); + if (currentOpcode_Is_CallMethNoArgs()) + return replaceNoArgWithZero(flagType); + Py_INCREF(flagType); + return flagType; + } + } + bool useFakeShortcuts = !(Enum::enumOption & Enum::ENOPT_NO_FAKESHORTCUT); + if (useFakeShortcuts) { + AutoDecRef tpDict(PepType_GetDict(type_base)); + auto *dict = tpDict.object(); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(dict, &pos, &key, &value)) { + /* + * Part 2: Check for a duplication into outer scope. + * ------------------------------------------------- + * We need to replace the shortcut + + QtCore.Qt.AlignLeft + + * by the correct call + + QtCore.Qt.AlignmentFlag.AlignLeft + + * That means: We need to search all Enums of the class. + */ + if (Py_TYPE(value) == EnumMeta) { + auto *valtype = reinterpret_cast<PyTypeObject *>(value); + AutoDecRef valtypeDict(PepType_GetDict(valtype)); + auto *member_map = PyDict_GetItem(valtypeDict.object(), _member_map_); + if (member_map && PyDict_Check(member_map)) { + result = PyDict_GetItem(member_map, name); + Py_XINCREF(result); + if (result) + return result; + } + } + } + } + } + return nullptr; +} + PyObject *mangled_type_getattro(PyTypeObject *type, PyObject *name) { /* @@ -180,11 +319,10 @@ PyObject *mangled_type_getattro(PyTypeObject *type, PyObject *name) * with the complex `tp_getattro` of `QObject` and other instances. * What we change here is the meta class of `QObject`. */ - static getattrofunc const type_getattro = PyType_Type.tp_getattro; + static getattrofunc const type_getattro = PepExt_Type_GetGetAttroSlot(&PyType_Type); static PyObject *const ignAttr1 = PyName::qtStaticMetaObject(); static PyObject *const ignAttr2 = PyMagicName::get(); static PyTypeObject *const EnumMeta = getPyEnumMeta(); - static PyObject *const _member_map_ = String::createStaticString("_member_map_"); if (SelectFeatureSet != nullptr) SelectFeatureSet(type); @@ -194,7 +332,7 @@ PyObject *mangled_type_getattro(PyTypeObject *type, PyObject *name) // The PYI files now look correct, but the old duplication is // emulated here. This should be removed in Qt 7, see `parser.py`. // - // FIXME PYSIDE7 should remove this forgivingness: + // FIXME PYSIDE7 should remove this forgiveness: // // The duplication of enum values into the enclosing scope, allowing to write // Qt.AlignLeft instead of Qt.Alignment.AlignLeft, is still implemented but @@ -211,88 +349,16 @@ PyObject *mangled_type_getattro(PyTypeObject *type, PyObject *name) } if (!ret && name != ignAttr1 && name != ignAttr2) { - PyObject *error_type, *error_value, *error_traceback; + PyObject *error_type{}, *error_value{}, *error_traceback{}; PyErr_Fetch(&error_type, &error_value, &error_traceback); - - // This is similar to `find_name_in_mro`, but instead of looking directly into - // tp_dict, we also search for the attribute in local classes of that dict (Part 2). - PyObject *mro = type->tp_mro; - assert(PyTuple_Check(mro)); - size_t idx, n = PyTuple_GET_SIZE(mro); - for (idx = 0; idx < n; ++idx) { - auto *base = PyTuple_GET_ITEM(mro, idx); - auto *type_base = reinterpret_cast<PyTypeObject *>(base); - auto sotp = PepType_SOTP(type_base); - // The EnumFlagInfo structure tells us if there are Enums at all. - const char **enumFlagInfo = sotp->enumFlagInfo; - if (!(enumFlagInfo)) - continue; - if (!sotp->enumFlagsDict) - initEnumFlagsDict(type_base); - bool useFakeRenames = !(Enum::enumOption & Enum::ENOPT_NO_FAKERENAMES); - if (useFakeRenames) { - auto *rename = PyDict_GetItem(sotp->enumFlagsDict, name); - if (rename) { - /* - * Part 1: Look into the enumFlagsDict if we have an old flags name. - * ------------------------------------------------------------- - * We need to replace the parameterless - - QtCore.Qt.Alignment() - - * by the one-parameter call - - QtCore.Qt.AlignmentFlag(0) - - * That means: We need to bind the zero as default into a wrapper and - * return that to be called. - * - * Addendum: - * --------- - * We first need to look into the current opcode of the bytecode to find - * out if we have a call like above or just a type lookup. - */ - auto *flagType = PyDict_GetItem(type_base->tp_dict, rename); - if (currentOpcode_Is_CallMethNoArgs()) - return replaceNoArgWithZero(flagType); - Py_INCREF(flagType); - return flagType; - } - } - bool useFakeShortcuts = !(Enum::enumOption & Enum::ENOPT_NO_FAKESHORTCUT); - if (useFakeShortcuts) { - auto *dict = type_base->tp_dict; - PyObject *key, *value; - Py_ssize_t pos = 0; - while (PyDict_Next(dict, &pos, &key, &value)) { - /* - * Part 2: Check for a duplication into outer scope. - * ------------------------------------------------- - * We need to replace the shortcut - - QtCore.Qt.AlignLeft - - * by the correct call - - QtCore.Qt.AlignmentFlag.AlignLeft - - * That means: We need to search all Enums of the class. - */ - if (Py_TYPE(value) == EnumMeta) { - auto *valtype = reinterpret_cast<PyTypeObject *>(value); - auto *member_map = PyDict_GetItem(valtype->tp_dict, _member_map_); - if (member_map && PyDict_Check(member_map)) { - auto *result = PyDict_GetItem(member_map, name); - if (result) { - Py_INCREF(result); - return result; - } - } - } - } - } + ret = lookupUnqualifiedOrOldEnum(type, name); + if (ret) { + Py_DECREF(error_type); + Py_XDECREF(error_value); + Py_XDECREF(error_traceback); + } else { + PyErr_Restore(error_type, error_value, error_traceback); } - PyErr_Restore(error_type, error_value, error_traceback); } return ret; } @@ -302,12 +368,14 @@ PyObject *Sbk_TypeGet___dict__(PyTypeObject *type, void * /* context */) /* * This is the override for getting a dict. */ - auto dict = type->tp_dict; + AutoDecRef tpDict(PepType_GetDict(type)); + auto *dict = tpDict.object();; if (dict == nullptr) Py_RETURN_NONE; if (SelectFeatureSet != nullptr) { SelectFeatureSet(type); - dict = type->tp_dict; + tpDict.reset(PepType_GetDict(type)); + dict = tpDict.object(); } return PyDictProxy_New(dict); } diff --git a/sources/shiboken6/libshiboken/sbkmodule.cpp b/sources/shiboken6/libshiboken/sbkmodule.cpp index aeae34a36..76087fbb5 100644 --- a/sources/shiboken6/libshiboken/sbkmodule.cpp +++ b/sources/shiboken6/libshiboken/sbkmodule.cpp @@ -2,54 +2,499 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "sbkmodule.h" +#include "autodecref.h" #include "basewrapper.h" #include "bindingmanager.h" +#include "sbkstring.h" +#include "sbkcppstring.h" +#include "sbkconverter_p.h" + #include <unordered_map> +#include <unordered_set> +#include <vector> +#include <cstring> + +/// This hash maps module objects to arrays of converters. +using ModuleConvertersMap = std::unordered_map<PyObject *, SbkConverter **> ; /// This hash maps module objects to arrays of Python types. -using ModuleTypesMap = std::unordered_map<PyObject *, PyTypeObject **> ; +using ModuleTypesMap = std::unordered_map<PyObject *, Shiboken::Module::TypeInitStruct *> ; -/// This hash maps module objects to arrays of converters. -using ModuleConvertersMap = std::unordered_map<PyObject *, SbkConverter **>; +struct TypeCreationStruct +{ + Shiboken::Module::TypeCreationFunction func; + std::vector<std::string> subtypeNames; +}; + +/// This hash maps type names to type creation structs. +using NameToTypeFunctionMap = std::unordered_map<std::string, TypeCreationStruct> ; + +/// This hash maps module objects to maps of names to functions. +using ModuleToFuncsMap = std::unordered_map<PyObject *, NameToTypeFunctionMap> ; /// All types produced in imported modules are mapped here. static ModuleTypesMap moduleTypes; static ModuleConvertersMap moduleConverters; +static ModuleToFuncsMap moduleToFuncs; namespace Shiboken { namespace Module { +// PYSIDE-2404: Replacing the arguments generated by cpythonTypeNameExt +// by a function call. +LIBSHIBOKEN_API PyTypeObject *get(TypeInitStruct &typeStruct) +{ + if (typeStruct.type != nullptr) + return typeStruct.type; + + static PyObject *sysModules = PyImport_GetModuleDict(); + + // The slow path for initialization. + // We get the type by following the chain from the module. + // As soon as types[index] gets filled, we can stop. + + std::string_view names(typeStruct.fullName); + const bool usePySide = names.compare(0, 8, "PySide6.") == 0; + auto dotPos = usePySide ? names.find('.', 8) : names.find('.'); + auto startPos = dotPos + 1; + AutoDecRef modName(String::fromCppStringView(names.substr(0, dotPos))); + auto *modOrType = PyDict_GetItem(sysModules, modName); + if (modOrType == nullptr) { + PyErr_Format(PyExc_SystemError, "Module \"%U\" should already be in sys.modules", + modName.object()); + return nullptr; + } + + do { + dotPos = names.find('.', startPos); + auto typeName = dotPos != std::string::npos + ? names.substr(startPos, dotPos - startPos) + : names.substr(startPos); + startPos = dotPos + 1; + AutoDecRef obTypeName(String::fromCppStringView(typeName)); + modOrType = PyObject_GetAttr(modOrType, obTypeName); + } while (typeStruct.type == nullptr && dotPos != std::string::npos); + + return typeStruct.type; +} + +static void incarnateHelper(PyObject *module, const std::string_view names, + const NameToTypeFunctionMap &nameToFunc) +{ + auto dotPos = names.find('.'); + std::string::size_type startPos = 0; + auto *modOrType{module}; + while (dotPos != std::string::npos) { + auto typeName = names.substr(startPos, dotPos - startPos); + AutoDecRef obTypeName(String::fromCppStringView(typeName)); + modOrType = PyObject_GetAttr(modOrType, obTypeName); + startPos = dotPos + 1; + dotPos = names.find('.', startPos); + } + // now we have the type to create. + auto funcIter = nameToFunc.find(std::string(names)); + // - call this function that returns a PyTypeObject + auto tcStruct = funcIter->second; + auto initFunc = tcStruct.func; + PyTypeObject *type = initFunc(modOrType); + auto name = names.substr(startPos); + PyObject_SetAttrString(modOrType, name.data(), reinterpret_cast<PyObject *>(type)); +} + +static void incarnateSubtypes(PyObject *module, + const std::vector<std::string> &nameList, + NameToTypeFunctionMap &nameToFunc) +{ + for (auto const & tableIter : nameList) { + std::string_view names(tableIter); + incarnateHelper(module, names, nameToFunc); + } +} + +static PyTypeObject *incarnateType(PyObject *module, const char *name, + NameToTypeFunctionMap &nameToFunc) +{ + // - locate the name and retrieve the generating function + auto funcIter = nameToFunc.find(name); + if (funcIter == nameToFunc.end()) { + // attribute does really not exist. + PyErr_SetNone(PyExc_AttributeError); + return nullptr; + } + // - call this function that returns a PyTypeObject + auto tcStruct = funcIter->second; + auto initFunc = tcStruct.func; + auto *modOrType{module}; + + // PYSIDE-2404: Make sure that no switching happens during type creation. + auto saveFeature = initSelectableFeature(nullptr); + PyTypeObject *type = initFunc(modOrType); + if (!tcStruct.subtypeNames.empty()) + incarnateSubtypes(module, tcStruct.subtypeNames, nameToFunc); + initSelectableFeature(saveFeature); + + // - assign this object to the name in the module + auto *res = reinterpret_cast<PyObject *>(type); + Py_INCREF(res); + PyModule_AddObject(module, name, res); // steals reference + // - remove the entry, if not by something cleared. + if (!nameToFunc.empty()) + nameToFunc.erase(funcIter); + // - return the PyTypeObject. + return type; +} + +// PYSIDE-2404: Make sure that the mentioned classes really exist. +// Used in `Pyside::typeName`. Because the result will be cached by +// the creation of the type(s), this is efficient. +void loadLazyClassesWithName(const char *name) +{ + for (auto const & tableIter : moduleToFuncs) { + auto nameToFunc = tableIter.second; + auto funcIter = nameToFunc.find(name); + if (funcIter != nameToFunc.end()) { + // attribute exists in the lazy types. + auto *module = tableIter.first; + incarnateType(module, name, nameToFunc); + } + } +} + +// PYSIDE-2404: Completely load all not yet loaded classes. +// This is needed to resolve a star import. +void resolveLazyClasses(PyObject *module) +{ + // - locate the module in the moduleTofuncs mapping + auto tableIter = moduleToFuncs.find(module); + if (tableIter == moduleToFuncs.end()) + return; + + // - see if there are still unloaded elements + auto &nameToFunc = tableIter->second; + + // - incarnate all types. + while (!nameToFunc.empty()) { + auto it = nameToFunc.begin(); + auto attrNameStr = it->first; + incarnateType(module, attrNameStr.c_str(), nameToFunc); + } +} + +// PYSIDE-2404: Override the gettattr function of modules. +static getattrofunc origModuleGetattro{}; + +// PYSIDE-2404: Use the patched module getattr to do on-demand initialization. +// This modifies _all_ modules but should have no impact. +static PyObject *PyModule_lazyGetAttro(PyObject *module, PyObject *name) +{ + // - check if the attribute is present and return it. + auto *attr = PyObject_GenericGetAttr(module, name); + // - we handle AttributeError, only. + if (!(attr == nullptr && PyErr_ExceptionMatches(PyExc_AttributeError))) + return attr; + + PyErr_Clear(); + // - locate the module in the moduleTofuncs mapping + auto tableIter = moduleToFuncs.find(module); + // - if this is not our module, use the original + if (tableIter == moduleToFuncs.end()) + return origModuleGetattro(module, name); + + // - locate the name and retrieve the generating function + const char *attrNameStr = Shiboken::String::toCString(name); + auto &nameToFunc = tableIter->second; + // - create the real type and handle subtypes + auto *type = incarnateType(module, attrNameStr, nameToFunc); + auto *ret = reinterpret_cast<PyObject *>(type); + // - if attribute does really not exist use the original + if (ret == nullptr && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + return origModuleGetattro(module, name); + } + return ret; +} + +// PYSIDE-2404: Supply a new module dir for not yet visible entries. +// This modification is only for "our" modules. +static PyObject *_module_dir_template(PyObject * /* self */, PyObject *args) +{ + static PyObject *const _dict = Shiboken::String::createStaticString("__dict__"); + // The dir function must replace all of the builtin function. + PyObject *module{}; + if (!PyArg_ParseTuple(args, "O", &module)) + return nullptr; + + auto tableIter = moduleToFuncs.find(module); + assert(tableIter != moduleToFuncs.end()); + Shiboken::AutoDecRef dict(PyObject_GetAttr(module, _dict)); + auto *ret = PyDict_Keys(dict); + // Now add all elements that were not yet in the dict. + auto &nameToFunc = tableIter->second; + for (const auto &funcIter : nameToFunc) { + const char *name = funcIter.first.c_str(); + Shiboken::AutoDecRef pyName(PyUnicode_FromString(name)); + PyList_Append(ret, pyName); + } + return ret; +} + +static PyMethodDef module_methods[] = { + {"__dir__", (PyCFunction)_module_dir_template, METH_VARARGS, nullptr}, + {nullptr, nullptr, 0, nullptr} +}; + +// Python 3.8 - 3.12 +static int const LOAD_CONST_312 = 100; +static int const IMPORT_NAME_312 = 108; + +static bool isImportStar(PyObject *module) +{ + // Find out whether we have a star import. This must work even + // when we have no import support from feature. + static PyObject *const _f_code = Shiboken::String::createStaticString("f_code"); + static PyObject *const _f_lasti = Shiboken::String::createStaticString("f_lasti"); + static PyObject *const _f_back = Shiboken::String::createStaticString("f_back"); + static PyObject *const _co_code = Shiboken::String::createStaticString("co_code"); + static PyObject *const _co_consts = Shiboken::String::createStaticString("co_consts"); + static PyObject *const _co_names = Shiboken::String::createStaticString("co_names"); + + auto *obFrame = reinterpret_cast<PyObject *>(PyEval_GetFrame()); + if (obFrame == nullptr) + return true; // better assume worst-case. + + Py_INCREF(obFrame); + AutoDecRef dec_frame(obFrame); + + // Calculate the offset of the running import_name opcode on the stack. + // Right before that there must be a load_const with the tuple `("*",)`. + while (dec_frame.object() != Py_None) { + AutoDecRef dec_f_code(PyObject_GetAttr(dec_frame, _f_code)); + AutoDecRef dec_co_code(PyObject_GetAttr(dec_f_code, _co_code)); + AutoDecRef dec_f_lasti(PyObject_GetAttr(dec_frame, _f_lasti)); + Py_ssize_t f_lasti = PyLong_AsSsize_t(dec_f_lasti); + Py_ssize_t code_len; + char *co_code{}; + PyBytes_AsStringAndSize(dec_co_code, &co_code, &code_len); + uint8_t opcode2 = co_code[f_lasti]; + uint8_t opcode1 = co_code[f_lasti - 2]; + if (opcode1 == LOAD_CONST_312 && opcode2 == IMPORT_NAME_312) { + uint8_t oparg1 = co_code[f_lasti - 1]; + uint8_t oparg2 = co_code[f_lasti + 1]; + AutoDecRef dec_co_consts(PyObject_GetAttr(dec_f_code, _co_consts)); + auto *fromlist = PyTuple_GetItem(dec_co_consts, oparg1); + if (PyTuple_Check(fromlist) && PyTuple_Size(fromlist) == 1 + && Shiboken::String::toCString(PyTuple_GetItem(fromlist, 0))[0] == '*') { + AutoDecRef dec_co_names(PyObject_GetAttr(dec_f_code, _co_names)); + const char *name = String::toCString(PyTuple_GetItem(dec_co_names, oparg2)); + const char *modName = PyModule_GetName(module); + if (std::strcmp(name, modName) == 0) + return true; + } + } + dec_frame.reset(PyObject_GetAttr(dec_frame, _f_back)); + } + return false; +} + +// PYSIDE-2404: These modules produce ambiguous names which we cannot handle, yet. +static std::unordered_set<std::string> dontLazyLoad{ + "sample", + "smart", + "testbinding" +}; + +static const std::unordered_set<std::string> knownModules{ + "shiboken6.Shiboken", + "minimal", + "other", + "sample", + "smart", + "scriptableapplication", + "testbinding" +}; + +static bool canNotLazyLoad(PyObject *module) +{ + const char *modName = PyModule_GetName(module); + + // There are no more things that must be disabled :-D + return dontLazyLoad.find(modName) != dontLazyLoad.end(); +} + +static bool shouldLazyLoad(PyObject *module) +{ + const char *modName = PyModule_GetName(module); + + if (knownModules.find(modName) != knownModules.end()) + return true; + return std::strncmp(modName, "PySide6.", 8) == 0; +} + +void checkIfShouldLoadImmediately(PyObject *module, const std::string &name, + const NameToTypeFunctionMap &nameToFunc) +{ + static const char *flag = getenv("PYSIDE6_OPTION_LAZY"); + static const int value = flag != nullptr ? std::atoi(flag) : 1; + + // PYSIDE-2404: Lazy Loading + // + // Options: + // 0 - switch lazy loading off. + // 1 - lazy loading for all known modules. + // 3 - lazy loading for any module. + // + // By default we lazy load all known modules (option = 1). + if (value == 0 // completely disabled + || canNotLazyLoad(module) // for some reason we cannot lazy load + || (value == 1 && !shouldLazyLoad(module)) // not a known module + ) { + incarnateHelper(module, name, nameToFunc); + } +} + +void AddTypeCreationFunction(PyObject *module, + const char *name, + TypeCreationFunction func) +{ + // - locate the module in the moduleTofuncs mapping + auto tableIter = moduleToFuncs.find(module); + assert(tableIter != moduleToFuncs.end()); + // - Assign the name/generating function tcStruct. + auto &nameToFunc = tableIter->second; + TypeCreationStruct tcStruct{func, {}}; + auto nit = nameToFunc.find(name); + if (nit == nameToFunc.end()) + nameToFunc.insert(std::make_pair(name, tcStruct)); + else + nit->second = tcStruct; + + checkIfShouldLoadImmediately(module, name, nameToFunc); +} + +void AddTypeCreationFunction(PyObject *module, + const char *containerName, + TypeCreationFunction func, + const char *namePath) +{ + // - locate the module in the moduleTofuncs mapping + auto tableIter = moduleToFuncs.find(module); + assert(tableIter != moduleToFuncs.end()); + // - Assign the name/generating function tcStruct. + auto &nameToFunc = tableIter->second; + auto nit = nameToFunc.find(containerName); + + // - insert namePath into the subtype vector of the main type. + nit->second.subtypeNames.push_back(namePath); + // - insert it also as its own entry. + nit = nameToFunc.find(namePath); + TypeCreationStruct tcStruct{func, {}}; + if (nit == nameToFunc.end()) + nameToFunc.insert(std::make_pair(namePath, tcStruct)); + else + nit->second = tcStruct; + + checkIfShouldLoadImmediately(module, namePath, nameToFunc); +} + PyObject *import(const char *moduleName) { PyObject *sysModules = PyImport_GetModuleDict(); PyObject *module = PyDict_GetItemString(sysModules, moduleName); - if (module) + if (module != nullptr) Py_INCREF(module); else module = PyImport_ImportModule(moduleName); - if (!module) - PyErr_Format(PyExc_ImportError,"could not import module '%s'", moduleName); + if (module == nullptr) + PyErr_Format(PyExc_ImportError, "could not import module '%s'", moduleName); return module; } -PyObject *create(const char * /* moduleName */, void *moduleData) +// PYSIDE-2404: Redirecting import for "import *" support. +// +// The first import will be handled by the isImportStar function. +// But the same module might be imported twice, which would give no +// introspection due to module caching. + +static PyObject *origImportFunc{}; + +static PyObject *lazy_import(PyObject * /* self */, PyObject *args, PyObject *kwds) +{ + auto *ret = PyObject_Call(origImportFunc, args, kwds); + if (ret != nullptr) { + // PYSIDE-2404: Support star import when lazy loading. + if (PyTuple_Size(args) >= 4) { + auto *fromlist = PyTuple_GetItem(args, 3); + if (PyTuple_Check(fromlist) && PyTuple_Size(fromlist) == 1 + && Shiboken::String::toCString(PyTuple_GetItem(fromlist, 0))[0] == '*') + Shiboken::Module::resolveLazyClasses(ret); + } + } + return ret; +} + +static PyMethodDef lazy_methods[] = { + {"__lazy_import__", (PyCFunction)lazy_import, METH_VARARGS | METH_KEYWORDS, nullptr}, + {nullptr, nullptr, 0, nullptr} +}; + +PyObject *create(const char * /* modName */, void *moduleData) { + static auto *sysModules = PyImport_GetModuleDict(); + static auto *builtins = PyEval_GetBuiltins(); + static auto *partial = Pep_GetPartialFunction(); + static bool lazy_init{}; + Shiboken::init(); - return PyModule_Create(reinterpret_cast<PyModuleDef *>(moduleData)); + auto *module = PyModule_Create(reinterpret_cast<PyModuleDef *>(moduleData)); + + // Setup of a dir function for "missing" classes. + auto *moduleDirTemplate = PyCFunction_NewEx(module_methods, nullptr, nullptr); + // Turn this function into a bound object, so we have access to the module. + auto *moduleDir = PyObject_CallFunctionObjArgs(partial, moduleDirTemplate, module, nullptr); + PyModule_AddObject(module, module_methods->ml_name, moduleDir); // steals reference + // Insert an initial empty table for the module. + NameToTypeFunctionMap empty; + moduleToFuncs.insert(std::make_pair(module, empty)); + + // A star import must be done unconditionally. Use the complete name. + if (isImportStar(module)) + dontLazyLoad.insert(PyModule_GetName(module)); + + if (!lazy_init) { + // Install the getattr patch. + origModuleGetattro = PyModule_Type.tp_getattro; + PyModule_Type.tp_getattro = PyModule_lazyGetAttro; + // Add the lazy import redirection. + origImportFunc = PyDict_GetItemString(builtins, "__import__"); + auto *func = PyCFunction_NewEx(lazy_methods, nullptr, nullptr); + PyDict_SetItemString(builtins, "__import__", func); + // Everything is set. + lazy_init = true; + } + // PYSIDE-2404: Nuitka inserts some additional code in standalone mode + // in an invisible virtual module (i.e. `QtCore-postLoad`) + // that gets imported before the running import can call + // `_PyImport_FixupExtensionObject` which does the insertion + // into `sys.modules`. This can cause a race condition. + // Insert the module early into the module dict to prevend recursion. + PyDict_SetItemString(sysModules, PyModule_GetName(module), module); + // Clear the non-existing name cache because we have a new module. + Shiboken::Conversions::clearNegativeLazyCache(); + return module; } -void registerTypes(PyObject *module, PyTypeObject **types) +void registerTypes(PyObject *module, TypeInitStruct *types) { auto iter = moduleTypes.find(module); if (iter == moduleTypes.end()) moduleTypes.insert(std::make_pair(module, types)); } -PyTypeObject **getTypes(PyObject *module) +TypeInitStruct *getTypes(PyObject *module) { auto iter = moduleTypes.find(module); return (iter == moduleTypes.end()) ? 0 : iter->second; diff --git a/sources/shiboken6/libshiboken/sbkmodule.h b/sources/shiboken6/libshiboken/sbkmodule.h index a4f7837f5..2c407e09d 100644 --- a/sources/shiboken6/libshiboken/sbkmodule.h +++ b/sources/shiboken6/libshiboken/sbkmodule.h @@ -12,8 +12,22 @@ extern "C" struct SbkConverter; } -namespace Shiboken { -namespace Module { +namespace Shiboken::Module { + +struct TypeInitStruct +{ + PyTypeObject *type; + const char *fullName; +}; + +/// PYSIDE-2404: Replacing the arguments in cpythonTypeNameExt by a function. +LIBSHIBOKEN_API PyTypeObject *get(TypeInitStruct &typeStruct); + +/// PYSIDE-2404: Make sure that mentioned classes really exist. +LIBSHIBOKEN_API void loadLazyClassesWithName(const char *name); + +/// PYSIDE-2404: incarnate all classes for star imports. +LIBSHIBOKEN_API void resolveLazyClasses(PyObject *module); /** * Imports and returns the module named \p moduleName, or a NULL pointer in case of failure. @@ -30,19 +44,31 @@ LIBSHIBOKEN_API PyObject *import(const char *moduleName); */ LIBSHIBOKEN_API PyObject *create(const char *moduleName, void *moduleData); +using TypeCreationFunction = PyTypeObject *(*)(PyObject *module); + +/// Adds a type creation function to the module. +LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module, + const char *name, + TypeCreationFunction func); + +LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module, + const char *name, + TypeCreationFunction func, + const char *containerName); + /** * Registers the list of types created by \p module. * \param module Module where the types were created. * \param types Array of PyTypeObject *objects representing the types created on \p module. */ -LIBSHIBOKEN_API void registerTypes(PyObject *module, PyTypeObject **types); +LIBSHIBOKEN_API void registerTypes(PyObject *module, TypeInitStruct *types); /** * Retrieves the array of types. * \param module Module where the types were created. * \returns A pointer to the PyTypeObject *array of types. */ -LIBSHIBOKEN_API PyTypeObject **getTypes(PyObject *module); +LIBSHIBOKEN_API TypeInitStruct *getTypes(PyObject *module); /** * Registers the list of converters created by \p module for non-wrapper types. @@ -58,6 +84,6 @@ LIBSHIBOKEN_API void registerTypeConverters(PyObject *module, SbkConverter **con */ LIBSHIBOKEN_API SbkConverter **getTypeConverters(PyObject *module); -} } // namespace Shiboken::Module +} // namespace Shiboken::Module #endif // SBK_MODULE_H diff --git a/sources/shiboken6/libshiboken/sbknumpy.cpp b/sources/shiboken6/libshiboken/sbknumpy.cpp index 060a2b6f4..b6422e73f 100644 --- a/sources/shiboken6/libshiboken/sbknumpy.cpp +++ b/sources/shiboken6/libshiboken/sbknumpy.cpp @@ -17,9 +17,27 @@ namespace Shiboken::Numpy { +#ifdef HAVE_NUMPY +static void initNumPy() +{ + // PYSIDE-2404: Delay-initialize numpy from check() as it causes a + // significant startup delay (~770 allocations in memray) + static bool initialized = false; + if (initialized) + return; + initialized = true; + // Expanded from macro "import_array" in __multiarray_api.h + // Make sure to read about the magic defines PY_ARRAY_UNIQUE_SYMBOL etc., + // when changing this or spreading the code over several source files. + if (_import_array() < 0) + PyErr_Print(); +} +#endif // HAVE_NUMPY + bool check(PyObject *pyIn) { #ifdef HAVE_NUMPY + initNumPy(); return PyArray_Check(pyIn); #else SBK_UNUSED(pyIn); diff --git a/sources/shiboken6/libshiboken/sbknumpyarrayconverter.cpp b/sources/shiboken6/libshiboken/sbknumpyarrayconverter.cpp index c8541adf5..835a97524 100644 --- a/sources/shiboken6/libshiboken/sbknumpyarrayconverter.cpp +++ b/sources/shiboken6/libshiboken/sbknumpyarrayconverter.cpp @@ -94,8 +94,7 @@ std::ostream &operator<<(std::ostream &str, PyArrayObject *o) return str; } -namespace Shiboken { -namespace Conversions { +namespace Shiboken::Conversions { // Internals from sbkarrayconverter.cpp SbkArrayConverter *createArrayConverter(IsArrayConvertibleToCppFunc toCppCheckFunc); @@ -105,6 +104,7 @@ SbkArrayConverter *unimplementedArrayConverter(); template <int dimension> static bool isPrimitiveArray(PyObject *pyIn, int expectedNpType) { + Shiboken::Numpy::initNumPy(); if (!PyArray_Check(pyIn)) return false; auto *pya = reinterpret_cast<PyArrayObject *>(pyIn); @@ -210,6 +210,9 @@ static PythonToCppFunc checkArray2(PyObject *pyIn, int dim1, int dim2) template <class T> static void setOrExtendArrayConverter(int dimension, IsArrayConvertibleToCppFunc toCppCheckFunc) { + // PYSIDE-2404/FIXME: When adding a C++ -> Python conversion, be sure + // to delay-initialize numpy in the converter (similar to the + // initialization in check() for the Python -> C++ conversion). SbkArrayConverter *arrayConverter = ArrayTypeConverter<T>(dimension); if (arrayConverter == unimplementedArrayConverter()) { arrayConverter = createArrayConverter(toCppCheckFunc); @@ -235,15 +238,6 @@ static inline void extendArrayConverter2() void initNumPyArrayConverters() { - // Expanded from macro "import_array" in __multiarray_api.h - // Make sure to read about the magic defines PY_ARRAY_UNIQUE_SYMBOL etc., - // when changing this or spreading the code over several source files. - if (_import_array() < 0) { - if (debugNumPy) - PyErr_Print(); - PyErr_Clear(); - return; - } // Extend the converters for primitive types by NumPy ones. extendArrayConverter1<short, NPY_SHORT>(); extendArrayConverter2<short, NPY_SHORT>(); @@ -273,5 +267,4 @@ void initNumPyArrayConverters() extendArrayConverter2<double, NPY_DOUBLE>(); } -} // namespace Conversions -} // namespace Shiboken +} // namespace Shiboken::Conversions diff --git a/sources/shiboken6/libshiboken/sbknumpyview.cpp b/sources/shiboken6/libshiboken/sbknumpyview.cpp index 44a4b5587..bafbf8038 100644 --- a/sources/shiboken6/libshiboken/sbknumpyview.cpp +++ b/sources/shiboken6/libshiboken/sbknumpyview.cpp @@ -6,12 +6,54 @@ #include "helper.h" #include <iostream> #include <iomanip> +#include <optional> #ifdef HAVE_NUMPY namespace Shiboken { namespace Numpy { +static std::optional<View::Type> viewTypeFromNumPy(int npt) +{ + switch (npt) { + case NPY_SHORT: + return View::Int16; + case NPY_USHORT: + return View::Unsigned16; + case NPY_INT: + return View::Int; + case NPY_UINT: + return View::Unsigned; + case NPY_LONG: + if constexpr (sizeof(long) == sizeof(int)) + return View::Int; + if constexpr (sizeof(long) == sizeof(int64_t)) + return View::Int64; + break; + case NPY_ULONG: + if constexpr (sizeof(long) == sizeof(int)) + return View::Unsigned; + if constexpr (sizeof(long) == sizeof(int64_t)) + return View::Unsigned64; + break; + case NPY_LONGLONG: + if constexpr (sizeof(long long) == 8) + return View::Int64; + break; + case NPY_ULONGLONG: + if constexpr (sizeof(long long) == 8) + return View::Unsigned64; + break; + case NPY_FLOAT: + return View::Float; + case NPY_DOUBLE: + return View::Double; + default: + break; + } + return {}; +} + View View::fromPyObject(PyObject *pyIn) { if (pyIn == nullptr || PyArray_Check(pyIn) == 0) @@ -23,27 +65,13 @@ View View::fromPyObject(PyObject *pyIn) if (ndim > 2) return {}; - View::Type type; - switch (PyArray_TYPE(ar)) { - case NPY_INT: - type = View::Int; - break; - case NPY_UINT: - type = View::Unsigned; - break; - case NPY_FLOAT: - type = View::Float; - break; - case NPY_DOUBLE: - type = View::Double; - break; - default: + const auto typeO = viewTypeFromNumPy(PyArray_TYPE(ar)); + if (!typeO.has_value()) return {}; - } View result; result.ndim = ndim; - result.type = type; + result.type = typeO.value(); result.data = PyArray_DATA(ar); result.dimensions[0] = PyArray_DIMS(ar)[0]; result.stride[0] = PyArray_STRIDES(ar)[0]; @@ -91,11 +119,29 @@ std::ostream &operator<<(std::ostream &str, const debugPyArrayObject &a) } str << "], type="; switch (type) { + case NPY_SHORT: + str << "short"; + break; + case NPY_USHORT: + str << "ushort"; + break; case NPY_INT: - str << "int"; + str << "int32"; break; case NPY_UINT: - str << "uint"; + str << "uint32"; + break; + case NPY_LONG: + str << "long"; + break; + case NPY_ULONG: + str << "ulong"; + break; + case NPY_LONGLONG: + str << "long long"; + break; + case NPY_ULONGLONG: + str << "ulong long"; break; case NPY_FLOAT: str << "float"; @@ -122,12 +168,30 @@ std::ostream &operator<<(std::ostream &str, const debugPyArrayObject &a) if (const int dim0 = PyArray_DIMS(ar)[0]) { auto *data = PyArray_DATA(ar); switch (type) { + case NPY_SHORT: + debugArray(str, reinterpret_cast<const short *>(data), dim0); + break; + case NPY_USHORT: + debugArray(str, reinterpret_cast<const unsigned short *>(data), dim0); + break; case NPY_INT: debugArray(str, reinterpret_cast<const int *>(data), dim0); break; case NPY_UINT: debugArray(str, reinterpret_cast<const unsigned *>(data), dim0); break; + case NPY_LONG: + debugArray(str, reinterpret_cast<const long *>(data), dim0); + break; + case NPY_ULONG: + debugArray(str, reinterpret_cast<const unsigned long*>(data), dim0); + break; + case NPY_LONGLONG: + debugArray(str, reinterpret_cast<const long long *>(data), dim0); + break; + case NPY_ULONGLONG: + debugArray(str, reinterpret_cast<const unsigned long long *>(data), dim0); + break; case NPY_FLOAT: debugArray(str, reinterpret_cast<const float *>(data), dim0); break; diff --git a/sources/shiboken6/libshiboken/sbknumpyview.h b/sources/shiboken6/libshiboken/sbknumpyview.h index d41e2c716..918913b78 100644 --- a/sources/shiboken6/libshiboken/sbknumpyview.h +++ b/sources/shiboken6/libshiboken/sbknumpyview.h @@ -22,7 +22,7 @@ LIBSHIBOKEN_API bool check(PyObject *pyIn); /// numpy headers. struct LIBSHIBOKEN_API View { - enum Type { Int, Unsigned, Float, Double}; + enum Type { Int, Unsigned, Float, Double, Int16, Unsigned16, Int64, Unsigned64 }; static View fromPyObject(PyObject *pyIn); diff --git a/sources/shiboken6/libshiboken/sbksmartpointer.cpp b/sources/shiboken6/libshiboken/sbksmartpointer.cpp new file mode 100644 index 000000000..ee28f7db8 --- /dev/null +++ b/sources/shiboken6/libshiboken/sbksmartpointer.cpp @@ -0,0 +1,58 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "sbksmartpointer.h" +#include "sbkstring.h" +#include "autodecref.h" + +#include <unordered_set> + +namespace Shiboken::SmartPointer +{ + +PyObject *repr(PyObject *pointer, PyObject *pointee) +{ + Shiboken::AutoDecRef pointerRepr(Shiboken::String::repr(pointer)); + if (pointer == nullptr) + return pointerRepr.release(); + + Shiboken::AutoDecRef pointeeRepr(pointee != nullptr + ? PyObject_Repr(pointee) + : Shiboken::String::repr(pointee)); + + return PyUnicode_FromFormat("%U (%U)", pointerRepr.object(), pointeeRepr.object()); +} + +// __dir__ for a smart pointer. Add the __dir__ entries of the pointee to the list. +PyObject *dir(PyObject *pointer, PyObject *pointee) +{ + if (pointer == nullptr) + return PyList_New(0); + // Get the pointer's dir entries. Note: PyObject_Dir() cannot be called on + // self, will crash. Work around by using the type dict keys. + AutoDecRef tpDict(PepType_GetDict(Py_TYPE(pointer))); + auto *result = PyMapping_Keys(tpDict); + + if (pointee != nullptr && pointee != Py_None) { + // Add the entries of the pointee that do not exist in the pointer's list. + // Since Python internally caches strings; we can use a set of PyObject *. + std::unordered_set<PyObject *> knownStrings; + for (Py_ssize_t i = 0, size = PySequence_Size(result); i < size; ++i) { + Shiboken::AutoDecRef item(PySequence_GetItem(result, i)); + knownStrings.insert(item.object()); + } + const auto knownEnd = knownStrings.end(); + + Shiboken::AutoDecRef pointeeDir(PyObject_Dir(pointee)); + for (Py_ssize_t i = 0, size = PySequence_Size(pointeeDir.object()); i < size; ++i) { + Shiboken::AutoDecRef item(PySequence_GetItem(pointeeDir, i)); + if (knownStrings.find(item.object()) == knownEnd) + PyList_Append(result, item.object()); + } + } + + PyList_Sort(result); + return result; +} + +} // namespace Shiboken::SmartPointer diff --git a/sources/shiboken6/libshiboken/sbksmartpointer.h b/sources/shiboken6/libshiboken/sbksmartpointer.h new file mode 100644 index 000000000..5e2022722 --- /dev/null +++ b/sources/shiboken6/libshiboken/sbksmartpointer.h @@ -0,0 +1,18 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef SBK_SBKSMARTPOINTER_H +#define SBK_SBKSMARTPOINTER_H + +#include "sbkpython.h" +#include "shibokenmacros.h" + +namespace Shiboken::SmartPointer +{ + +LIBSHIBOKEN_API PyObject *repr(PyObject *pointer, PyObject *pointee); +LIBSHIBOKEN_API PyObject *dir(PyObject *pointer, PyObject *pointee); + +} // namespace Shiboken::SmartPointer + +#endif // SBK_SBKSMARTPOINTER_H diff --git a/sources/shiboken6/libshiboken/sbkstaticstrings.cpp b/sources/shiboken6/libshiboken/sbkstaticstrings.cpp index 0c8beaabb..023de0ea4 100644 --- a/sources/shiboken6/libshiboken/sbkstaticstrings.cpp +++ b/sources/shiboken6/libshiboken/sbkstaticstrings.cpp @@ -24,6 +24,7 @@ STATIC_STRING_IMPL(im_self, "im_self") STATIC_STRING_IMPL(loads, "loads") STATIC_STRING_IMPL(multi, "multi") STATIC_STRING_IMPL(name, "name") +STATIC_STRING_IMPL(orig_dict, "orig_dict") STATIC_STRING_IMPL(qApp, "qApp") STATIC_STRING_IMPL(result, "result") STATIC_STRING_IMPL(select_id, "select_id") @@ -75,7 +76,6 @@ STATIC_STRING_IMPL(iter, "__iter__") STATIC_STRING_IMPL(mro, "__mro__") STATIC_STRING_IMPL(new_, "__new__") STATIC_STRING_IMPL(objclass, "__objclass__") -STATIC_STRING_IMPL(signature, "__signature__") STATIC_STRING_IMPL(weakrefoffset, "__weakrefoffset__") STATIC_STRING_IMPL(opaque_container, "__opaque_container__") } // namespace PyMagicName diff --git a/sources/shiboken6/libshiboken/sbkstaticstrings.h b/sources/shiboken6/libshiboken/sbkstaticstrings.h index 02cc8a7f6..017790ee3 100644 --- a/sources/shiboken6/libshiboken/sbkstaticstrings.h +++ b/sources/shiboken6/libshiboken/sbkstaticstrings.h @@ -23,6 +23,7 @@ LIBSHIBOKEN_API PyObject *im_self(); LIBSHIBOKEN_API PyObject *loads(); LIBSHIBOKEN_API PyObject *multi(); LIBSHIBOKEN_API PyObject *name(); +LIBSHIBOKEN_API PyObject *orig_dict(); LIBSHIBOKEN_API PyObject *result(); LIBSHIBOKEN_API PyObject *select_id(); LIBSHIBOKEN_API PyObject *value(); diff --git a/sources/shiboken6/libshiboken/sbkstring.cpp b/sources/shiboken6/libshiboken/sbkstring.cpp index 8f2dc6d52..b5e87ca5a 100644 --- a/sources/shiboken6/libshiboken/sbkstring.cpp +++ b/sources/shiboken6/libshiboken/sbkstring.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "sbkstring.h" +#include "sbkenum.h" #include "sbkstaticstrings_p.h" #include "autodecref.h" @@ -14,6 +15,11 @@ bool checkIterable(PyObject *obj) return PyObject_HasAttr(obj, Shiboken::PyMagicName::iter()); } +bool checkIterableArgument(PyObject *obj) +{ + return checkIterable(obj) && !Shiboken::Enum::check(obj); +} + static PyObject *initPathLike() { PyObject *PathLike{}; @@ -233,4 +239,16 @@ PyObject *getSnakeCaseName(PyObject *name, bool lower) return name; } +// Return a generic representation of a PyObject as does PyObject_Repr(). +// Note: PyObject_Repr() may not be called on self from __repr__() as this +// causes a recursion. +PyObject *repr(PyObject *o) +{ + if (o == nullptr) + return PyUnicode_FromString("<NULL>"); + if (o == Py_None) + return PyUnicode_FromString("None"); + return PyUnicode_FromFormat("<%s object at %p>", Py_TYPE(o)->tp_name, o); +} + } // namespace Shiboken::String diff --git a/sources/shiboken6/libshiboken/sbkstring.h b/sources/shiboken6/libshiboken/sbkstring.h index a24c01def..ebc5428c7 100644 --- a/sources/shiboken6/libshiboken/sbkstring.h +++ b/sources/shiboken6/libshiboken/sbkstring.h @@ -13,6 +13,8 @@ namespace String { LIBSHIBOKEN_API bool check(PyObject *obj); LIBSHIBOKEN_API bool checkIterable(PyObject *obj); + /// Check for iterable function arguments (excluding enumerations) + LIBSHIBOKEN_API bool checkIterableArgument(PyObject *obj); LIBSHIBOKEN_API bool checkPath(PyObject *path); LIBSHIBOKEN_API bool checkType(PyTypeObject *obj); LIBSHIBOKEN_API bool checkChar(PyObject *obj); @@ -29,6 +31,7 @@ namespace String LIBSHIBOKEN_API PyObject *createStaticString(const char *str); LIBSHIBOKEN_API PyObject *getSnakeCaseName(const char *name, bool lower); LIBSHIBOKEN_API PyObject *getSnakeCaseName(PyObject *name, bool lower); + LIBSHIBOKEN_API PyObject *repr(PyObject *o); } // namespace String } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/sbktypefactory.cpp b/sources/shiboken6/libshiboken/sbktypefactory.cpp index 0da1a8e23..079548eed 100644 --- a/sources/shiboken6/libshiboken/sbktypefactory.cpp +++ b/sources/shiboken6/libshiboken/sbktypefactory.cpp @@ -7,6 +7,8 @@ extern "C" { +using Shiboken::AutoDecRef; + PyTypeObject *SbkType_FromSpec(PyType_Spec *spec) { return SbkType_FromSpec_BMDWB(spec, nullptr, nullptr, 0, 0, nullptr); @@ -37,6 +39,60 @@ static PyObject *_PyType_FromSpecWithBases(PyType_Spec *, PyObject *); #endif // PYPY_VERSION +// PYSIDE-2230: Not so temporary fix for Python 3.12. +// A tp_new is no longer allowed in a meta class. +// Hopefully, the Python devs will supply the missing support. +// It turned out that they will not fix that, as expected. +// Note: Python 3.12 is the first version that grabs the metaclass from base classes. +static PyObject *_PyType_FromSpecWithBasesHack(PyType_Spec *spec, + PyObject *bases, + PyTypeObject *meta) +{ + PyTypeObject *keepMeta{}; + newfunc keepNew{}; + AutoDecRef basesPatch{}; + + if (bases) { + if (bases == Py_None) { + // PYSIDE-2230: This is the SbkObject entry which has no base to provide + // the metaclass. We patch it in by modifying `object`s class. + assert(meta); + auto *base = reinterpret_cast<PyObject *>(&PyBaseObject_Type); + base->ob_type = meta; + basesPatch.reset(Py_BuildValue("(O)", &PyBaseObject_Type)); + bases = basesPatch.object(); + } + + Py_ssize_t n = PyTuple_GET_SIZE(bases); + for (auto idx = 0; idx < n; ++idx) { + PyTypeObject *base = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(bases, idx)); + PyTypeObject *meta = Py_TYPE(base); + if (meta->tp_new != PyType_Type.tp_new) { + // make sure there is no second meta class + assert(keepMeta == nullptr); + keepMeta = meta; + keepNew = meta->tp_new; + meta->tp_new = PyType_Type.tp_new; + } + } + } + +#if !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030C0000 + auto *ret = PyType_FromMetaclass(meta, nullptr /*module*/, spec, bases); +#else + auto *ret = _PyType_FromSpecWithBases(spec, bases); +#endif + + if (keepMeta) + keepMeta->tp_new = keepNew; + if (basesPatch.object()) { + // undo the metaclass patch. + auto *base = PyTuple_GET_ITEM(basesPatch.object(), 0); + base->ob_type = &PyType_Type; + } + return ret; +} + PyTypeObject *SbkType_FromSpec_BMDWB(PyType_Spec *spec, PyObject *bases, PyTypeObject *meta, @@ -61,7 +117,7 @@ PyTypeObject *SbkType_FromSpec_BMDWB(PyType_Spec *spec, int package_level = atoi(spec->name); const char *mod = new_spec.name = colon + 1; - PyObject *obType = _PyType_FromSpecWithBases(&new_spec, bases); + PyObject *obType = _PyType_FromSpecWithBasesHack(&new_spec, bases, meta); if (obType == nullptr) return nullptr; @@ -73,8 +129,8 @@ PyTypeObject *SbkType_FromSpec_BMDWB(PyType_Spec *spec, qual = dot + 1; } int mlen = qual - mod - 1; - Shiboken::AutoDecRef module(Shiboken::String::fromCString(mod, mlen)); - Shiboken::AutoDecRef qualname(Shiboken::String::fromCString(qual)); + AutoDecRef module(Shiboken::String::fromCString(mod, mlen)); + AutoDecRef qualname(Shiboken::String::fromCString(qual)); auto *type = reinterpret_cast<PyTypeObject *>(obType); @@ -98,9 +154,10 @@ PyTypeObject *SbkType_FromSpec_BMDWB(PyType_Spec *spec, // PyType_Ready too early. (at least in PyPy, which caused pretty long debugging.) auto *ht = reinterpret_cast<PyHeapTypeObject *>(type); ht->ht_qualname = qualname; - if (PyDict_SetItem(type->tp_dict, Shiboken::PyMagicName::qualname(), qualname)) + AutoDecRef tpDict(PepType_GetDict(type)); + if (PyDict_SetItem(tpDict.object(), Shiboken::PyMagicName::qualname(), qualname)) return nullptr; - if (PyDict_SetItem(type->tp_dict, Shiboken::PyMagicName::module(), module)) + if (PyDict_SetItem(tpDict.object(), Shiboken::PyMagicName::module(), module)) return nullptr; PyType_Ready(type); #else @@ -329,7 +386,7 @@ _PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) /// Here is the only change needed: Do not finalize type creation. // if (PyType_Ready(type) < 0) // goto fail; - type->tp_dict = PyDict_New(); + PepType_SetDict(type, PyDict_New()); /// This is not found in PyPy: // if (type->tp_dictoffset) { // res->ht_cached_keys = _PyDict_NewKeysForClass(); diff --git a/sources/shiboken6/libshiboken/sbkversion.h.in b/sources/shiboken6/libshiboken/sbkversion.h.in index 7f99abc3e..5c0b38fdb 100644 --- a/sources/shiboken6/libshiboken/sbkversion.h.in +++ b/sources/shiboken6/libshiboken/sbkversion.h.in @@ -10,8 +10,8 @@ #define SHIBOKEN_MICRO_VERSION @shiboken_MICRO_VERSION@ #define SHIBOKEN_RELEASE_LEVEL "final" #define SHIBOKEN_SERIAL 0 -#define PYTHON_VERSION_MAJOR @PYTHON_VERSION_MAJOR@ -#define PYTHON_VERSION_MINOR @PYTHON_VERSION_MINOR@ -#define PYTHON_VERSION_PATCH @PYTHON_VERSION_PATCH@ +#define PYTHON_VERSION_MAJOR @Python_VERSION_MAJOR@ +#define PYTHON_VERSION_MINOR @Python_VERSION_MINOR@ +#define PYTHON_VERSION_PATCH @Python_VERSION_PATCH@ #endif diff --git a/sources/shiboken6/libshiboken/sbkwindows.h b/sources/shiboken6/libshiboken/sbkwindows.h new file mode 100644 index 000000000..9e753fa5e --- /dev/null +++ b/sources/shiboken6/libshiboken/sbkwindows.h @@ -0,0 +1,17 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef SBKWINDOWS_H +#define SBKWINDOWS_H + +#ifdef _WIN32 +# ifndef NOMINMAX +# define NOMINMAX +# endif +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include <windows.h> +#endif + +#endif // SBKWINDOWS_H diff --git a/sources/shiboken6/libshiboken/shiboken.h b/sources/shiboken6/libshiboken/shiboken.h index 13a15e1f4..fcf777ae0 100644 --- a/sources/shiboken6/libshiboken/shiboken.h +++ b/sources/shiboken6/libshiboken/shiboken.h @@ -11,10 +11,10 @@ #include "gilstate.h" #include "threadstatesaver.h" #include "helper.h" +#include "pyobjectholder.h" #include "sbkarrayconverter.h" #include "sbkconverter.h" #include "sbkenum.h" -#include "sbkenum_p.h" // PYSIDE-1735: This is during the migration, only. #include "sbkerrors.h" #include "sbkmodule.h" #include "sbkstring.h" diff --git a/sources/shiboken6/libshiboken/signature.h b/sources/shiboken6/libshiboken/signature.h index 1cc100f3d..e0130b5a6 100644 --- a/sources/shiboken6/libshiboken/signature.h +++ b/sources/shiboken6/libshiboken/signature.h @@ -13,7 +13,6 @@ extern "C" LIBSHIBOKEN_API int InitSignatureStrings(PyTypeObject *, const char *[]); LIBSHIBOKEN_API void FinishSignatureInitialization(PyObject *, const char *[]); LIBSHIBOKEN_API void SetError_Argument(PyObject *, const char *, PyObject *); -LIBSHIBOKEN_API PyObject *Sbk_TypeGet___signature__(PyObject *, PyObject *); LIBSHIBOKEN_API PyObject *Sbk_TypeGet___doc__(PyObject *); LIBSHIBOKEN_API PyObject *GetFeatureDict(); diff --git a/sources/shiboken6/libshiboken/signature/signature.cpp b/sources/shiboken6/libshiboken/signature/signature.cpp index f83618779..3255cb56d 100644 --- a/sources/shiboken6/libshiboken/signature/signature.cpp +++ b/sources/shiboken6/libshiboken/signature/signature.cpp @@ -197,7 +197,7 @@ PyObject *GetSignature_Wrapper(PyObject *ob, PyObject *modifier) PyObject *props = PyDict_GetItem(dict, func_name); if (props == nullptr) { // handle `__init__` like the class itself - if (strcmp(String::toCString(func_name), "__init__") == 0) + if (PyUnicode_CompareWithASCIIString(func_name, "__init__") == 0) return GetSignature_TypeMod(objclass, modifier); Py_RETURN_NONE; } @@ -227,6 +227,8 @@ PyObject *GetSignature_TypeMod(PyObject *ob, PyObject *modifier) // The `modifier` argument is a string that is passed in from `loader.py`. // Configuration what the modifiers mean is completely in Python. // +// PYSIDE-2101: The __signature__ attribute is gone due to rlcompleter. +// PyObject *get_signature_intern(PyObject *ob, PyObject *modifier) { @@ -246,6 +248,9 @@ PyObject *get_signature_intern(PyObject *ob, PyObject *modifier) return pyside_tp_get___signature__(ob, modifier); if (Py_TYPE(ob) == &PyWrapperDescr_Type) return pyside_wd_get___signature__(ob, modifier); + // For classmethods we use the simple wrapper description implementation. + if (Py_TYPE(ob) == &PyClassMethodDescr_Type) + return pyside_wd_get___signature__(ob, modifier); return nullptr; } @@ -302,7 +307,7 @@ static PyObject *feature_import(PyObject * /* self */, PyObject *args, PyObject PyMethodDef signature_methods[] = { {"__feature_import__", (PyCFunction)feature_import, METH_VARARGS | METH_KEYWORDS, nullptr}, {"get_signature", (PyCFunction)get_signature, METH_VARARGS, - "get the __signature__, but pass an optional string parameter"}, + "get the signature, passing an optional string parameter"}, {nullptr, nullptr, 0, nullptr} }; @@ -385,9 +390,7 @@ PyObject *PySide_BuildSignatureProps(PyObject *type_key) #ifdef PYPY_VERSION static bool get_lldebug_flag() { - PyObject *sysmodule = PyImport_AddModule("sys"); - auto *dic = PyModule_GetDict(sysmodule); - dic = PyDict_GetItemString(dic, "pypy_translation_info"); + auto *dic = PySys_GetObject("pypy_translation_info"); int lldebug = PyObject_IsTrue(PyDict_GetItemString(dic, "translation.lldebug")); int lldebug0 = PyObject_IsTrue(PyDict_GetItemString(dic, "translation.lldebug0")); return lldebug || lldebug0; @@ -428,8 +431,6 @@ static int PySide_FinishSignatures(PyObject *module, const char *signatures[]) if (PyCFunction_Check(func)) if (PyDict_SetItem(pyside_globals->map_dict, func, module) < 0) return -1; - if (_finish_nested_classes(obdict) < 0) - return -1; // The finish_import function will not work the first time since phase 2 // was not yet run. But that is ok, because the first import is always for // the shiboken module (or a test module). @@ -451,10 +452,12 @@ static int PySide_FinishSignatures(PyObject *module, const char *signatures[]) int InitSignatureStrings(PyTypeObject *type, const char *signatures[]) { + // PYSIDE-2404: This function now also builds the mapping for static methods. + // It was one missing spot to let Lazy import work. init_shibokensupport_module(); auto *ob_type = reinterpret_cast<PyObject *>(type); int ret = PySide_BuildSignatureArgs(ob_type, signatures); - if (ret < 0) { + if (ret < 0 || _build_func_to_type(ob_type) < 0) { PyErr_Print(); PyErr_SetNone(PyExc_ImportError); } @@ -471,6 +474,8 @@ void FinishSignatureInitialization(PyObject *module, const char *signatures[]) * Still, it is not possible to call init phase 2 from here, * because the import is still running. Do it from Python! */ + init_shibokensupport_module(); + #ifndef PYPY_VERSION static const bool patch_types = true; #else @@ -534,7 +539,7 @@ static PyObject *adjustFuncName(const char *func_name) // Find the feature flags auto type = reinterpret_cast<PyTypeObject *>(obtype.object()); - auto dict = type->tp_dict; + AutoDecRef dict(PepType_GetDict(type)); int id = currentSelectId(type); id = id < 0 ? 0 : id; // if undefined, set to zero auto lower = id & 0x01; @@ -583,7 +588,9 @@ void SetError_Argument(PyObject *args, const char *func_name, PyObject *info) PyObject *e, *v, *t; // Note: These references are all borrowed. PyErr_Fetch(&e, &v, &t); + Py_DECREF(e); info = v; + Py_XDECREF(t); } // PYSIDE-1019: Modify the function name expression according to feature. AutoDecRef new_func_name(adjustFuncName(func_name)); @@ -614,14 +621,10 @@ void SetError_Argument(PyObject *args, const char *func_name, PyObject *info) * But the __doc__ attribute existed already by inheritance, and calling * PyType_Modified() is not supported. So we added the getsets explicitly * to the metatype. + * + * PYSIDE-2101: The __signature__ attribute is gone due to rlcompleter. */ -PyObject *Sbk_TypeGet___signature__(PyObject *ob, PyObject *modifier) -{ - init_shibokensupport_module(); - return pyside_tp_get___signature__(ob, modifier); -} - PyObject *Sbk_TypeGet___doc__(PyObject *ob) { init_shibokensupport_module(); diff --git a/sources/shiboken6/libshiboken/signature/signature_extend.cpp b/sources/shiboken6/libshiboken/signature/signature_extend.cpp index d571f90f3..7292f8216 100644 --- a/sources/shiboken6/libshiboken/signature/signature_extend.cpp +++ b/sources/shiboken6/libshiboken/signature/signature_extend.cpp @@ -70,8 +70,6 @@ PyObject *pyside_cf_get___signature__(PyObject *func, PyObject *modifier) PyObject *pyside_sm_get___signature__(PyObject *sm, PyObject *modifier) { AutoDecRef func(PyObject_GetAttr(sm, PyMagicName::func())); - if (Py_TYPE(func) == PepFunction_TypePtr) - return PyObject_GetAttr(func, PyMagicName::signature()); return _get_written_signature(GetSignature_Function, func, modifier); } @@ -79,7 +77,7 @@ PyObject *pyside_md_get___signature__(PyObject *ob_md, PyObject *modifier) { AutoDecRef func(name_key_to_func(ob_md)); if (func.object() == Py_None) - return Py_None; + Py_RETURN_NONE; if (func.isNull()) Py_FatalError("missing mapping in MethodDescriptor"); return pyside_cf_get___signature__(func, modifier); @@ -123,13 +121,15 @@ static PyObject *handle_doc(PyObject *ob, PyObject *old_descr) { AutoDecRef ob_type_mod(GetClassOrModOf(ob)); const char *name; - if (PyModule_Check(ob_type_mod.object())) + bool isModule = PyModule_Check(ob_type_mod.object()); + if (isModule) name = PyModule_GetName(ob_type_mod.object()); else name = reinterpret_cast<PyTypeObject *>(ob_type_mod.object())->tp_name; PyObject *res{}; - if (handle_doc_in_progress || name == nullptr || strncmp(name, "PySide6.", 8) != 0) { + if (handle_doc_in_progress || name == nullptr + || (isModule && strncmp(name, "PySide6.", 8) != 0)) { res = PyObject_CallMethodObjArgs(old_descr, PyMagicName::get(), ob, nullptr); } else { handle_doc_in_progress++; @@ -169,59 +169,29 @@ static PyObject *pyside_wd_get___doc__(PyObject *wd) return handle_doc(wd, old_wd_doc_descr); } -// the default setter for all objects -static int pyside_set___signature__(PyObject *op, PyObject *value) -{ - // By this additional check, this function refuses write access. - // We consider both nullptr and Py_None as not been written. - AutoDecRef has_val(get_signature_intern(op, nullptr)); - if (!(has_val.isNull() || has_val == Py_None)) { - PyErr_Format(PyExc_AttributeError, - "Attribute '__signature__' of '%.50s' object is not writable", - Py_TYPE(op)->tp_name); - return -1; - } - int ret = value == nullptr ? PyDict_DelItem(pyside_globals->value_dict, op) - : PyDict_SetItem(pyside_globals->value_dict, op, value); - Py_XINCREF(value); - return ret; -} - // PYSIDE-535: We cannot patch types easily in PyPy. // Let's use the `get_signature` function, instead. static PyGetSetDef new_PyCFunction_getsets[] = { {const_cast<char *>("__doc__"), reinterpret_cast<getter>(pyside_cf_get___doc__), nullptr, nullptr, nullptr}, - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(pyside_cf_get___signature__), - reinterpret_cast<setter>(pyside_set___signature__), - nullptr, nullptr}, {nullptr, nullptr, nullptr, nullptr, nullptr} }; static PyGetSetDef new_PyStaticMethod_getsets[] = { {const_cast<char *>("__doc__"), reinterpret_cast<getter>(pyside_sm_get___doc__), nullptr, nullptr, nullptr}, - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(pyside_sm_get___signature__), - reinterpret_cast<setter>(pyside_set___signature__), - nullptr, nullptr}, {nullptr, nullptr, nullptr, nullptr, nullptr} }; static PyGetSetDef new_PyMethodDescr_getsets[] = { {const_cast<char *>("__doc__"), reinterpret_cast<getter>(pyside_md_get___doc__), nullptr, nullptr, nullptr}, - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(pyside_md_get___signature__), - reinterpret_cast<setter>(pyside_set___signature__), - nullptr, nullptr}, {nullptr, nullptr, nullptr, nullptr, nullptr} }; static PyGetSetDef new_PyWrapperDescr_getsets[] = { {const_cast<char *>("__doc__"), reinterpret_cast<getter>(pyside_wd_get___doc__), nullptr, nullptr, nullptr}, - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(pyside_wd_get___signature__), - reinterpret_cast<setter>(pyside_set___signature__), - nullptr, nullptr}, {nullptr, nullptr, nullptr, nullptr, nullptr} }; diff --git a/sources/shiboken6/libshiboken/signature/signature_globals.cpp b/sources/shiboken6/libshiboken/signature/signature_globals.cpp index d0ea86fc6..3a79a12d5 100644 --- a/sources/shiboken6/libshiboken/signature/signature_globals.cpp +++ b/sources/shiboken6/libshiboken/signature/signature_globals.cpp @@ -88,11 +88,9 @@ static safe_globals_struc *init_phase_1() * Due to MSVC's limitation to 64k strings, we needed to assemble pieces. */ auto **block_ptr = reinterpret_cast<const char **>(PySide_CompressedSignaturePackage); - int npieces = 0; PyObject *piece{}; AutoDecRef zipped_string_sequence(PyList_New(0)); for (; **block_ptr != 0; ++block_ptr) { - npieces++; // we avoid the string/unicode dilemma by not using PyString_XXX: piece = Py_BuildValue("s", *block_ptr); if (piece == nullptr || PyList_Append(zipped_string_sequence, piece) < 0) @@ -189,6 +187,16 @@ static int init_phase_2(safe_globals_struc *p, PyMethodDef *methods) p->feature_imported_func = PyObject_GetAttrString(loader, "feature_imported"); if (p->feature_imported_func == nullptr) break; + + // We call stuff like the feature initialization late, + // after all the function pointers are in place. + PyObject *post_init_func = PyObject_GetAttrString(loader, "post_init"); + if (post_init_func == nullptr) + break; + PyObject *ret = PyObject_CallFunctionObjArgs(post_init_func, nullptr); + if (ret == nullptr) + break; + return 0; } while (0); @@ -201,12 +209,12 @@ static int init_phase_2(safe_globals_struc *p, PyMethodDef *methods) #ifndef _WIN32 //////////////////////////////////////////////////////////////////////////// // a stack trace for linux-like platforms -#include <stdio.h> +#include <cstdio> #if defined(__GLIBC__) # include <execinfo.h> #endif #include <signal.h> -#include <stdlib.h> +#include <cstdlib> #include <unistd.h> static void handler(int sig) { @@ -219,7 +227,7 @@ static void handler(int sig) { // print out all the frames to stderr #endif - fprintf(stderr, "Error: signal %d:\n", sig); + std::fprintf(stderr, "Error: signal %d:\n", sig); #if defined(__GLIBC__) backtrace_symbols_fd(array, size, STDERR_FILENO); #endif @@ -229,7 +237,7 @@ static void handler(int sig) { //////////////////////////////////////////////////////////////////////////// #endif // _WIN32 -safe_globals pyside_globals = nullptr; +safe_globals_struc *pyside_globals = nullptr; void init_shibokensupport_module(void) { diff --git a/sources/shiboken6/libshiboken/signature/signature_helper.cpp b/sources/shiboken6/libshiboken/signature/signature_helper.cpp index ef0c021d5..cf84cfa13 100644 --- a/sources/shiboken6/libshiboken/signature/signature_helper.cpp +++ b/sources/shiboken6/libshiboken/signature/signature_helper.cpp @@ -51,10 +51,13 @@ int add_more_getsets(PyTypeObject *type, PyGetSetDef *gsp, PyObject **doc_descr) /* * This function is used to assign a new `__signature__` attribute, * and also to override a `__doc__` or `__name__` attribute. + * + * PYSIDE-2101: The __signature__ attribute is gone due to rlcompleter. */ assert(PyType_Check(type)); PyType_Ready(type); - PyObject *dict = type->tp_dict; + AutoDecRef tpDict(PepType_GetDict(type)); + auto *dict = tpDict.object(); for (; gsp->name != nullptr; gsp++) { PyObject *have_descr = PyDict_GetItemString(dict, gsp->name); if (have_descr != nullptr) { @@ -291,7 +294,7 @@ PyObject *_address_to_stringlist(PyObject *numkey) return res_list; } -static int _build_func_to_type(PyObject *obtype) +int _build_func_to_type(PyObject *obtype) { /* * There is no general way to directly get the type of a static method. @@ -307,7 +310,17 @@ static int _build_func_to_type(PyObject *obtype) * We also check for hidden methods, see below. */ auto *type = reinterpret_cast<PyTypeObject *>(obtype); - PyObject *dict = type->tp_dict; + AutoDecRef tpDict(PepType_GetDict(type)); + auto *dict = tpDict.object(); + + // PYSIDE-2404: Get the original dict for late initialization. + // The dict might have been switched before signature init. + static const auto *pyTypeType_tp_dict = PepType_GetDict(&PyType_Type); + if (Py_TYPE(dict) != Py_TYPE(pyTypeType_tp_dict)) { + tpDict.reset(PyObject_GetAttr(dict, PyName::orig_dict())); + dict = tpDict.object(); + } + PyMethodDef *meth = type->tp_methods; if (meth == nullptr) @@ -373,26 +386,4 @@ static int _build_func_to_type(PyObject *obtype) return 0; } -int _finish_nested_classes(PyObject *obdict) -{ - PyObject *key, *value, *obtype; - PyTypeObject *subtype; - Py_ssize_t pos = 0; - - if (obdict == nullptr) - return -1; - while (PyDict_Next(obdict, &pos, &key, &value)) { - if (PyType_Check(value)) { - obtype = value; - if (_build_func_to_type(obtype) < 0) - return -1; - // now continue with nested cases - subtype = reinterpret_cast<PyTypeObject *>(obtype); - if (_finish_nested_classes(subtype->tp_dict) < 0) - return -1; - } - } - return 0; -} - } // extern "C" diff --git a/sources/shiboken6/libshiboken/signature_p.h b/sources/shiboken6/libshiboken/signature_p.h index 7ea03877a..d0c4ee537 100644 --- a/sources/shiboken6/libshiboken/signature_p.h +++ b/sources/shiboken6/libshiboken/signature_p.h @@ -10,7 +10,7 @@ extern "C" { // signature_globals.cpp -typedef struct safe_globals_struc { +struct safe_globals_struc { // init part 1: get arg_dict PyObject *helper_module; PyObject *arg_dict; @@ -25,9 +25,9 @@ typedef struct safe_globals_struc { PyObject *finish_import_func; PyObject *feature_import_func; PyObject *feature_imported_func; -} safe_globals_struc, *safe_globals; +}; -extern safe_globals pyside_globals; +extern safe_globals_struc *pyside_globals; extern PyMethodDef signature_methods[]; void init_shibokensupport_module(void); @@ -63,6 +63,7 @@ PyObject *_get_class_of_cf(PyObject *ob_cf); PyObject *_get_class_of_sm(PyObject *ob_sm); PyObject *_get_class_of_descr(PyObject *ob); PyObject *_address_to_stringlist(PyObject *numkey); +int _build_func_to_type(PyObject *obtype); int _finish_nested_classes(PyObject *dict); #ifdef PYPY_VERSION diff --git a/sources/shiboken6/libshiboken/voidptr.cpp b/sources/shiboken6/libshiboken/voidptr.cpp index e768ff608..8bb3f6ac8 100644 --- a/sources/shiboken6/libshiboken/voidptr.cpp +++ b/sources/shiboken6/libshiboken/voidptr.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "voidptr.h" +#include "pep384ext.h" #include "sbkconverter.h" #include "basewrapper.h" #include "basewrapper_p.h" @@ -10,12 +11,12 @@ extern "C" { // Void pointer object definition. -typedef struct { +struct SbkVoidPtrObject { PyObject_HEAD void *cptr; Py_ssize_t size; bool isWritable; -} SbkVoidPtrObject; +}; PyObject *SbkVoidPtrObject_new(PyTypeObject *type, PyObject * /* args */, PyObject * /* kwds */) { @@ -24,8 +25,7 @@ PyObject *SbkVoidPtrObject_new(PyTypeObject *type, PyObject * /* args */, PyObje // like this, actual call forgotten: // SbkVoidPtrObject *self = // reinterpret_cast<SbkVoidPtrObject *>(type->tp_alloc); - PyObject *ob = type->tp_alloc(type, 0); - auto *self = reinterpret_cast<SbkVoidPtrObject *>(ob); + auto *self = PepExt_TypeCallAlloc<SbkVoidPtrObject>(type, 0); if (self != nullptr) { self->cptr = nullptr; @@ -156,10 +156,9 @@ PyObject *SbkVoidPtrObject_int(PyObject *v) PyObject *toBytes(PyObject *self, PyObject * /* args */) { auto *sbkObject = reinterpret_cast<SbkVoidPtrObject *>(self); - if (sbkObject->size < 0) { - PyErr_SetString(PyExc_IndexError, "VoidPtr does not have a size set."); - return nullptr; - } + if (sbkObject->size < 0) + return PyErr_Format(PyExc_IndexError, "VoidPtr does not have a size set."); + PyObject *bytes = PyBytes_FromStringAndSize(reinterpret_cast<const char *>(sbkObject->cptr), sbkObject->size); Py_XINCREF(bytes); @@ -256,38 +255,42 @@ static PyBufferProcs SbkVoidPtrObjectBufferProc = { (releasebufferproc)nullptr // bf_releasebuffer }; -// Void pointer type definition. -static PyType_Slot SbkVoidPtrType_slots[] = { - {Py_tp_repr, reinterpret_cast<void *>(SbkVoidPtrObject_repr)}, - {Py_nb_int, reinterpret_cast<void *>(SbkVoidPtrObject_int)}, - {Py_sq_length, reinterpret_cast<void *>(SbkVoidPtrObject_length)}, - {Py_tp_str, reinterpret_cast<void *>(SbkVoidPtrObject_str)}, - {Py_tp_richcompare, reinterpret_cast<void *>(SbkVoidPtrObject_richcmp)}, - {Py_tp_init, reinterpret_cast<void *>(SbkVoidPtrObject_init)}, - {Py_tp_new, reinterpret_cast<void *>(SbkVoidPtrObject_new)}, - {Py_tp_dealloc, reinterpret_cast<void *>(Sbk_object_dealloc)}, - {Py_tp_methods, reinterpret_cast<void *>(SbkVoidPtrObject_methods)}, - {0, nullptr} -}; -static PyType_Spec SbkVoidPtrType_spec = { - "2:shiboken6.Shiboken.VoidPtr", - sizeof(SbkVoidPtrObject), - 0, - Py_TPFLAGS_DEFAULT, - SbkVoidPtrType_slots, -}; - - +static PyTypeObject *createVoidPtrType() +{ + PyType_Slot SbkVoidPtrType_slots[] = { + {Py_tp_repr, reinterpret_cast<void *>(SbkVoidPtrObject_repr)}, + {Py_nb_int, reinterpret_cast<void *>(SbkVoidPtrObject_int)}, + {Py_sq_length, reinterpret_cast<void *>(SbkVoidPtrObject_length)}, + {Py_tp_str, reinterpret_cast<void *>(SbkVoidPtrObject_str)}, + {Py_tp_richcompare, reinterpret_cast<void *>(SbkVoidPtrObject_richcmp)}, + {Py_tp_init, reinterpret_cast<void *>(SbkVoidPtrObject_init)}, + {Py_tp_new, reinterpret_cast<void *>(SbkVoidPtrObject_new)}, + {Py_tp_dealloc, reinterpret_cast<void *>(Sbk_object_dealloc)}, + {Py_tp_methods, reinterpret_cast<void *>(SbkVoidPtrObject_methods)}, + {0, nullptr} + }; + + PyType_Spec SbkVoidPtrType_spec = { + "2:shiboken6.Shiboken.VoidPtr", + sizeof(SbkVoidPtrObject), + 0, + Py_TPFLAGS_DEFAULT, + SbkVoidPtrType_slots, + }; + + return SbkType_FromSpec_BMDWB(&SbkVoidPtrType_spec, + nullptr, nullptr, 0, 0, + &SbkVoidPtrObjectBufferProc); } PyTypeObject *SbkVoidPtr_TypeF(void) { - static PyTypeObject *type = SbkType_FromSpec_BMDWB(&SbkVoidPtrType_spec, - nullptr, nullptr, 0, 0, - &SbkVoidPtrObjectBufferProc); + static auto *type = createVoidPtrType(); return type; } +} // extern "C" + namespace VoidPtr { static int voidPointerInitialized = false; |