diff options
Diffstat (limited to 'sources/shiboken6/libshiboken')
63 files changed, 3563 insertions, 3177 deletions
diff --git a/sources/shiboken6/libshiboken/CMakeLists.txt b/sources/shiboken6/libshiboken/CMakeLists.txt index eaed2ef72..b5bbb498a 100644 --- a/sources/shiboken6/libshiboken/CMakeLists.txt +++ b/sources/shiboken6/libshiboken/CMakeLists.txt @@ -1,3 +1,6 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + project(libshiboken) option(ENABLE_VERSION_SUFFIX "Used to use current version in suffix to generated files. This is used to allow multiples versions installed simultaneous." FALSE) @@ -27,7 +30,7 @@ if(SHIBOKEN_IS_CROSS_BUILD) set(host_python_path "${QFP_PYTHON_HOST_PATH}") set(use_pyc_in_embedding FALSE) else() - set(host_python_path "${PYTHON_EXECUTABLE}") + set(host_python_path "${Python_EXECUTABLE}") if(PYTHON_LIMITED_API) set(use_pyc_in_embedding FALSE) else() @@ -55,39 +58,49 @@ set(libshiboken_VERSION "${libshiboken_MAJOR_VERSION}.${libshiboken_MINOR_VERSIO set(libshiboken_SOVERSION "${shiboken6_library_so_version}") set(libshiboken_SRC -basewrapper.cpp -debugfreehook.cpp -gilstate.cpp -helper.cpp -sbkarrayconverter.cpp -sbkcontainer.cpp -sbkconverter.cpp -sbkenum.cpp -sbkerrors.cpp -sbkfeature_base.cpp -sbkmodule.cpp -sbknumpy.cpp -sbkcppstring.cpp -sbkstring.cpp -sbkstaticstrings.cpp -sbktypefactory.cpp -bindingmanager.cpp -threadstatesaver.cpp -shibokenbuffer.cpp -pep384impl.cpp -voidptr.cpp -bufferprocs_py37.cpp +autodecref.h +basewrapper.cpp basewrapper.h basewrapper_p.h +bindingmanager.cpp bindingmanager.h +bufferprocs_py37.cpp bufferprocs_py37.h +debugfreehook.cpp debugfreehook.h +gilstate.cpp gilstate.h +helper.cpp helper.h +pep384impl.cpp pep384impl.h +pyobjectholder.h +sbkarrayconverter.cpp sbkarrayconverter.h sbkarrayconverter_p.h +sbkcontainer.cpp sbkcontainer.h +sbkconverter.cpp sbkconverter.h sbkconverter_p.h +sbkcppstring.cpp sbkcppstring.h sbkcpptonumpy.h +sbkenum.cpp sbkenum.h +sbkerrors.cpp sbkerrors.h +sbkfeature_base.cpp sbkfeature_base.h +sbkmodule.cpp sbkmodule.h +sbknumpy.cpp sbknumpycheck.h +sbknumpyview.h +sbkpython.h +sbksmartpointer.cpp sbksmartpointer.h +sbkstaticstrings.cpp sbkstaticstrings.h sbkstaticstrings_p.h +sbkstring.cpp sbkstring.h +sbktypefactory.cpp sbktypefactory.h +sbkwindows.h +shiboken.h +shibokenbuffer.cpp shibokenbuffer.h +shibokenmacros.h +threadstatesaver.cpp threadstatesaver.h +voidptr.cpp voidptr.h embed/signature_bootstrap_inc.h embed/signature_inc.h -signature/signature.cpp +signature/signature.cpp signature.h signature_p.h signature/signature_globals.cpp signature/signature_extend.cpp signature/signature_helper.cpp ) -set(APIEXTRACTOR_EXTRA_INCLUDES ${APIEXTRACTOR_EXTRA_INCLUDES} ${LIBXSLT_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR}) +# This is needed to let the header obey a variable in "pep384impl.h". +# Note: This must be set on the cpp file! +set_property(SOURCE "pep384impl.cpp" PROPERTY SKIP_UNITY_BUILD_INCLUSION ON) add_library(libshiboken SHARED ${libshiboken_SRC}) add_library(Shiboken6::libshiboken ALIAS libshiboken) @@ -146,11 +159,12 @@ install(FILES bindingmanager.h gilstate.h helper.h + pyobjectholder.h sbkarrayconverter.h sbkcontainer.h sbkconverter.h + sbkcpptonumpy.h sbkenum.h - sbkenum_p.h sbkerrors.h sbkfeature_base.h sbkmodule.h @@ -158,6 +172,7 @@ install(FILES sbknumpyview.h sbkstring.h sbkcppstring.h + sbksmartpointer.h sbkstaticstrings.h sbktypefactory.h shiboken.h @@ -165,7 +180,9 @@ install(FILES threadstatesaver.h shibokenbuffer.h sbkpython.h + sbkwindows.h pep384impl.h + pep384ext.h voidptr.h bufferprocs_py37.h "${CMAKE_CURRENT_BINARY_DIR}/sbkversion.h" @@ -179,4 +196,4 @@ install(TARGETS libshiboken EXPORT Shiboken6Targets ARCHIVE DESTINATION "${LIB_INSTALL_DIR}" RUNTIME DESTINATION bin) install(EXPORT Shiboken6Targets NAMESPACE Shiboken6:: - DESTINATION ${LIB_INSTALL_DIR}/cmake/Shiboken6-${shiboken6_VERSION}) + DESTINATION ${LIB_INSTALL_DIR}/cmake/Shiboken6) diff --git a/sources/shiboken6/libshiboken/autodecref.h b/sources/shiboken6/libshiboken/autodecref.h index d2b660676..62a8584e1 100644 --- a/sources/shiboken6/libshiboken/autodecref.h +++ b/sources/shiboken6/libshiboken/autodecref.h @@ -5,7 +5,8 @@ #define AUTODECREF_H #include "sbkpython.h" -#include "basewrapper.h" + +#include <utility> struct SbkObject; namespace Shiboken @@ -14,29 +15,27 @@ namespace Shiboken /** * AutoDecRef holds a PyObject pointer and decrement its reference counter when destroyed. */ -struct LIBSHIBOKEN_API AutoDecRef +struct AutoDecRef { public: AutoDecRef(const AutoDecRef &) = delete; - AutoDecRef(AutoDecRef &&) = delete; + AutoDecRef(AutoDecRef &&o) noexcept : m_pyObj{std::exchange(o.m_pyObj, nullptr)} {} AutoDecRef &operator=(const AutoDecRef &) = delete; - AutoDecRef &operator=(AutoDecRef &&) = delete; + AutoDecRef &operator=(AutoDecRef &&o) noexcept + { + m_pyObj = std::exchange(o.m_pyObj, nullptr); + return *this; + } - /** - * AutoDecRef constructor. - * \param pyobj A borrowed reference to a Python object - */ - explicit AutoDecRef(PyObject *pyObj) : m_pyObj(pyObj) {} - /** - * AutoDecRef constructor. - * \param pyobj A borrowed reference to a Python object - */ - explicit AutoDecRef(SbkObject *pyObj) : m_pyObj(reinterpret_cast<PyObject *>(pyObj)) {} - /** - * AutoDecref constructor. - * To be used later with reset(): - */ - AutoDecRef() : m_pyObj(nullptr) {} + /// AutoDecRef constructor. + /// \param pyobj A borrowed reference to a Python object + explicit AutoDecRef(PyObject *pyObj) noexcept : m_pyObj(pyObj) {} + /// AutoDecRef constructor. + /// \param pyobj A borrowed reference to a wrapped Python object + explicit AutoDecRef(SbkObject *pyObj) noexcept : m_pyObj(reinterpret_cast<PyObject *>(pyObj)) {} + /// AutoDecref default constructor. + /// To be used later with reset(): + AutoDecRef() noexcept = default; /// Decref the borrowed python reference ~AutoDecRef() @@ -44,18 +43,19 @@ public: Py_XDECREF(m_pyObj); } - inline bool isNull() const { return m_pyObj == nullptr; } + [[nodiscard]] bool isNull() const { return m_pyObj == nullptr; } /// Returns the pointer of the Python object being held. - inline PyObject *object() { return m_pyObj; } - inline operator PyObject *() { return m_pyObj; } + [[nodiscard]] PyObject *object() const { return m_pyObj; } + [[nodiscard]] operator PyObject *() const { return m_pyObj; } #ifndef Py_LIMITED_API - inline operator PyTupleObject *() { return reinterpret_cast<PyTupleObject *>(m_pyObj); } + [[deprecated]] inline operator PyTupleObject *() + { return reinterpret_cast<PyTupleObject *>(m_pyObj); } #endif inline operator bool() const { return m_pyObj != nullptr; } inline PyObject *operator->() { return m_pyObj; } template<typename T> - T cast() + [[deprecated]] T cast() { return reinterpret_cast<T>(m_pyObj); } @@ -79,10 +79,9 @@ public: } private: - PyObject *m_pyObj; + PyObject *m_pyObj = nullptr; }; } // namespace Shiboken #endif // AUTODECREF_H - diff --git a/sources/shiboken6/libshiboken/basewrapper.cpp b/sources/shiboken6/libshiboken/basewrapper.cpp index f1622c30b..1ac65c00c 100644 --- a/sources/shiboken6/libshiboken/basewrapper.cpp +++ b/sources/shiboken6/libshiboken/basewrapper.cpp @@ -5,9 +5,12 @@ #include "basewrapper_p.h" #include "bindingmanager.h" #include "helper.h" +#include "pep384ext.h" #include "sbkconverter.h" #include "sbkenum.h" +#include "sbkerrors.h" #include "sbkfeature_base.h" +#include "sbkmodule.h" #include "sbkstring.h" #include "sbkstaticstrings.h" #include "sbkstaticstrings_p.h" @@ -21,6 +24,7 @@ #include <algorithm> #include "threadstatesaver.h" #include "signature.h" +#include "signature_p.h" #include "voidptr.h" #include <iostream> @@ -33,7 +37,73 @@ namespace { void _destroyParentInfo(SbkObject *obj, bool keepReference); } -static void callDestructor(const Shiboken::DtorAccumulatorVisitor::DestructorEntries &dts) +namespace Shiboken +{ +// Walk through the first level of non-user-type Sbk base classes relevant for +// C++ object allocation. Return true from the predicate to terminate. +template <class Predicate> +bool walkThroughBases(PyTypeObject *currentType, Predicate predicate) +{ + PyObject *bases = currentType->tp_bases; + const Py_ssize_t numBases = PyTuple_Size(bases); + bool result = false; + for (Py_ssize_t i = 0; !result && i < numBases; ++i) { + auto type = reinterpret_cast<PyTypeObject *>(PyTuple_GetItem(bases, i)); + if (PyType_IsSubtype(type, SbkObject_TypeF()) != 0) { + result = PepType_SOTP(type)->is_user_type + ? walkThroughBases(type, predicate) : predicate(type); + } + } + return result; +} + +int getTypeIndexOnHierarchy(PyTypeObject *baseType, PyTypeObject *desiredType) +{ + int index = -1; + walkThroughBases(baseType, [&index, desiredType](PyTypeObject *node) { + ++index; + return PyType_IsSubtype(node, desiredType) != 0; + }); + return index; +} + +int getNumberOfCppBaseClasses(PyTypeObject *baseType) +{ + int count = 0; + walkThroughBases(baseType, [&count](PyTypeObject *) { + ++count; + return false; + }); + return count; +} + +std::vector<PyTypeObject *> getCppBaseClasses(PyTypeObject *baseType) +{ + std::vector<PyTypeObject *> cppBaseClasses; + walkThroughBases(baseType, [&cppBaseClasses](PyTypeObject *node) { + cppBaseClasses.push_back(node); + return false; + }); + return cppBaseClasses; +} + +using DestructorEntries = std::vector<DestructorEntry>; + +DestructorEntries getDestructorEntries(SbkObject *o) +{ + DestructorEntries result; + void **cptrs = o->d->cptr; + walkThroughBases(Py_TYPE(o), [&result, cptrs](PyTypeObject *node) { + auto *sotp = PepType_SOTP(node); + auto index = result.size(); + result.push_back(DestructorEntry{sotp->cpp_dtor, + cptrs[index]}); + return false; + }); + return result; +} + +static void callDestructor(const DestructorEntries &dts) { for (const auto &e : dts) { Shiboken::ThreadStateSaver threadSaver; @@ -42,6 +112,8 @@ static void callDestructor(const Shiboken::DtorAccumulatorVisitor::DestructorEnt } } +} // namespace Shiboken + extern "C" { @@ -53,7 +125,7 @@ void Sbk_object_dealloc(PyObject *self) // This was not needed before Python 3.8 (Python issue 35810) Py_DECREF(Py_TYPE(self)); } - Py_TYPE(self)->tp_free(self); + PepExt_TypeCallFree(self); } static void SbkObjectType_tp_dealloc(PyTypeObject *pyType); @@ -70,6 +142,7 @@ void setDestroyQApplication(DestroyQAppHook func) // PYSIDE-535: Use the C API in PyPy instead of `op->ob_dict`, directly LIBSHIBOKEN_API PyObject *SbkObject_GetDict_NoRef(PyObject *op) { + assert(Shiboken::Object::checkType(op)); #ifdef PYPY_VERSION Shiboken::GilState state; auto *ret = PyObject_GenericGetDict(op, nullptr); @@ -103,20 +176,18 @@ check_set_special_type_attr(PyTypeObject *type, PyObject *value, const char *nam // PYSIDE-1177: Add a setter to allow setting type doc. static int -type_set_doc(PyTypeObject *type, PyObject *value, void *context) +type_set_doc(PyTypeObject *type, PyObject *value, void * /* context */) { if (!check_set_special_type_attr(type, value, "__doc__")) return -1; PyType_Modified(type); - return PyDict_SetItem(type->tp_dict, Shiboken::PyMagicName::doc(), value); + Shiboken::AutoDecRef tpDict(PepType_GetDict(type)); + return PyDict_SetItem(tpDict.object(), Shiboken::PyMagicName::doc(), value); } // PYSIDE-908: The function PyType_Modified does not work in PySide, so we need to -// explicitly pass __doc__. For __signature__ it _did_ actually work, because -// it was not existing before. We add them both for clarity. +// explicitly pass __doc__. static PyGetSetDef SbkObjectType_tp_getset[] = { - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(Sbk_TypeGet___signature__), - nullptr, nullptr, nullptr}, {const_cast<char *>("__doc__"), reinterpret_cast<getter>(Sbk_TypeGet___doc__), reinterpret_cast<setter>(type_set_doc), nullptr, nullptr}, {const_cast<char *>("__dict__"), reinterpret_cast<getter>(Sbk_TypeGet___dict__), @@ -124,31 +195,83 @@ static PyGetSetDef SbkObjectType_tp_getset[] = { {nullptr, nullptr, nullptr, nullptr, nullptr} // Sentinel }; -static PyType_Slot SbkObjectType_Type_slots[] = { - {Py_tp_dealloc, reinterpret_cast<void *>(SbkObjectType_tp_dealloc)}, - {Py_tp_getattro, reinterpret_cast<void *>(mangled_type_getattro)}, - {Py_tp_base, static_cast<void *>(&PyType_Type)}, - {Py_tp_alloc, reinterpret_cast<void *>(PyType_GenericAlloc)}, - {Py_tp_new, reinterpret_cast<void *>(SbkObjectType_tp_new)}, - {Py_tp_free, reinterpret_cast<void *>(PyObject_GC_Del)}, - {Py_tp_getset, reinterpret_cast<void *>(SbkObjectType_tp_getset)}, - {0, nullptr} -}; - -// PYSIDE-535: The tp_itemsize field is inherited and does not need to be set. -// In PyPy, it _must_ not be set, because it would have the meaning that a -// `__len__` field must be defined. Not doing so creates a hard-to-find crash. -static PyType_Spec SbkObjectType_Type_spec = { - "1:Shiboken.ObjectType", - 0, - 0, // sizeof(PyMemberDef), not for PyPy without a __len__ defined - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, - SbkObjectType_Type_slots, -}; +static PyTypeObject *createObjectTypeType() +{ + // PYSIDE-2676: When using the new type extension, we need to use an + // extra meta type that provides the extra size. + // This is a hairy part of Python 3.12 . + // + // The problem here is that we use the type extension both in types + // and also in meta types. This was invisible with extender dicts. + // Please study carefully: + // https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize + + PyType_Slot SbkObjectTypeMeta_Type_slots[] = { + {Py_tp_base, static_cast<void *>(&PyType_Type)}, + {Py_tp_alloc, reinterpret_cast<void *>(PyType_GenericAlloc)}, + {0, nullptr} + }; + + PyType_Spec SbkObjectTypeMeta_Type_spec = { + "1:Shiboken.ObjectTypeMeta", + -long(sizeof(SbkObjectTypePrivate)), + 0, // sizeof(PyMemberDef), not for PyPy without a __len__ defined + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_TYPE_SUBCLASS, + SbkObjectTypeMeta_Type_slots, + }; + + auto specMeta = &SbkObjectTypeMeta_Type_spec; + + PyType_Slot SbkObjectType_Type_slots[] = { + {Py_tp_dealloc, reinterpret_cast<void *>(SbkObjectType_tp_dealloc)}, + {Py_tp_getattro, reinterpret_cast<void *>(mangled_type_getattro)}, + {Py_tp_base, static_cast<void *>(&PyType_Type)}, + {Py_tp_alloc, reinterpret_cast<void *>(PyType_GenericAlloc)}, + {Py_tp_new, reinterpret_cast<void *>(SbkObjectType_tp_new)}, + {Py_tp_free, reinterpret_cast<void *>(PyObject_GC_Del)}, + {Py_tp_getset, reinterpret_cast<void *>(SbkObjectType_tp_getset)}, + {0, nullptr} + }; + + // PYSIDE-535: The tp_itemsize field is inherited and does not need to be set. + // In PyPy, it _must_ not be set, because it would have the meanin + // that a `__len__` field must be defined. Not doing so creates + // a hard-to-find crash. + // + // PYSIDE-2230: In Python < 3.12, the decision which base class should create + // the instance is arbitrarily drawn by the size of the type. + // Ignoring this creates a bug in the new version of bug_825 that + // selects the wrong metatype. + // + PyType_Spec SbkObjectType_Type_spec = { + "1:Shiboken.ObjectType", + static_cast<int>(PyType_Type.tp_basicsize) + 1, // see above + 0, // sizeof(PyMemberDef), not for PyPy without a __len__ defined + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_TYPE_SUBCLASS, + SbkObjectType_Type_slots, + }; + + PyType_Spec SbkObjectType_Type_spec_312 = { + "1:Shiboken.ObjectType", + -long(sizeof(SbkObjectTypePrivate)), + 0, // sizeof(PyMemberDef), not for PyPy without a __len__ defined + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_TYPE_SUBCLASS, + SbkObjectType_Type_slots, + }; + + if (_PepRuntimeVersion() >= 0x030C00) { + auto *meta = SbkType_FromSpec(specMeta); + auto spec = &SbkObjectType_Type_spec_312; + return SbkType_FromSpecWithMeta(spec, meta); + } + + auto spec = &SbkObjectType_Type_spec; + return SbkType_FromSpec(spec); +} PyTypeObject *SbkObjectType_TypeF(void) { - static auto *type = SbkType_FromSpec(&SbkObjectType_Type_spec); + static auto *type = createObjectTypeType(); return type; } @@ -185,10 +308,8 @@ static int SbkObject_tp_traverse(PyObject *self, visitproc visit, void *arg) if (sbkSelf->ob_dict) Py_VISIT(sbkSelf->ob_dict); -#if PY_VERSION_HEX >= 0x03090000 // This was not needed before Python 3.9 (Python issue 35810 and 40217) Py_VISIT(Py_TYPE(self)); -#endif return 0; } @@ -208,40 +329,53 @@ static int SbkObject_tp_clear(PyObject *self) return 0; } -static PyType_Slot SbkObject_Type_slots[] = { - {Py_tp_getattro, reinterpret_cast<void *>(SbkObject_GenericGetAttr)}, - {Py_tp_setattro, reinterpret_cast<void *>(SbkObject_GenericSetAttr)}, - {Py_tp_dealloc, reinterpret_cast<void *>(SbkDeallocWrapperWithPrivateDtor)}, - {Py_tp_traverse, reinterpret_cast<void *>(SbkObject_tp_traverse)}, - {Py_tp_clear, reinterpret_cast<void *>(SbkObject_tp_clear)}, - // unsupported: {Py_tp_weaklistoffset, (void *)offsetof(SbkObject, weakreflist)}, - {Py_tp_getset, reinterpret_cast<void *>(SbkObject_tp_getset)}, - // unsupported: {Py_tp_dictoffset, (void *)offsetof(SbkObject, ob_dict)}, - {0, nullptr} -}; -static PyType_Spec SbkObject_Type_spec = { - "1:Shiboken.Object", - sizeof(SbkObject), - 0, - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, - SbkObject_Type_slots, -}; - -static const char *SbkObject_SignatureStrings[] = { - "Shiboken.Object(self)", - nullptr}; // Sentinel +static PyTypeObject *createObjectType() +{ + PyType_Slot SbkObject_Type_slots[] = { + {Py_tp_getattro, reinterpret_cast<void *>(SbkObject_GenericGetAttr)}, + {Py_tp_setattro, reinterpret_cast<void *>(SbkObject_GenericSetAttr)}, + {Py_tp_dealloc, reinterpret_cast<void *>(SbkDeallocWrapperWithPrivateDtor)}, + {Py_tp_traverse, reinterpret_cast<void *>(SbkObject_tp_traverse)}, + {Py_tp_clear, reinterpret_cast<void *>(SbkObject_tp_clear)}, + // unsupported: {Py_tp_weaklistoffset, (void *)offsetof(SbkObject, weakreflist)}, + {Py_tp_getset, reinterpret_cast<void *>(SbkObject_tp_getset)}, + // unsupported: {Py_tp_dictoffset, (void *)offsetof(SbkObject, ob_dict)}, + {0, nullptr} + }; + + PyType_Spec SbkObject_Type_spec = { + "1:Shiboken.Object", + sizeof(SbkObject), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + SbkObject_Type_slots, + }; + + // PYSIDE-2230: When creating this type, we cannot easily handle the metaclass. + // In versions < Python 3.12, the metaclass can only be set + // indirectly by a base which has that metaclass. + // But before 3.12 is the minimum version, we cannot use the new + // function, although we would need this for 3.12 :-D + // We do a special patching here that is triggered through Py_None. + auto *type = SbkType_FromSpec_BMDWB(&SbkObject_Type_spec, + Py_None, // bases, spectial flag! + SbkObjectType_TypeF(), + offsetof(SbkObject, ob_dict), + offsetof(SbkObject, weakreflist), + nullptr); // bufferprocs + return type; +} PyTypeObject *SbkObject_TypeF(void) { - static auto *type = SbkType_FromSpec_BMDWB(&SbkObject_Type_spec, - nullptr, // bases - SbkObjectType_TypeF(), - offsetof(SbkObject, ob_dict), - offsetof(SbkObject, weakreflist), - nullptr); // bufferprocs + static auto *type = createObjectType(); // bufferprocs return type; } +static const char *SbkObject_SignatureStrings[] = { + "Shiboken.Object(self)", + nullptr}; // Sentinel + static int mainThreadDeletionHandler(void *) { if (Py_IsInitialized()) @@ -304,9 +438,8 @@ static void SbkDeallocWrapperCommon(PyObject *pyObj, bool canDelete) if (sotp->delete_in_main_thread && Shiboken::currentThreadId() != Shiboken::mainThreadId()) { auto &bindingManager = Shiboken::BindingManager::instance(); if (sotp->is_multicpp) { - Shiboken::DtorAccumulatorVisitor visitor(sbkObj); - Shiboken::walkThroughClassHierarchy(Py_TYPE(pyObj), &visitor); - for (const auto &e : visitor.entries()) + const auto entries = Shiboken::getDestructorEntries(sbkObj); + for (const auto &e : entries) bindingManager.addToDeletionInMainThread(e); } else { Shiboken::DestructorEntry e{sotp->cpp_dtor, sbkObj->d->cptr[0]}; @@ -324,10 +457,9 @@ static void SbkDeallocWrapperCommon(PyObject *pyObj, bool canDelete) if (canDelete) { if (sotp->is_multicpp) { - Shiboken::DtorAccumulatorVisitor visitor(sbkObj); - Shiboken::walkThroughClassHierarchy(Py_TYPE(pyObj), &visitor); + const auto entries = Shiboken::getDestructorEntries(sbkObj); Shiboken::Object::deallocData(sbkObj, true); - callDestructor(visitor.entries()); + callDestructor(entries); } else { void *cptr = sbkObj->d->cptr[0]; Shiboken::Object::deallocData(sbkObj, true); @@ -353,6 +485,17 @@ static void SbkDeallocWrapperCommon(PyObject *pyObj, bool canDelete) } } +static inline PyObject *_Sbk_NewVarObject(PyTypeObject *type) +{ + // PYSIDE-1970: Support __slots__, implemented by PyVarObject + auto const baseSize = sizeof(SbkObject); + auto varCount = Py_SIZE(type); + auto *self = PyObject_GC_NewVar(PyObject, type, varCount); + if (varCount) + std::memset(reinterpret_cast<char *>(self) + baseSize, 0, varCount * sizeof(void *)); + return self; +} + void SbkDeallocWrapper(PyObject *pyObj) { SbkDeallocWrapperCommon(pyObj, true); @@ -376,7 +519,7 @@ void SbkObjectType_tp_dealloc(PyTypeObject *sbkType) auto pyObj = reinterpret_cast<PyObject *>(sbkType); PyObject_GC_UnTrack(pyObj); -#ifndef Py_LIMITED_API +#if !defined(Py_LIMITED_API) && !defined(PYPY_VERSION) # if PY_VERSION_HEX >= 0x030A0000 Py_TRASHCAN_BEGIN(pyObj, 1); # else @@ -394,7 +537,7 @@ void SbkObjectType_tp_dealloc(PyTypeObject *sbkType) Shiboken::Conversions::deleteConverter(sotp->converter); PepType_SOTP_delete(sbkType); } -#ifndef Py_LIMITED_API +#if !defined(Py_LIMITED_API) && !defined(PYPY_VERSION) # if PY_VERSION_HEX >= 0x030A0000 Py_TRASHCAN_END; # else @@ -432,7 +575,7 @@ PyObject *MakeQAppWrapper(PyTypeObject *type) } // monitoring the last application state - PyObject *qApp_curr = type != nullptr ? PyObject_GC_New(PyObject, type) : Py_None; + PyObject *qApp_curr = type != nullptr ? _Sbk_NewVarObject(type) : Py_None; static PyObject *builtins = PyEval_GetBuiltins(); if (PyDict_SetItem(builtins, Shiboken::PyName::qApp(), qApp_curr) < 0) return nullptr; @@ -465,7 +608,7 @@ static PyTypeObject *SbkObjectType_tp_new(PyTypeObject *metatype, PyObject *args PyObject *dict; static const char *kwlist[] = { "name", "bases", "dict", nullptr}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "sO!O!:sbktype", const_cast<char **>(kwlist), + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO!O!:sbktype", const_cast<char **>(kwlist), &name, &PyTuple_Type, &pyBases, &PyDict_Type, &dict)) @@ -473,22 +616,26 @@ static PyTypeObject *SbkObjectType_tp_new(PyTypeObject *metatype, PyObject *args for (int i=0, i_max=PyTuple_GET_SIZE(pyBases); i < i_max; i++) { PyObject *baseType = PyTuple_GET_ITEM(pyBases, i); - if (reinterpret_cast<PyTypeObject *>(baseType)->tp_new == SbkDummyNew) { + if (PepExt_Type_GetNewSlot(reinterpret_cast<PyTypeObject *>(baseType)) == SbkDummyNew) { // PYSIDE-595: A base class does not allow inheritance. return reinterpret_cast<PyTypeObject *>(SbkDummyNew(metatype, args, kwds)); } } - // PYSIDE-939: This is a temporary patch that circumvents the problem - // with Py_TPFLAGS_METHOD_DESCRIPTOR until this is finally solved. - // PyType_Ready uses mro(). We need to temporarily remove the flag from it's type. - // We cannot use PyMethodDescr_Type since it is not exported by Python 2.7 . - static PyTypeObject *PyMethodDescr_TypePtr = Py_TYPE( - PyObject_GetAttr(reinterpret_cast<PyObject *>(&PyType_Type), Shiboken::PyName::mro())); - auto hold = PyMethodDescr_TypePtr->tp_flags; - PyMethodDescr_TypePtr->tp_flags &= ~Py_TPFLAGS_METHOD_DESCRIPTOR; - auto *newType = PepType_Type_tp_new(metatype, args, kwds); - PyMethodDescr_TypePtr->tp_flags = hold; + // PYSIDE-939: This is still a temporary patch that circumvents the problem + // with Py_TPFLAGS_METHOD_DESCRIPTOR. The problem exists in Python 3.8 + // until 3.9.12, only. We check the runtime and hope for this version valishing. + // https://github.com/python/cpython/issues/92112 will not be fixed for 3.8 :/ + PyTypeObject *newType{}; + static auto triplet = _PepRuntimeVersion(); + if (triplet >= (3 << 16 | 8 << 8 | 0) && triplet < (3 << 16 | 9 << 8 | 13)) { + auto hold = PyMethodDescr_Type.tp_flags; + PyMethodDescr_Type.tp_flags &= ~Py_TPFLAGS_METHOD_DESCRIPTOR; + newType = PepType_Type_tp_new(metatype, args, kwds); + PyMethodDescr_Type.tp_flags = hold; + } else { + newType = PepType_Type_tp_new(metatype, args, kwds); + } if (!newType) return nullptr; @@ -537,11 +684,11 @@ static PyTypeObject *SbkObjectType_tp_new(PyTypeObject *metatype, PyObject *args return newType; } -static PyObject *_setupNew(SbkObject *self, PyTypeObject *subtype) +static PyObject *_setupNew(PyObject *obSelf, PyTypeObject *subtype) { auto *obSubtype = reinterpret_cast<PyObject *>(subtype); auto *sbkSubtype = subtype; - auto *obSelf = reinterpret_cast<PyObject *>(self); + auto *self = reinterpret_cast<SbkObject *>(obSelf); Py_INCREF(obSubtype); auto d = new SbkObjectPrivate; @@ -565,18 +712,19 @@ static PyObject *_setupNew(SbkObject *self, PyTypeObject *subtype) return obSelf; } -PyObject *SbkObject_tp_new(PyTypeObject *subtype, PyObject *, PyObject *) +PyObject *SbkObject_tp_new(PyTypeObject *subtype, PyObject * /* args */, PyObject * /* kwds */) { - SbkObject *self = PyObject_GC_New(SbkObject, subtype); + PyObject *self = _Sbk_NewVarObject(subtype); return _setupNew(self, subtype); } PyObject *SbkQApp_tp_new(PyTypeObject *subtype, PyObject *, PyObject *) { - auto self = reinterpret_cast<SbkObject *>(MakeQAppWrapper(subtype)); + auto *obSelf = MakeQAppWrapper(subtype); + auto *self = reinterpret_cast<SbkObject *>(obSelf); if (self == nullptr) return nullptr; - auto ret = _setupNew(self, subtype); + auto ret = _setupNew(obSelf, subtype); auto priv = self->d; priv->isQAppSingleton = 1; return ret; @@ -585,9 +733,9 @@ PyObject *SbkQApp_tp_new(PyTypeObject *subtype, PyObject *, PyObject *) PyObject *SbkDummyNew(PyTypeObject *type, PyObject *, PyObject *) { // PYSIDE-595: Give the same error as type_call does when tp_new is NULL. + const char regret[] = "¯\\_(ツ)_/¯"; PyErr_Format(PyExc_TypeError, - "cannot create '%.100s' instances ¯\\_(ツ)_/¯", - type->tp_name); + "cannot create '%.100s' instances %s", type->tp_name, regret); return nullptr; } @@ -618,6 +766,12 @@ PyObject *FallbackRichCompare(PyObject *self, PyObject *other, int op) return res; } +bool SbkObjectType_Check(PyTypeObject *type) +{ + static auto *meta = SbkObjectType_TypeF(); + return Py_TYPE(type) == meta || PyType_IsSubtype(Py_TYPE(type), meta); +} + } //extern "C" @@ -642,52 +796,9 @@ void _destroyParentInfo(SbkObject *obj, bool keepReference) namespace Shiboken { -bool walkThroughClassHierarchy(PyTypeObject *currentType, HierarchyVisitor *visitor) -{ - PyObject *bases = currentType->tp_bases; - Py_ssize_t numBases = PyTuple_GET_SIZE(bases); - bool result = false; - for (int i = 0; !result && i < numBases; ++i) { - auto type = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(bases, i)); - if (PyType_IsSubtype(type, reinterpret_cast<PyTypeObject *>(SbkObject_TypeF()))) { - result = PepType_SOTP(type)->is_user_type - ? walkThroughClassHierarchy(type, visitor) : visitor->visit(type); - } - } - return result; -} // Wrapper metatype and base type ---------------------------------------------------------- -HierarchyVisitor::HierarchyVisitor() = default; -HierarchyVisitor::~HierarchyVisitor() = default; - -bool BaseCountVisitor::visit(PyTypeObject *) -{ - m_count++; - return false; -} - -bool BaseAccumulatorVisitor::visit(PyTypeObject *node) -{ - m_bases.push_back(node); - return false; -} - -bool GetIndexVisitor::visit(PyTypeObject *node) -{ - m_index++; - return PyType_IsSubtype(node, m_desiredType); -} - -bool DtorAccumulatorVisitor::visit(PyTypeObject *node) -{ - auto *sotp = PepType_SOTP(node); - m_entries.push_back(DestructorEntry{sotp->cpp_dtor, - m_pyObject->d->cptr[m_entries.size()]}); - return false; -} - void _initMainThreadId(); // helper.cpp namespace Conversions { void init(); } @@ -705,9 +816,6 @@ void init() //Init private data Pep384_Init(); - if (PyType_Ready(SbkEnumType_TypeF()) < 0) - Py_FatalError("[libshiboken] Failed to initialize Shiboken.SbkEnumType metatype."); - if (PyType_Ready(SbkObjectType_TypeF()) < 0) Py_FatalError("[libshiboken] Failed to initialize Shiboken.BaseWrapperType metatype."); @@ -720,14 +828,20 @@ void init() } // PYSIDE-1415: Publish Shiboken objects. -void initSignature(PyObject *module) +// PYSIDE-1735: Initialize the whole Shiboken startup. +void initShibokenSupport(PyObject *module) { - auto *type = SbkObject_TypeF(); - if (InitSignatureStrings(type, SbkObject_SignatureStrings) < 0) - return; - Py_INCREF(SbkObject_TypeF()); PyModule_AddObject(module, "Object", reinterpret_cast<PyObject *>(SbkObject_TypeF())); + + // PYSIDE-1735: When the initialization was moved into Shiboken import, this + // Py_INCREF became necessary. No idea why. + Py_INCREF(module); + init_shibokensupport_module(); + + auto *type = SbkObject_TypeF(); + if (InitSignatureStrings(type, SbkObject_SignatureStrings) < 0) + Py_FatalError("Error in initShibokenSupport"); } // setErrorAboutWrongArguments now gets overload info from the signature module. @@ -737,6 +851,32 @@ void setErrorAboutWrongArguments(PyObject *args, const char *funcName, PyObject SetError_Argument(args, funcName, info); } +PyObject *returnWrongArguments(PyObject *args, const char *funcName, PyObject *info) +{ + setErrorAboutWrongArguments(args, funcName, info); + return {}; +} + +int returnWrongArguments_Zero(PyObject *args, const char *funcName, PyObject *info) +{ + setErrorAboutWrongArguments(args, funcName, info); + return 0; +} + +int returnWrongArguments_MinusOne(PyObject *args, const char *funcName, PyObject *info) +{ + setErrorAboutWrongArguments(args, funcName, info); + return -1; +} + +PyObject *returnFromRichCompare(PyObject *result) +{ + if (result && !PyErr_Occurred()) + return result; + Shiboken::Errors::setOperatorNotImplemented(); + return {}; +} + PyObject *checkInvalidArgumentCount(Py_ssize_t numArgs, Py_ssize_t minArgs, Py_ssize_t maxArgs) { PyObject *result = nullptr; @@ -754,20 +894,6 @@ PyObject *checkInvalidArgumentCount(Py_ssize_t numArgs, Py_ssize_t minArgs, Py_s return result; } -class FindBaseTypeVisitor : public HierarchyVisitor -{ -public: - explicit FindBaseTypeVisitor(PyTypeObject *typeToFind) : m_typeToFind(typeToFind) {} - - bool visit(PyTypeObject *node) override - { - return node == m_typeToFind; - } - -private: - PyTypeObject *m_typeToFind; -}; - std::vector<SbkObject *> splitPyObject(PyObject *pyObj) { std::vector<SbkObject *> result; @@ -808,8 +934,8 @@ bool isUserType(PyTypeObject *type) bool canCallConstructor(PyTypeObject *myType, PyTypeObject *ctorType) { - FindBaseTypeVisitor visitor(ctorType); - if (!walkThroughClassHierarchy(myType, &visitor)) { + auto findBasePred = [ctorType](PyTypeObject *type) { return type == ctorType; }; + if (!walkThroughBases(myType, findBasePred)) { PyErr_Format(PyExc_TypeError, "%s isn't a direct base class of %s", ctorType->tp_name, myType->tp_name); return false; } @@ -881,31 +1007,35 @@ introduceWrapperType(PyObject *enclosingObject, const char *originalName, PyType_Spec *typeSpec, ObjectDestructor cppObjDtor, - PyTypeObject *baseType, - PyObject *baseTypes, + PyObject *bases, unsigned wrapperFlags) { - auto *base = baseType ? baseType : SbkObject_TypeF(); - typeSpec->slots[0].pfunc = reinterpret_cast<void *>(base); - auto *bases = baseTypes ? baseTypes : PyTuple_Pack(1, base); + const auto basesSize = PySequence_Fast_GET_SIZE(bases); + assert(basesSize > 0); + typeSpec->slots[0].pfunc = PySequence_Fast_GET_ITEM(bases, 0); auto *type = SbkType_FromSpecBasesMeta(typeSpec, bases, SbkObjectType_TypeF()); - for (int i = 0; i < PySequence_Fast_GET_SIZE(bases); ++i) { - auto *st = reinterpret_cast<PyTypeObject *>(PySequence_Fast_GET_ITEM(bases, i)); - BindingManager::instance().addClassInheritance(st, type); - } - auto sotp = PepType_SOTP(type); if (wrapperFlags & DeleteInMainThread) sotp->delete_in_main_thread = 1; + sotp->type_behaviour = (wrapperFlags & Value) != 0 + ? BEHAVIOUR_VALUETYPE : BEHAVIOUR_OBJECTTYPE; setOriginalName(type, originalName); setDestructorFunction(type, cppObjDtor); auto *ob_type = reinterpret_cast<PyObject *>(type); - if (wrapperFlags & InnerClass) + if (wrapperFlags & InnerClass) { + // PYSIDE-2230: Instead of tp_dict, use the enclosing type. + // This stays interface compatible. + if (PyType_Check(enclosingObject)) { + AutoDecRef tpDict(PepType_GetDict(reinterpret_cast<PyTypeObject *>(enclosingObject))); + return PyDict_SetItemString(tpDict, typeName, ob_type) == 0 ? type : nullptr; + } + assert(PyDict_Check(enclosingObject)); return PyDict_SetItemString(enclosingObject, typeName, ob_type) == 0 ? type : nullptr; + } // PyModule_AddObject steals type's reference. Py_INCREF(ob_type); @@ -920,16 +1050,19 @@ introduceWrapperType(PyObject *enclosingObject, void setSubTypeInitHook(PyTypeObject *type, SubTypeInitHook func) { + assert(SbkObjectType_Check(type)); PepType_SOTP(type)->subtype_init = func; } void *getTypeUserData(PyTypeObject *type) { + assert(SbkObjectType_Check(type)); return PepType_SOTP(type)->user_data; } void setTypeUserData(PyTypeObject *type, void *userData, DeleteUserDataFunc d_func) { + assert(SbkObjectType_Check(type)); auto *sotp = PepType_SOTP(type); sotp->user_data = userData; sotp->d_func = d_func; @@ -1040,9 +1173,7 @@ void callCppDestructors(SbkObject *pyObj) PyTypeObject *type = Py_TYPE(pyObj); auto *sotp = PepType_SOTP(type); if (sotp->is_multicpp) { - Shiboken::DtorAccumulatorVisitor visitor(pyObj); - Shiboken::walkThroughClassHierarchy(type, &visitor); - callDestructor(visitor.entries()); + callDestructor(getDestructorEntries(pyObj)); } else { Shiboken::ThreadStateSaver threadSaver; threadSaver.save(); @@ -1191,11 +1322,10 @@ void makeValid(SbkObject *self) // If has ref to other objects make all valid again if (self->d->referredObjects) { - RefCountMap &refCountMap = *(self->d->referredObjects); - RefCountMap::iterator iter; - for (auto it = refCountMap.begin(), end = refCountMap.end(); it != end; ++it) { - if (Shiboken::Object::checkType(it->second)) - makeValid(reinterpret_cast<SbkObject *>(it->second)); + const RefCountMap &refCountMap = *(self->d->referredObjects); + for (const auto &p : refCountMap) { + if (Shiboken::Object::checkType(p.second)) + makeValid(reinterpret_cast<SbkObject *>(p.second)); } } } @@ -1231,7 +1361,8 @@ bool setCppPointer(SbkObject *sbkObj, PyTypeObject *desiredType, void *cptr) const bool alreadyInitialized = sbkObj->d->cptr[idx] != nullptr; if (alreadyInitialized) - PyErr_SetString(PyExc_RuntimeError, "You can't initialize an object twice!"); + PyErr_Format(PyExc_RuntimeError, "You can't initialize an %s object in class %s twice!", + desiredType->tp_name, type->tp_name); else sbkObj->d->cptr[idx] = cptr; @@ -1331,10 +1462,15 @@ PyObject *newObject(PyTypeObject *instanceType, { // Try to find the exact type of cptr. if (!isExactType) { - if (PyTypeObject *exactType = ObjectType::typeForTypeName(typeName)) + if (PyTypeObject *exactType = ObjectType::typeForTypeName(typeName)) { instanceType = exactType; - else - instanceType = BindingManager::instance().resolveType(&cptr, instanceType); + } else { + auto resolved = BindingManager::instance().findDerivedType(cptr, instanceType); + if (resolved.first != nullptr) { + instanceType = resolved.first; + cptr = resolved.second; + } + } } bool shouldCreate = true; @@ -1545,7 +1681,7 @@ void deallocData(SbkObject *self, bool cleanup) } delete self->d; // PYSIDE-205: always delete d. Py_XDECREF(self->ob_dict); - Py_TYPE(self)->tp_free(self); + PepExt_TypeCallFree(reinterpret_cast<PyObject *>(self)); } void setTypeUserData(SbkObject *wrapper, void *userData, DeleteUserDataFunc d_func) @@ -1637,6 +1773,11 @@ static std::vector<PyTypeObject *> getBases(SbkObject *self) : std::vector<PyTypeObject *>(1, Py_TYPE(self)); } +static bool isValueType(SbkObject *self) +{ + return PepType_SOTP(Py_TYPE(self))->type_behaviour == BEHAVIOUR_VALUETYPE; +} + void _debugFormat(std::ostream &s, SbkObject *self) { assert(self); @@ -1660,6 +1801,8 @@ void _debugFormat(std::ostream &s, SbkObject *self) s << " [validCppObject]"; if (d->cppObjectCreated) s << " [wasCreatedByPython]"; + s << (isValueType(self) ? " [value]" : " [object]"); + if (d->parentInfo) { if (auto *parent = d->parentInfo->parent) s << ", parent=" << reinterpret_cast<PyObject *>(parent)->ob_type->tp_name @@ -1690,8 +1833,9 @@ std::string info(SbkObject *self) s << "hasOwnership...... " << bool(self->d->hasOwnership) << "\n" "containsCppWrapper " << self->d->containsCppWrapper << "\n" "validCppObject.... " << self->d->validCppObject << "\n" - "wasCreatedByPython " << self->d->cppObjectCreated << "\n"; - + "wasCreatedByPython " << self->d->cppObjectCreated << "\n" + "value...... " << isValueType(self) << "\n" + "reference count... " << reinterpret_cast<PyObject *>(self)->ob_refcnt << '\n'; if (self->d->parentInfo && self->d->parentInfo->parent) { s << "parent............ "; @@ -1709,17 +1853,17 @@ std::string info(SbkObject *self) } if (self->d->referredObjects && !self->d->referredObjects->empty()) { - Shiboken::RefCountMap &map = *self->d->referredObjects; + const Shiboken::RefCountMap &map = *self->d->referredObjects; s << "referred objects.. "; std::string lastKey; - for (auto it = map.begin(), end = map.end(); it != end; ++it) { - if (it->first != lastKey) { + for (const auto &p : map) { + if (p.first != lastKey) { if (!lastKey.empty()) s << " "; - s << '"' << it->first << "\" => "; - lastKey = it->first; + s << '"' << p.first << "\" => "; + lastKey = p.first; } - Shiboken::AutoDecRef obj(PyObject_Str(it->second)); + Shiboken::AutoDecRef obj(PyObject_Str(p.second)); s << String::toCString(obj) << ' '; } s << '\n'; diff --git a/sources/shiboken6/libshiboken/basewrapper.h b/sources/shiboken6/libshiboken/basewrapper.h index 6ef3cfd30..4835c4810 100644 --- a/sources/shiboken6/libshiboken/basewrapper.h +++ b/sources/shiboken6/libshiboken/basewrapper.h @@ -38,32 +38,30 @@ LIBSHIBOKEN_API void SbkDeallocQAppWrapper(PyObject *pyObj); LIBSHIBOKEN_API void SbkDeallocWrapperWithPrivateDtor(PyObject *self); /// Function signature for the multiple inheritance information initializers that should be provided by classes with multiple inheritance. -typedef int *(*MultipleInheritanceInitFunction)(const void *); +using MultipleInheritanceInitFunction = int *(*)(const void *); /** * Special cast function is used to correctly cast an object when it's * part of a multiple inheritance hierarchy. * The implementation of this function is auto generated by the generator and you don't need to care about it. */ -typedef void *(*SpecialCastFunction)(void *, PyTypeObject *); -typedef PyTypeObject *(*TypeDiscoveryFunc)(void *, PyTypeObject *); -typedef void *(*TypeDiscoveryFuncV2)(void *, PyTypeObject *); +using SpecialCastFunction = void *(*)(void *, PyTypeObject *); +using TypeDiscoveryFunc = PyTypeObject *(*)(void *, PyTypeObject *); +using TypeDiscoveryFuncV2 = void *(*)(void *, PyTypeObject *); // Used in userdata dealloc function -typedef void (*DeleteUserDataFunc)(void *); +using DeleteUserDataFunc = void (*)(void *); -typedef void (*ObjectDestructor)(void *); +using ObjectDestructor = void (*)(void *); -typedef void (*SubTypeInitHook)(PyTypeObject *, PyObject *, PyObject *); +using SubTypeInitHook = void (*)(PyTypeObject *, PyObject *, PyObject *); /// PYSIDE-1019: Set the function to select the current feature. /// Return value is the previous content. -typedef PyObject *(*SelectableFeatureHook)(PyTypeObject *); +using SelectableFeatureHook = void (*)(PyTypeObject *); +using SelectableFeatureCallback = void (*)(bool); LIBSHIBOKEN_API SelectableFeatureHook initSelectableFeature(SelectableFeatureHook func); - -/// PYSIDE-1019: Get access to PySide reserved bits. -LIBSHIBOKEN_API int SbkObjectType_GetReserved(PyTypeObject *type); -LIBSHIBOKEN_API void SbkObjectType_SetReserved(PyTypeObject *type, int value); +LIBSHIBOKEN_API void setSelectableFeatureCallback(SelectableFeatureCallback func); /// PYSIDE-1626: Enforcing a context switch without further action. LIBSHIBOKEN_API void SbkObjectType_UpdateFeature(PyTypeObject *type); @@ -72,8 +70,11 @@ LIBSHIBOKEN_API void SbkObjectType_UpdateFeature(PyTypeObject *type); LIBSHIBOKEN_API const char **SbkObjectType_GetPropertyStrings(PyTypeObject *type); LIBSHIBOKEN_API void SbkObjectType_SetPropertyStrings(PyTypeObject *type, const char **strings); +/// PYSIDE-1735: Store the enumFlagInfo. +LIBSHIBOKEN_API void SbkObjectType_SetEnumFlagInfo(PyTypeObject *type, const char **strings); + /// PYSIDE-1470: Set the function to kill a Q*Application. -typedef void(*DestroyQAppHook)(); +using DestroyQAppHook = void(*)(); LIBSHIBOKEN_API void setDestroyQApplication(DestroyQAppHook func); /// PYSIDE-535: Use the C API in PyPy instead of `op->ob_dict`, directly (borrowed ref) @@ -108,6 +109,12 @@ LIBSHIBOKEN_API PyObject *SbkDummyNew(PyTypeObject *type, PyObject *, PyObject * /// PYSIDE-74: Fallback used in all types now. LIBSHIBOKEN_API PyObject *FallbackRichCompare(PyObject *self, PyObject *other, int op); +/// PYSIDE-1970: Be easily able to see what is happening in the running code. +LIBSHIBOKEN_API void disassembleFrame(const char *marker); + +/// PYSIDE-2230: Check if an object is an SbkObject. +LIBSHIBOKEN_API bool SbkObjectType_Check(PyTypeObject *type); + } // extern "C" namespace Shiboken @@ -119,7 +126,7 @@ namespace Shiboken LIBSHIBOKEN_API void init(); /// PYSIDE-1415: Publish Shiboken objects. -LIBSHIBOKEN_API void initSignature(PyObject *module); +LIBSHIBOKEN_API void initShibokenSupport(PyObject *module); /// Delete the class T allocated on \p cptr. template<typename T> @@ -128,11 +135,25 @@ void callCppDestructor(void *cptr) delete reinterpret_cast<T *>(cptr); } -// setErrorAboutWrongArguments now gets overload information from the signature module. -// The extra info argument can contain additional data about the error. +/// setErrorAboutWrongArguments now gets overload information from the signature module. +/// The extra info argument can contain additional data about the error. LIBSHIBOKEN_API void setErrorAboutWrongArguments(PyObject *args, const char *funcName, PyObject *info); +/// Return values for the different retun variants. +/// This is used instead of goto. +LIBSHIBOKEN_API PyObject *returnWrongArguments(PyObject *args, const char *funcName, + PyObject *info); + +LIBSHIBOKEN_API int returnWrongArguments_Zero(PyObject *args, const char *funcName, + PyObject *info); + +LIBSHIBOKEN_API int returnWrongArguments_MinusOne(PyObject *args, const char *funcName, + PyObject *info); + +/// A simple special version for the end of rich comparison. +LIBSHIBOKEN_API PyObject *returnFromRichCompare(PyObject *result); + // Return error information object if the argument count is wrong LIBSHIBOKEN_API PyObject *checkInvalidArgumentCount(Py_ssize_t numArgs, Py_ssize_t minArgs, @@ -179,14 +200,15 @@ LIBSHIBOKEN_API const char *getOriginalName(PyTypeObject *self); LIBSHIBOKEN_API void setTypeDiscoveryFunctionV2(PyTypeObject *self, TypeDiscoveryFuncV2 func); LIBSHIBOKEN_API void copyMultipleInheritance(PyTypeObject *self, PyTypeObject *other); LIBSHIBOKEN_API void setMultipleInheritanceFunction(PyTypeObject *self, MultipleInheritanceInitFunction func); -LIBSHIBOKEN_API MultipleInheritanceInitFunction getMultipleInheritanceFunction(PyTypeObject *self); +LIBSHIBOKEN_API MultipleInheritanceInitFunction getMultipleInheritanceFunction(PyTypeObject *type); LIBSHIBOKEN_API void setDestructorFunction(PyTypeObject *self, ObjectDestructor func); enum WrapperFlags { InnerClass = 0x1, - DeleteInMainThread = 0x2 + DeleteInMainThread = 0x2, + Value = 0x4 }; /** @@ -207,13 +229,12 @@ enum WrapperFlags * \returns true if the initialization went fine, false otherwise. */ LIBSHIBOKEN_API PyTypeObject *introduceWrapperType(PyObject *enclosingObject, - const char *typeName, - const char *originalName, - PyType_Spec *typeSpec, - ObjectDestructor cppObjDtor, - PyTypeObject *baseType, - PyObject *baseTypes, - unsigned wrapperFlags = 0); + const char *typeName, + const char *originalName, + PyType_Spec *typeSpec, + ObjectDestructor cppObjDtor, + PyObject *bases, + unsigned wrapperFlags = 0); /** * Set the subtype init hook for a type. diff --git a/sources/shiboken6/libshiboken/basewrapper_p.h b/sources/shiboken6/libshiboken/basewrapper_p.h index 775b9bd5c..fb9140793 100644 --- a/sources/shiboken6/libshiboken/basewrapper_p.h +++ b/sources/shiboken6/libshiboken/basewrapper_p.h @@ -30,14 +30,12 @@ using ChildrenList = std::set<SbkObject *>; /// Structure used to store information about object parent and children. struct ParentInfo { - /// Default ctor. - ParentInfo() : parent(nullptr), hasWrapperRef(false) {} /// Pointer to parent object. - SbkObject *parent; + SbkObject *parent = nullptr; /// List of object children. ChildrenList children; /// has internal ref - bool hasWrapperRef; + bool hasWrapperRef = false; }; } // namespace Shiboken @@ -51,6 +49,12 @@ extern "C" */ struct SbkObjectPrivate { + SbkObjectPrivate() noexcept = default; + SbkObjectPrivate(const SbkObjectPrivate &) = delete; + SbkObjectPrivate(SbkObjectPrivate &&o) = delete; + SbkObjectPrivate &operator=(const SbkObjectPrivate &) = delete; + SbkObjectPrivate &operator=(SbkObjectPrivate &&o) = delete; + /// Pointer to the C++ class. void ** cptr; /// True when Python is responsible for freeing the used memory. @@ -97,15 +101,6 @@ struct SbkObjectTypePrivate TypeDiscoveryFuncV2 type_discovery; /// Pointer to a function responsible for deletion of the C++ instance calling the proper destructor. ObjectDestructor cpp_dtor; - /// PYSIDE-1019: Caching the current select Id - unsigned int pyside_reserved_bits : 8; // MSVC has bug with the sign bit! - /// True if this type holds two or more C++ instances, e.g.: a Python class which inherits from two C++ classes. - unsigned int is_multicpp : 1; - /// True if this type was defined by the user. - unsigned int is_user_type : 1; - /// Tells is the type is a value type or an object-type, see BEHAVIOUR_ *constants. - unsigned int type_behaviour : 2; - unsigned int delete_in_main_thread : 1; /// C++ name char *original_name; /// Type user data @@ -113,6 +108,18 @@ struct SbkObjectTypePrivate DeleteUserDataFunc d_func; void (*subtype_init)(PyTypeObject *, PyObject *, PyObject *); const char **propertyStrings; + const char **enumFlagInfo; + PyObject *enumFlagsDict; + PyObject *enumTypeDict; + + /// True if this type holds two or more C++ instances, e.g.: a Python class which inherits from two C++ classes. + unsigned int is_multicpp : 1; + /// True if this type was defined by the user (a class written in Python inheriting + /// a class provided by a Shiboken binding). + unsigned int is_user_type : 1; + /// Tells is the type is a value type or an object-type, see BEHAVIOUR_ *constants. + unsigned int type_behaviour : 2; + unsigned int delete_in_main_thread : 1; }; @@ -136,107 +143,7 @@ struct DestructorEntry **/ std::vector<SbkObject *> splitPyObject(PyObject *pyObj); -/** -* Visitor class used by walkOnClassHierarchy function. -*/ -class HierarchyVisitor -{ -public: - HierarchyVisitor(const HierarchyVisitor &) = delete; - HierarchyVisitor(HierarchyVisitor &&) = delete; - HierarchyVisitor &operator=(const HierarchyVisitor &) = delete; - HierarchyVisitor &operator=(HierarchyVisitor &&) = delete; - - HierarchyVisitor(); - virtual ~HierarchyVisitor(); - - virtual bool visit(PyTypeObject *node) = 0; // return true to terminate -}; - -class BaseCountVisitor : public HierarchyVisitor -{ -public: - bool visit(PyTypeObject *) override; - - int count() const { return m_count; } - -private: - int m_count = 0; -}; - -class BaseAccumulatorVisitor : public HierarchyVisitor -{ -public: - using Result = std::vector<PyTypeObject *>; - - bool visit(PyTypeObject *node) override; - - Result bases() const { return m_bases; } - -private: - Result m_bases; -}; - -class GetIndexVisitor : public HierarchyVisitor -{ -public: - explicit GetIndexVisitor(PyTypeObject *desiredType) : m_desiredType(desiredType) {} - - bool visit(PyTypeObject *node) override; - - int index() const { return m_index; } - -private: - int m_index = -1; - PyTypeObject *m_desiredType; -}; - -/// Collect destructors and C++ instances of each C++ object held by a Python -/// object -class DtorAccumulatorVisitor : public HierarchyVisitor -{ -public: - explicit DtorAccumulatorVisitor(SbkObject *pyObj) : m_pyObject(pyObj) {} - - bool visit(PyTypeObject *node) override; - - using DestructorEntries = std::vector<DestructorEntry>; - - const DestructorEntries &entries() const { return m_entries; } - -private: - DestructorEntries m_entries; - SbkObject *m_pyObject; -}; - -/// \internal Internal function used to walk on classes inheritance trees. -/** -* Walk on class hierarchy using a DFS algorithm. -* For each pure Shiboken type found, HierarchyVisitor::visit is called and the algorithm -* considers all children of this type as visited. -*/ -bool walkThroughClassHierarchy(PyTypeObject *currentType, HierarchyVisitor *visitor); - -inline int getTypeIndexOnHierarchy(PyTypeObject *baseType, PyTypeObject *desiredType) -{ - GetIndexVisitor visitor(desiredType); - walkThroughClassHierarchy(baseType, &visitor); - return visitor.index(); -} - -inline int getNumberOfCppBaseClasses(PyTypeObject *baseType) -{ - BaseCountVisitor visitor; - walkThroughClassHierarchy(baseType, &visitor); - return visitor.count(); -} - -inline std::vector<PyTypeObject *> getCppBaseClasses(PyTypeObject *baseType) -{ - BaseAccumulatorVisitor visitor; - walkThroughClassHierarchy(baseType, &visitor); - return visitor.bases(); -} +int getNumberOfCppBaseClasses(PyTypeObject *baseType); namespace Object { diff --git a/sources/shiboken6/libshiboken/bindingmanager.cpp b/sources/shiboken6/libshiboken/bindingmanager.cpp index 9d74e9721..83c927ae5 100644 --- a/sources/shiboken6/libshiboken/bindingmanager.cpp +++ b/sources/shiboken6/libshiboken/bindingmanager.cpp @@ -6,101 +6,162 @@ #include "basewrapper_p.h" #include "bindingmanager.h" #include "gilstate.h" +#include "helper.h" +#include "sbkmodule.h" #include "sbkstring.h" #include "sbkstaticstrings.h" #include "sbkfeature_base.h" #include "debugfreehook.h" #include <cstddef> +#include <cstring> #include <fstream> +#include <iostream> #include <mutex> +#include <string_view> #include <unordered_map> +#include <unordered_set> + +// GraphNode for the dependency graph. It keeps a pointer to +// the TypeInitStruct to be able to lazily create the type and hashes +// by the full type name. +struct GraphNode +{ + explicit GraphNode(Shiboken::Module::TypeInitStruct *i) : name(i->fullName), initStruct(i) {} + explicit GraphNode(const char *n) : name(n), initStruct(nullptr) {} // Only for searching + + std::string_view name; + Shiboken::Module::TypeInitStruct *initStruct; + + friend bool operator==(const GraphNode &n1, const GraphNode &n2) { return n1.name == n2.name; } + friend bool operator!=(const GraphNode &n1, const GraphNode &n2) { return n1.name != n2.name; } +}; + +template <> +struct std::hash<GraphNode> { + size_t operator()(const GraphNode &n) const noexcept + { + return std::hash<std::string_view>{}(n.name); + } +}; namespace Shiboken { using WrapperMap = std::unordered_map<const void *, SbkObject *>; -class Graph +template <class NodeType> +class BaseGraph { public: - using NodeList = std::vector<PyTypeObject *>; - using Edges = std::unordered_map<PyTypeObject *, NodeList>; + using NodeList = std::vector<NodeType>; + using NodeSet = std::unordered_set<NodeType>; + + using Edges = std::unordered_map<NodeType, NodeList>; Edges m_edges; - Graph() = default; + BaseGraph() = default; - void addEdge(PyTypeObject *from, PyTypeObject *to) + void addEdge(NodeType from, NodeType to) { m_edges[from].push_back(to); } -#ifndef NDEBUG - void dumpDotGraph() + NodeSet nodeSet() const { - std::ofstream file("/tmp/shiboken_graph.dot"); - - file << "digraph D {\n"; - - for (auto i = m_edges.begin(), end = m_edges.end(); i != end; ++i) { - auto *node1 = i->first; - const NodeList &nodeList = i->second; - for (const PyTypeObject *o : nodeList) { - auto *node2 = o; - file << '"' << node2->tp_name << "\" -> \"" - << node1->tp_name << "\"\n"; - } + NodeSet result; + for (const auto &p : m_edges) { + result.insert(p.first); + for (const auto node2 : p.second) + result.insert(node2); } - file << "}\n"; + return result; } -#endif +}; + +class Graph : public BaseGraph<GraphNode> +{ +public: + using TypeCptrPair = BindingManager::TypeCptrPair; - PyTypeObject *identifyType(void **cptr, PyTypeObject *type, PyTypeObject *baseType) const + TypeCptrPair identifyType(void *cptr, PyTypeObject *type, PyTypeObject *baseType) const { - auto edgesIt = m_edges.find(type); - if (edgesIt != m_edges.end()) { - const NodeList &adjNodes = m_edges.find(type)->second; - for (PyTypeObject *node : adjNodes) { - PyTypeObject *newType = identifyType(cptr, node, baseType); - if (newType) - return newType; - } - } - void *typeFound = nullptr; - auto *sotp = PepType_SOTP(type); - if (sotp->type_discovery) - typeFound = sotp->type_discovery(*cptr, baseType); - if (typeFound) { - // This "typeFound != type" is needed for backwards compatibility with old modules using a newer version of - // libshiboken because old versions of type_discovery function used to return a PyTypeObject *instead of - // a possible variation of the C++ instance pointer (*cptr). - if (typeFound != type) - *cptr = typeFound; - return type; - } - return nullptr; + return identifyType(cptr, GraphNode(type->tp_name), type, baseType); } -}; + bool dumpTypeGraph(const char *fileName) const; -#ifndef NDEBUG -static void showWrapperMap(const WrapperMap &wrapperMap) +private: + TypeCptrPair identifyType(void *cptr, const GraphNode &typeNode, PyTypeObject *type, + PyTypeObject *baseType) const; +}; + +Graph::TypeCptrPair Graph::identifyType(void *cptr, + const GraphNode &typeNode, PyTypeObject *type, + PyTypeObject *baseType) const { - if (Py_VerboseFlag > 0) { - fprintf(stderr, "-------------------------------\n"); - fprintf(stderr, "WrapperMap: %p (size: %d)\n", &wrapperMap, (int) wrapperMap.size()); - for (auto it = wrapperMap.begin(), end = wrapperMap.end(); it != end; ++it) { - const SbkObject *sbkObj = it->second; - fprintf(stderr, "key: %p, value: %p (%s, refcnt: %d)\n", it->first, - static_cast<const void *>(sbkObj), - (Py_TYPE(sbkObj))->tp_name, - int(reinterpret_cast<const PyObject *>(sbkObj)->ob_refcnt)); + assert(typeNode.initStruct != nullptr || type != nullptr); + auto edgesIt = m_edges.find(typeNode); + if (edgesIt != m_edges.end()) { + const NodeList &adjNodes = edgesIt->second; + for (const auto &node : adjNodes) { + auto newType = identifyType(cptr, node, nullptr, baseType); + if (newType.first != nullptr) + return newType; } - fprintf(stderr, "-------------------------------\n"); } + + if (type == nullptr) { + if (typeNode.initStruct->type == nullptr) // Layzily create type + type = Shiboken::Module::get(*typeNode.initStruct); + else + type = typeNode.initStruct->type; + } + + auto *sotp = PepType_SOTP(type); + if (sotp->type_discovery != nullptr) { + if (void *derivedCPtr = sotp->type_discovery(cptr, baseType)) + return {type, derivedCPtr}; + } + return {nullptr, nullptr}; +} + +static void formatDotNode(std::string_view name, std::ostream &file) +{ + auto lastDot = name.rfind('.'); + file << " \"" << name << "\" [ label="; + if (lastDot != std::string::npos) { + file << '"' << name.substr(lastDot + 1) << "\" tooltip=\"" + << name.substr(0, lastDot) << '"'; + } else { + file << '"' << name << '"'; + } + file << " ]\n"; +} + +bool Graph::dumpTypeGraph(const char *fileName) const +{ + std::ofstream file(fileName); + if (!file.good()) + return false; + + file << "digraph D {\n"; + + // Define nodes with short names + for (const auto &node : nodeSet()) + formatDotNode(node.name, file); + + // Write edges + for (const auto &p : m_edges) { + const auto &node1 = p.first; + const NodeList &nodeList = p.second; + for (const auto &node2 : nodeList) + file << " \"" << node2.name << "\" -> \"" << node1.name << "\"\n"; + } + file << "}\n"; + return true; } -#endif struct BindingManager::BindingManagerPrivate { using DestructorEntries = std::vector<DestructorEntry>; @@ -113,20 +174,19 @@ struct BindingManager::BindingManagerPrivate { std::recursive_mutex wrapperMapLock; Graph classHierarchy; DestructorEntries deleteInMainThread; - bool destroying; - BindingManagerPrivate() : destroying(false) {} - bool releaseWrapper(void *cptr, SbkObject *wrapper); - void assignWrapper(SbkObject *wrapper, const void *cptr); + bool releaseWrapper(void *cptr, SbkObject *wrapper, const int *bases = nullptr); + bool releaseWrapperHelper(void *cptr, SbkObject *wrapper); + void assignWrapper(SbkObject *wrapper, const void *cptr, const int *bases = nullptr); + void assignWrapperHelper(SbkObject *wrapper, const void *cptr); }; -bool BindingManager::BindingManagerPrivate::releaseWrapper(void *cptr, SbkObject *wrapper) +inline bool BindingManager::BindingManagerPrivate::releaseWrapperHelper(void *cptr, SbkObject *wrapper) { // The wrapper argument is checked to ensure that the correct wrapper is released. // Returns true if the correct wrapper is found and released. // If wrapper argument is NULL, no such check is performed. - std::lock_guard<std::recursive_mutex> guard(wrapperMapLock); auto iter = wrapperMapper.find(cptr); if (iter != wrapperMapper.end() && (wrapper == nullptr || iter->second == wrapper)) { wrapperMapper.erase(iter); @@ -135,15 +195,41 @@ bool BindingManager::BindingManagerPrivate::releaseWrapper(void *cptr, SbkObject return false; } -void BindingManager::BindingManagerPrivate::assignWrapper(SbkObject *wrapper, const void *cptr) +bool BindingManager::BindingManagerPrivate::releaseWrapper(void *cptr, SbkObject *wrapper, + const int *bases) { assert(cptr); std::lock_guard<std::recursive_mutex> guard(wrapperMapLock); + const bool result = releaseWrapperHelper(cptr, wrapper); + if (bases != nullptr) { + auto *base = static_cast<uint8_t *>(cptr); + for (const auto *offset = bases; *offset != -1; ++offset) + releaseWrapperHelper(base + *offset, wrapper); + } + return result; +} + +inline void BindingManager::BindingManagerPrivate::assignWrapperHelper(SbkObject *wrapper, + const void *cptr) +{ auto iter = wrapperMapper.find(cptr); if (iter == wrapperMapper.end()) wrapperMapper.insert(std::make_pair(cptr, wrapper)); } +void BindingManager::BindingManagerPrivate::assignWrapper(SbkObject *wrapper, const void *cptr, + const int *bases) +{ + assert(cptr); + std::lock_guard<std::recursive_mutex> guard(wrapperMapLock); + assignWrapperHelper(wrapper, cptr); + if (bases != nullptr) { + const auto *base = static_cast<const uint8_t *>(cptr); + for (const auto *offset = bases; *offset != -1; ++offset) + assignWrapperHelper(wrapper, base + *offset); + } +} + BindingManager::BindingManager() { m_d = new BindingManager::BindingManagerPrivate; @@ -159,7 +245,8 @@ BindingManager::~BindingManager() debugRemoveFreeHook(); #endif #ifndef NDEBUG - showWrapperMap(m_d->wrapperMapper); + if (Shiboken::pyVerbose() > 0) + dumpWrapperMap(); #endif /* Cleanup hanging references. We just invalidate them as when * the BindingManager is being destroyed the interpreter is alredy @@ -195,15 +282,7 @@ void BindingManager::registerWrapper(SbkObject *pyObj, void *cptr) if (d->mi_init && !d->mi_offsets) d->mi_offsets = d->mi_init(cptr); - m_d->assignWrapper(pyObj, cptr); - if (d->mi_offsets) { - int *offset = d->mi_offsets; - while (*offset != -1) { - if (*offset > 0) - m_d->assignWrapper(pyObj, reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(cptr) + *offset)); - offset++; - } - } + m_d->assignWrapper(pyObj, cptr, d->mi_offsets); } void BindingManager::releaseWrapper(SbkObject *sbkObj) @@ -213,17 +292,10 @@ void BindingManager::releaseWrapper(SbkObject *sbkObj) int numBases = ((d && d->is_multicpp) ? getNumberOfCppBaseClasses(Py_TYPE(sbkObj)) : 1); void ** cptrs = reinterpret_cast<SbkObject *>(sbkObj)->d->cptr; + const int *mi_offsets = d != nullptr ? d->mi_offsets : nullptr; for (int i = 0; i < numBases; ++i) { - auto *cptr = reinterpret_cast<unsigned char *>(cptrs[i]); - m_d->releaseWrapper(cptr, sbkObj); - if (d && d->mi_offsets) { - int *offset = d->mi_offsets; - while (*offset != -1) { - if (*offset > 0) - m_d->releaseWrapper(reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(cptr) + *offset), sbkObj); - offset++; - } - } + if (cptrs[i] != nullptr) + m_d->releaseWrapper(cptrs[i], sbkObj, mi_offsets); } sbkObj->d->validCppObject = false; } @@ -256,7 +328,7 @@ PyObject *BindingManager::getOverride(const void *cptr, SbkObject *wrapper = retrieveWrapper(cptr); // The refcount can be 0 if the object is dieing and someone called // a virtual method from the destructor - if (!wrapper || reinterpret_cast<const PyObject *>(wrapper)->ob_refcnt == 0) + if (!wrapper || Py_REFCNT(reinterpret_cast<const PyObject *>(wrapper)) == 0) return nullptr; // PYSIDE-1626: Touch the type to initiate switching early. @@ -276,6 +348,8 @@ PyObject *BindingManager::getOverride(const void *cptr, auto *obWrapper = reinterpret_cast<PyObject *>(wrapper); auto *wrapper_dict = SbkObject_GetDict_NoRef(obWrapper); if (PyObject *method = PyDict_GetItem(wrapper_dict, pyMethodName)) { + // Note: This special case was implemented for duck-punching, which happens + // in the instance dict. It does not work with properties. Py_INCREF(method); return method; } @@ -317,36 +391,53 @@ PyObject *BindingManager::getOverride(const void *cptr, } if (method != nullptr) { - PyObject *defaultMethod; + PyObject *defaultMethod{}; PyObject *mro = Py_TYPE(wrapper)->tp_mro; int size = PyTuple_GET_SIZE(mro); + bool defaultFound = false; // The first class in the mro (index 0) is the class being checked and it should not be tested. // The last class in the mro (size - 1) is the base Python object class which should not be tested also. for (int idx = 1; idx < size - 1; ++idx) { auto *parent = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(mro, idx)); - if (parent->tp_dict) { - defaultMethod = PyDict_GetItem(parent->tp_dict, pyMethodName); - if (defaultMethod && function != defaultMethod) - return method; + AutoDecRef tpDict(PepType_GetDict(parent)); + auto *parentDict = tpDict.object(); + if (parentDict) { + defaultMethod = PyDict_GetItem(parentDict, pyMethodName); + if (defaultMethod) { + defaultFound = true; + if (function != defaultMethod) + return method; + } } } - + // PYSIDE-2255: If no default method was found, use the method. + if (!defaultFound) + return method; Py_DECREF(method); } return nullptr; } -void BindingManager::addClassInheritance(PyTypeObject *parent, PyTypeObject *child) +void BindingManager::addClassInheritance(Module::TypeInitStruct *parent, + Module::TypeInitStruct *child) { - m_d->classHierarchy.addEdge(parent, child); + m_d->classHierarchy.addEdge(GraphNode(parent), GraphNode(child)); } +BindingManager::TypeCptrPair BindingManager::findDerivedType(void *cptr, PyTypeObject *type) const +{ + return m_d->classHierarchy.identifyType(cptr, type, type); +} + +// FIXME PYSIDE7: remove, just for compatibility PyTypeObject *BindingManager::resolveType(void **cptr, PyTypeObject *type) { - PyTypeObject *identifiedType = m_d->classHierarchy.identifyType(cptr, type, type); - return identifiedType ? identifiedType : type; + auto result = findDerivedType(*cptr, type); + if (result.second != nullptr) + *cptr = result.second; + return result.first != nullptr ? result.first : type; } std::set<PyObject *> BindingManager::getAllPyObjects() @@ -364,10 +455,94 @@ std::set<PyObject *> BindingManager::getAllPyObjects() void BindingManager::visitAllPyObjects(ObjectVisitor visitor, void *data) { WrapperMap copy = m_d->wrapperMapper; - for (auto it = copy.begin(); it != copy.end(); ++it) { - if (hasWrapper(it->first)) - visitor(it->second, data); + for (const auto &p : copy) { + if (hasWrapper(p.first)) + visitor(p.second, data); + } +} + +bool BindingManager::dumpTypeGraph(const char *fileName) const +{ + return m_d->classHierarchy.dumpTypeGraph(fileName); +} + +void BindingManager::dumpWrapperMap() +{ + const auto &wrapperMap = m_d->wrapperMapper; + std::cerr << "-------------------------------\n" + << "WrapperMap size: " << wrapperMap.size() << " Types: " + << m_d->classHierarchy.nodeSet().size() << '\n'; + for (auto it = wrapperMap.begin(), end = wrapperMap.end(); it != end; ++it) { + const SbkObject *sbkObj = it->second; + std::cerr << "key: " << it->first << ", value: " + << static_cast<const void *>(sbkObj) << " (" + << (Py_TYPE(sbkObj))->tp_name << ", refcnt: " + << Py_REFCNT(reinterpret_cast<const PyObject *>(sbkObj)) << ")\n"; + } + std::cerr << "-------------------------------\n"; +} + +static bool isPythonType(PyTypeObject *type) +{ + // This is a type which should be called by multiple inheritance. + // It is either a pure Python type or a derived PySide type. + return !ObjectType::checkType(type) || ObjectType::isUserType(type); +} + +bool callInheritedInit(PyObject *self, PyObject *args, PyObject *kwds, + const char *fullName) +{ + using Shiboken::AutoDecRef; + + static PyObject *const _init = String::createStaticString("__init__"); + static PyObject *objectInit = + PyObject_GetAttr(reinterpret_cast<PyObject *>(&PyBaseObject_Type), _init); + + // A native C++ self cannot have multiple inheritance. + if (!Object::isUserType(self)) + return false; + + auto *startType = Py_TYPE(self); + auto *mro = startType->tp_mro; + Py_ssize_t idx, n = PyTuple_GET_SIZE(mro); + auto classNameLen = std::strrchr(fullName, '.') - fullName; + /* No need to check the last one: it's gonna be skipped anyway. */ + for (idx = 0; idx + 1 < n; ++idx) { + auto *lookType = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(mro, idx)); + const char *lookName = lookType->tp_name; + auto lookLen = long(std::strlen(lookName)); + if (std::strncmp(lookName, fullName, classNameLen) == 0 && lookLen == classNameLen) + break; + } + // We are now at the first non-Python class `QObject`. + // mro: ('C', 'A', 'QObject', 'Object', 'B', 'object') + // We want to catch class `B` and call its `__init__`. + for (idx += 1; idx + 1 < n; ++idx) { + auto *t = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(mro, idx)); + if (isPythonType(t)) + break; } + if (idx >= n) + return false; + + auto *obSubType = PyTuple_GET_ITEM(mro, idx); + auto *subType = reinterpret_cast<PyTypeObject *>(obSubType); + if (subType == &PyBaseObject_Type) + return false; + AutoDecRef func(PyObject_GetAttr(obSubType, _init)); + // PYSIDE-2654: If this has no implementation then we get object.__init__ + // but that is the same case like above. + if (func == objectInit) + return false; + // PYSIDE-2294: We need to explicitly ignore positional args in a mixin class. + SBK_UNUSED(args); + AutoDecRef newArgs(PyTuple_New(1)); + auto *newArgsOb = newArgs.object(); + Py_INCREF(self); + PyTuple_SET_ITEM(newArgsOb, 0, self); + // Note: This can fail, so please always check the error status. + AutoDecRef result(PyObject_Call(func, newArgs, kwds)); + return true; } } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/bindingmanager.h b/sources/shiboken6/libshiboken/bindingmanager.h index e299dad96..54c4e486a 100644 --- a/sources/shiboken6/libshiboken/bindingmanager.h +++ b/sources/shiboken6/libshiboken/bindingmanager.h @@ -5,17 +5,23 @@ #define BINDINGMANAGER_H #include "sbkpython.h" -#include <set> #include "shibokenmacros.h" +#include <set> +#include <utility> + struct SbkObject; namespace Shiboken { +namespace Module { +struct TypeInitStruct; +} + struct DestructorEntry; -typedef void (*ObjectVisitor)(SbkObject *, void *); +using ObjectVisitor = void (*)(SbkObject *, void *); class LIBSHIBOKEN_API BindingManager { @@ -38,7 +44,15 @@ public: SbkObject *retrieveWrapper(const void *cptr); PyObject *getOverride(const void *cptr, PyObject *nameCache[], const char *methodName); - void addClassInheritance(PyTypeObject *parent, PyTypeObject *child); + void addClassInheritance(Module::TypeInitStruct *parent, Module::TypeInitStruct *child); + /// Try to find the correct type of cptr via type discovery knowing that it's at least + /// of type \p type. If a derived class is found, it returns a cptr cast to the type + /// (which may be different in case of multiple inheritance. + /// \param cptr a pointer to the instance of type \p type + /// \param type type of cptr + using TypeCptrPair = std::pair<PyTypeObject *, void *>; + TypeCptrPair findDerivedType(void *cptr, PyTypeObject *type) const; + /** * Try to find the correct type of *cptr knowing that it's at least of type \p type. * In case of multiple inheritance this function may change the contents of cptr. @@ -46,7 +60,7 @@ public: * \param type type of *cptr * \warning This function is slow, use it only as last resort. */ - PyTypeObject *resolveType(void **cptr, PyTypeObject *type); + [[deprecated]] PyTypeObject *resolveType(void **cptr, PyTypeObject *type); std::set<PyObject *> getAllPyObjects(); @@ -59,6 +73,9 @@ public: */ void visitAllPyObjects(ObjectVisitor visitor, void *data); + bool dumpTypeGraph(const char *fileName) const; + void dumpWrapperMap(); + private: ~BindingManager(); BindingManager(); @@ -67,6 +84,9 @@ private: BindingManagerPrivate *m_d; }; +LIBSHIBOKEN_API bool callInheritedInit(PyObject *self, PyObject *args, PyObject *kwds, + const char *fullName); + } // namespace Shiboken #endif // BINDINGMANAGER_H diff --git a/sources/shiboken6/libshiboken/bufferprocs_py37.h b/sources/shiboken6/libshiboken/bufferprocs_py37.h index 06b42cabd..e16194e50 100644 --- a/sources/shiboken6/libshiboken/bufferprocs_py37.h +++ b/sources/shiboken6/libshiboken/bufferprocs_py37.h @@ -67,8 +67,8 @@ typedef struct bufferinfo { void *internal; } Pep_buffer; -typedef int (*getbufferproc)(PyObject *, Pep_buffer *, int); -typedef void (*releasebufferproc)(PyObject *, Pep_buffer *); +using getbufferproc =int (*)(PyObject *, Pep_buffer *, int); +using releasebufferproc = void (*)(PyObject *, Pep_buffer *); /* Maximum number of dimensions */ #define PyBUF_MAX_NDIM 64 diff --git a/sources/shiboken6/libshiboken/debugfreehook.cpp b/sources/shiboken6/libshiboken/debugfreehook.cpp index 1a80a2514..13df6bd6c 100644 --- a/sources/shiboken6/libshiboken/debugfreehook.cpp +++ b/sources/shiboken6/libshiboken/debugfreehook.cpp @@ -6,8 +6,8 @@ #include "gilstate.h" #if defined(_WIN32) && defined(_DEBUG) -#include <crtdbg.h> -#include <windows.h> +# include <sbkwindows.h> +# include <crtdbg.h> #endif #ifdef __GLIBC__ diff --git a/sources/shiboken6/libshiboken/embed/embedding_generator.py b/sources/shiboken6/libshiboken/embed/embedding_generator.py index 96f66b949..a058fd372 100644 --- a/sources/shiboken6/libshiboken/embed/embedding_generator.py +++ b/sources/shiboken6/libshiboken/embed/embedding_generator.py @@ -122,22 +122,23 @@ def _embed_file(fin, fout): limit = 50 text = fin.readlines() print(textwrap.dedent(""" - /* - * This is a ZIP archive of all Python files in the directory - * "shiboken6/shibokenmodule/files.dir/shibokensupport/signature" - * There is also a toplevel file "signature_bootstrap.py[c]" that will be - * directly executed from C++ as a bootstrap loader. - */ + // This is a ZIP archive of all Python files in the directory + // "shiboken6/shibokenmodule/files.dir/shibokensupport/signature" + // There is also a toplevel file "signature_bootstrap.py[c]" that will be + // directly executed from C++ as a bootstrap loader. """).strip(), file=fout) block, blocks = 0, len(text) // limit + 1 for idx, line in enumerate(text): if idx % limit == 0: + if block: + fout.write(')"\n') comma = "," if block else "" block += 1 - print(file=fout) - print(f'/* Block {block} of {blocks} */{comma}', file=fout) - print(f'\"{line.strip()}\"', file=fout) - print(f'/* Sentinel */, \"\"', file=fout) + fout.write(f'\n{comma} // Block {block} of {blocks}\nR"(') + else: + fout.write('\n') + fout.write(line.strip()) + fout.write(')"\n\n/* Sentinel */, ""\n') def _embed_bytefile(fin, fout, is_text): diff --git a/sources/shiboken6/libshiboken/embed/signature_bootstrap.py b/sources/shiboken6/libshiboken/embed/signature_bootstrap.py index c11a0367a..37f95cd35 100644 --- a/sources/shiboken6/libshiboken/embed/signature_bootstrap.py +++ b/sources/shiboken6/libshiboken/embed/signature_bootstrap.py @@ -26,6 +26,7 @@ recursion_trap = 0 import base64 import importlib import io +import os import sys import traceback import zipfile @@ -61,21 +62,83 @@ def bootstrap(): import shibokensupport yield except Exception as e: - print("Problem importing shibokensupport:") - print(f"{e.__class__.__name__}: {e}") + f = sys.stderr + print("Problem importing shibokensupport:", file=f) + print(f"{e.__class__.__name__}: {e}", file=f) traceback.print_exc() - print("sys.path:") + print("sys.path:", file=f) for p in sys.path: - print(" " + p) - sys.stdout.flush() + print(" " + p, file=f) + f.flush() sys.exit(-1) target.remove(support_path) - target, support_path = prepare_zipfile() + # Here we decide if we re-incarnate the embedded files or use embedding. + incarnated = find_incarnated_files() + if incarnated: + target, support_path = sys.path, os.fspath(incarnated) + else: + target, support_path = prepare_zipfile() with ensure_shibokensupport(target, support_path): from shibokensupport.signature import loader return loader +# Newer functionality: +# This function checks if the support directory exist and returns it. +# If does not exist, we try to create it and return it. +# Otherwise, we return None. + +def find_incarnated_files(): + import shiboken6 as root + files_dir = Path(root.__file__).resolve().parent / "files.dir" + handle_embedding_switch(files_dir) + if files_dir.exists(): + sys.path.insert(0, os.fspath(files_dir)) + # Note: To avoid recursion problems, we need to preload the loader. + # But that has the side-effect that we need to delay the feature + # initialization until all function pointers are set. + # See `post_init_func` in signature_globals.cpp . + import shibokensupport.signature.loader + del sys.path[0] + return files_dir + return None + + +def handle_embedding_switch(files_dir): + """ + This handles the optional environment variable `SBK_EMBED` + if not set : do nothing + if set to 0, false, no : de-virtualize the Python files + if set to 1, true, yes : virtualize again (delete "files.dir") + """ + env_name = "SBK_EMBED" + env_var = os.environ.get(env_name) + if not env_var: + return + if env_var.lower() in ("1", "t", "true", "y", "yes"): + import shutil + shutil.rmtree(files_dir, ignore_errors=True) + elif env_var.lower() in ("0", "f", "false", "n", "no"): + reincarnate_files(files_dir) + + +def reincarnate_files(files_dir): + target, zip = prepare_zipfile() + names = (_ for _ in zip.zfile.namelist() if _.endswith(".py")) + try: + # First check mkdir to get an error when we cannot write. + files_dir.mkdir(exist_ok=True) + except os.error as e: + print(f"SBK_EMBED=False: Warning: Cannot write into {files_dir}") + return None + try: + # Then check for a real error when unpacking the zip file. + zip.zfile.extractall(path=files_dir, members=names) + return files_dir + except Exception as e: + print(f"{e.__class__.__name__}: {e}", file=sys.stderr) + traceback.print_exc() + raise # New functionality: Loading from a zip archive. # There exists the zip importer, but as it is written, only real zip files are diff --git a/sources/shiboken6/libshiboken/helper.cpp b/sources/shiboken6/libshiboken/helper.cpp index 8f836316e..46af68956 100644 --- a/sources/shiboken6/libshiboken/helper.cpp +++ b/sources/shiboken6/libshiboken/helper.cpp @@ -5,28 +5,60 @@ #include "basewrapper_p.h" #include "sbkstring.h" #include "sbkstaticstrings.h" +#include "sbkstaticstrings.h" +#include "pep384impl.h" + +#include <algorithm> +#include <optional> #include <iomanip> #include <iostream> - +#include <climits> +#include <cstring> #include <cstdarg> +#include <cctype> #ifdef _WIN32 -# ifndef NOMINMAX -# define NOMINMAX -# endif -# include <windows.h> +# include <sbkwindows.h> #else # include <pthread.h> #endif -#include <algorithm> +static std::optional<std::string> getStringAttr(PyObject *obj, const char *what) +{ + if (PyObject_HasAttrString(obj, what) != 0) { // Check first to suppress error. + Shiboken::AutoDecRef result(PyObject_GetAttrString(obj, what)); + if (PyUnicode_Check(result.object()) != 0) + return _PepUnicode_AsString(result.object()); + } + return std::nullopt; +} -static void formatPyTypeObject(const PyTypeObject *obj, std::ostream &str) +static std::optional<int> getIntAttr(PyObject *obj, const char *what) { - if (obj) { - str << '"' << obj->tp_name << "\", 0x" << std::hex - << obj->tp_flags << std::dec; + if (PyObject_HasAttrString(obj, what) != 0) { // Check first to suppress error. + Shiboken::AutoDecRef result(PyObject_GetAttrString(obj, what)); + if (PyLong_Check(result.object()) != 0) + return PyLong_AsLong(result.object()); + } + return std::nullopt; +} + +static bool verbose = false; + +static void formatTypeTuple(PyObject *t, const char *what, std::ostream &str); + +static void formatPyTypeObject(const PyTypeObject *obj, std::ostream &str, bool verbose) +{ + if (obj == nullptr) { + str << '0'; + return; + } + + str << '"' << obj->tp_name << '"'; + if (verbose) { + bool immutableType = false; + str << ", 0x" << std::hex << obj->tp_flags << std::dec; if (obj->tp_flags & Py_TPFLAGS_HEAPTYPE) str << " [heaptype]"; if (obj->tp_flags & Py_TPFLAGS_BASETYPE) @@ -49,30 +81,59 @@ static void formatPyTypeObject(const PyTypeObject *obj, std::ostream &str) str << " [type]"; if (obj->tp_flags & Py_TPFLAGS_IS_ABSTRACT) str << " [abstract]"; -#if PY_VERSION_HEX >= 0x03080000 + if (obj->tp_flags & Py_TPFLAGS_READY) + str << " [ready]"; + if (obj->tp_flags & Py_TPFLAGS_READYING) + str << " [readying]"; if (obj->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) str << " [method_descriptor]"; -# if PY_VERSION_HEX >= 0x03090000 -# ifndef Py_LIMITED_API +# ifndef Py_LIMITED_API if (obj->tp_flags & Py_TPFLAGS_HAVE_VECTORCALL) str << " [vectorcall]"; -# endif // !Py_LIMITED_API -# if PY_VERSION_HEX >= 0x030A0000 - if (obj->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) +# endif // !Py_LIMITED_API +# if PY_VERSION_HEX >= 0x030A0000 + immutableType = (obj->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) != 0; + if (immutableType) str << " [immutabletype]"; if (obj->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION) str << " [disallow_instantiation]"; -# ifndef Py_LIMITED_API +# ifndef Py_LIMITED_API if (obj->tp_flags & Py_TPFLAGS_MAPPING) str << " [mapping]"; if (obj->tp_flags & Py_TPFLAGS_SEQUENCE) str << " [sequence]"; # endif // !Py_LIMITED_API -# endif // 3.10 -# endif // 3.9 -#endif // 3.8 - } else { - str << '0'; +# endif // 3.10 + if (obj->tp_basicsize != 0) + str << ", basicsize=" << obj->tp_basicsize; + if (verbose) { + formatTypeTuple(obj->tp_bases, "bases", str); + formatTypeTuple(obj->tp_mro, "mro", str); + if (!immutableType) { + auto *underlying = reinterpret_cast<const PyObject *>(obj)->ob_type; + if (underlying != nullptr && underlying != obj) { + str << ", underlying=\"" << underlying->tp_name << '"'; + } + } + } + } +} + +static void formatTypeTuple(PyObject *t, const char *what, std::ostream &str) +{ + const Py_ssize_t size = t != nullptr && PyTuple_Check(t) != 0 ? PyTuple_Size(t) : 0; + if (size > 0) { + str << ", " << what << "=[" << size << "]{"; + for (Py_ssize_t i = 0; i < size; ++i) { + if (i != 0) + str << ", "; + Shiboken::AutoDecRef item(PyTuple_GetItem(t, i)); + if (item.isNull()) + str << '0'; // Observed with non-ready types + else + str << '"' << reinterpret_cast<PyTypeObject *>(item.object())->tp_name << '"'; + } + str << '}'; } } @@ -151,13 +212,17 @@ static void formatPyUnicode(PyObject *obj, std::ostream &str) { // Note: The below call create the PyCompactUnicodeObject.utf8 representation str << '"' << _PepUnicode_AsString(obj) << '"'; + if (!verbose) + return; str << " (" << PyUnicode_GetLength(obj) << ')'; const auto kind = _PepUnicode_KIND(obj); switch (kind) { +#if PY_VERSION_HEX < 0x030C0000 case PepUnicode_WCHAR_KIND: str << " [wchar]"; break; +#endif case PepUnicode_1BYTE_KIND: str << " [1byte]"; break; @@ -178,8 +243,10 @@ static void formatPyUnicode(PyObject *obj, std::ostream &str) void *data =_PepUnicode_DATA(obj); str << ", data="; switch (kind) { +#if PY_VERSION_HEX < 0x030C0000 case PepUnicode_WCHAR_KIND: formatCharSequence(reinterpret_cast<const wchar_t *>(data), str); +#endif break; case PepUnicode_1BYTE_KIND: formatCharSequence(reinterpret_cast<const Py_UCS1 *>(data), str); @@ -208,22 +275,92 @@ static void formatPyUnicode(PyObject *obj, std::ostream &str) #endif // !Py_LIMITED_API } +static std::string getQualName(PyObject *obj) +{ + Shiboken::AutoDecRef result(PyObject_GetAttr(obj, Shiboken::PyMagicName::qualname())); + return result.object() != nullptr + ? _PepUnicode_AsString(result.object()) : std::string{}; +} + +static void formatPyFunction(PyObject *obj, std::ostream &str) +{ + str << '"' << getQualName(obj) << "()\""; +} + +static void formatPyMethod(PyObject *obj, std::ostream &str) +{ + if (auto *func = PyMethod_Function(obj)) + formatPyFunction(func, str); + str << ", instance=" << PyMethod_Self(obj); +} + +static void formatPyCodeObject(PyObject *obj, std::ostream &str) +{ + if (auto name = getStringAttr(obj, "co_name")) + str << '"' << name.value() << '"'; + if (auto qualName = getStringAttr(obj, "co_qualname")) + str << ", co_qualname=\"" << qualName.value() << '"'; + if (auto flags = getIntAttr(obj, "co_flags")) + str << ", flags=0x" << std::hex << flags.value() << std::dec; + if (auto c = getIntAttr(obj, "co_argcount")) + str << ", co_argcounts=" << c.value(); + if (auto c = getIntAttr(obj, "co_posonlyargcount")) + str << ", co_posonlyargcount=" << c.value(); + if (auto c = getIntAttr(obj, "co_kwonlyargcount")) + str << ", co_kwonlyargcount=" << c.value(); + if (auto fileName = getStringAttr(obj, "co_filename")) { + str << " @" << fileName.value(); + if (auto l = getIntAttr(obj, "co_firstlineno")) + str << ':'<< l.value(); + } +} + static void formatPyObjectHelper(PyObject *obj, std::ostream &str) { - str << ", refs=" << obj->ob_refcnt << ", "; + str << ", "; + if (obj == Py_None) { + str << "None"; + return; + } + if (obj == Py_True) { + str << "True"; + return; + } + if (obj == Py_False) { + str << "False"; + return; + } + const auto refs = Py_REFCNT(obj); + if (refs == UINT_MAX) // _Py_IMMORTAL_REFCNT + str << "immortal, "; + else + str << "refs=" << refs << ", "; if (PyType_Check(obj)) { str << "type: "; - formatPyTypeObject(reinterpret_cast<PyTypeObject *>(obj), str); + formatPyTypeObject(reinterpret_cast<PyTypeObject *>(obj), str, true); return; } - formatPyTypeObject(obj->ob_type, str); + formatPyTypeObject(obj->ob_type, str, false); str << ", "; - if (PyLong_Check(obj)) - str << PyLong_AsLong(obj); + if (PyLong_Check(obj)) { + const auto llv = PyLong_AsLongLong(obj); + if (PyErr_Occurred() != PyExc_OverflowError) { + str << llv; + } else { + PyErr_Clear(); + str << "0x" << std::hex << PyLong_AsUnsignedLongLong(obj) << std::dec; + } + } else if (PyFloat_Check(obj)) str << PyFloat_AsDouble(obj); else if (PyUnicode_Check(obj)) formatPyUnicode(obj, str); + else if (PyFunction_Check(obj) != 0) + formatPyFunction(obj, str); + else if (PyMethod_Check(obj) != 0) + formatPyMethod(obj, str); + else if (PepCode_Check(obj) != 0) + formatPyCodeObject(obj, str); else if (PySequence_Check(obj)) formatPySequence(obj, str); else if (PyDict_Check(obj)) @@ -263,7 +400,7 @@ debugPyBuffer::debugPyBuffer(const Py_buffer &b) : m_buffer(b) std::ostream &operator<<(std::ostream &str, const debugPyTypeObject &o) { str << "PyTypeObject("; - formatPyTypeObject(o.m_object, str); + formatPyTypeObject(o.m_object, str, true); str << ')'; return str; } @@ -299,6 +436,18 @@ std::ostream &operator<<(std::ostream &str, const debugPyBuffer &b) return str; } +std::ios_base &debugVerbose(std::ios_base &s) +{ + verbose = true; + return s; +} + +std::ios_base &debugBrief(std::ios_base &s) +{ + verbose = false; + return s; +} + #ifdef _WIN32 // Converts a Unicode string to a string encoded in the Windows console's // code page via wchar_t for use with argv (PYSIDE-1425). @@ -442,4 +591,47 @@ ThreadId mainThreadId() return _mainThreadId; } +const char *typeNameOf(const char *typeIdName) +{ + auto size = std::strlen(typeIdName); +#if defined(Q_CC_MSVC) // MSVC: "class QPaintDevice * __ptr64" + if (auto *lastStar = strchr(typeName, '*')) { + // MSVC: "class QPaintDevice * __ptr64" + while (*--lastStar == ' ') { + } + size = lastStar - typeName + 1; + } +#else // g++, Clang: "QPaintDevice *" -> "P12QPaintDevice" + if (size > 2 && typeIdName[0] == 'P' && std::isdigit(typeIdName[1])) { + ++typeIdName; + --size; + } +#endif + char *result = new char[size + 1]; + result[size] = '\0'; + std::memcpy(result, typeIdName, size); + return result; +} + +#if !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030A0000 && !defined(PYPY_VERSION) +static int _getPyVerbose() +{ + PyConfig config; + PyConfig_InitPythonConfig(&config); + return config.verbose; +} +#endif // !Py_LIMITED_API >= 3.10 + +int pyVerbose() +{ +#ifdef Py_LIMITED_API + return Pep_GetVerboseFlag(); +#elif PY_VERSION_HEX >= 0x030A0000 && !defined(PYPY_VERSION) + static const int result = _getPyVerbose(); + return result; +#else + return Py_VerboseFlag; +#endif +} + } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/helper.h b/sources/shiboken6/libshiboken/helper.h index 265bb6581..f226e8c24 100644 --- a/sources/shiboken6/libshiboken/helper.h +++ b/sources/shiboken6/libshiboken/helper.h @@ -36,6 +36,10 @@ LIBSHIBOKEN_API bool listToArgcArgv(PyObject *argList, int *argc, char ***argv, */ LIBSHIBOKEN_API int *sequenceToIntArray(PyObject *obj, bool zeroTerminated = false); +/// Fix a type name returned by typeid(t).name(), depending on compiler. +/// \returns Fixed name (allocated). +LIBSHIBOKEN_API const char *typeNameOf(const char *typeIdName); + /** * Creates and automatically deallocates C++ arrays. */ @@ -61,6 +65,8 @@ using ThreadId = unsigned long long; LIBSHIBOKEN_API ThreadId currentThreadId(); LIBSHIBOKEN_API ThreadId mainThreadId(); +LIBSHIBOKEN_API int pyVerbose(); + /** * An utility function used to call PyErr_WarnEx with a formatted message. */ @@ -106,7 +112,8 @@ LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugSbkObject LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugPyTypeObject &o); LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugPyBuffer &b); LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugPyArrayObject &b); - +LIBSHIBOKEN_API std::ios_base &debugVerbose(std::ios_base &s); +LIBSHIBOKEN_API std::ios_base &debugBrief(std::ios_base &s); } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/pep384_issue33738.cpp b/sources/shiboken6/libshiboken/pep384_issue33738.cpp deleted file mode 100644 index 7f3872a58..000000000 --- a/sources/shiboken6/libshiboken/pep384_issue33738.cpp +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (C) 2018 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only - -// There is a bug in Python 3.6 that turned the Index_Check function -// into a macro without taking care of the limited API. -// This leads to the single problem that we don't have -// access to PyLong_Type's nb_index field which is no heap type. -// We cannot easily create this function by inheritance since it is -// not inherited. -// -// Simple solution: Create the structure and write such a function. -// Long term: Submit a patch to python.org . - -// This structure comes from Python 3.7, but we have checked that -// it also works for Python 3.8 and 3.9. - -typedef struct { - /* Number implementations must check *both* - arguments for proper type and implement the necessary conversions - in the slot functions themselves. */ - - binaryfunc nb_add; - binaryfunc nb_subtract; - binaryfunc nb_multiply; - binaryfunc nb_remainder; - binaryfunc nb_divmod; - ternaryfunc nb_power; - unaryfunc nb_negative; - unaryfunc nb_positive; - unaryfunc nb_absolute; - inquiry nb_bool; - unaryfunc nb_invert; - binaryfunc nb_lshift; - binaryfunc nb_rshift; - binaryfunc nb_and; - binaryfunc nb_xor; - binaryfunc nb_or; - unaryfunc nb_int; - void *nb_reserved; /* the slot formerly known as nb_long */ - unaryfunc nb_float; - - binaryfunc nb_inplace_add; - binaryfunc nb_inplace_subtract; - binaryfunc nb_inplace_multiply; - binaryfunc nb_inplace_remainder; - ternaryfunc nb_inplace_power; - binaryfunc nb_inplace_lshift; - binaryfunc nb_inplace_rshift; - binaryfunc nb_inplace_and; - binaryfunc nb_inplace_xor; - binaryfunc nb_inplace_or; - - binaryfunc nb_floor_divide; - binaryfunc nb_true_divide; - binaryfunc nb_inplace_floor_divide; - binaryfunc nb_inplace_true_divide; - - unaryfunc nb_index; - - binaryfunc nb_matrix_multiply; - binaryfunc nb_inplace_matrix_multiply; -} PyNumberMethods; - -// temporary structure until we have a generator for the offsets -typedef struct _oldtypeobject { - PyVarObject ob_base; - void *X01; // const char *tp_name; - void *X02; // Py_ssize_t tp_basicsize; - void *X03; // Py_ssize_t tp_itemsize; - void *X04; // destructor tp_dealloc; - void *X05; // printfunc tp_print; - void *X06; // getattrfunc tp_getattr; - void *X07; // setattrfunc tp_setattr; - void *X08; // PyAsyncMethods *tp_as_async; - void *X09; // reprfunc tp_repr; - PyNumberMethods *tp_as_number; - -} PyOldTypeObject; - -static bool is_compatible_version() -{ - auto *sysmodule = PyImport_AddModule("sys"); - auto *dic = PyModule_GetDict(sysmodule); - auto *version = PyDict_GetItemString(dic, "version_info"); - auto *major = PyTuple_GetItem(version, 0); - auto *minor = PyTuple_GetItem(version, 1); - auto number = PyLong_AsLong(major) * 1000 + PyLong_AsLong(minor); - return number < 3010; -} - -/////////////////////////////////////////////////////////////////////// -// -// PYSIE-1797: The Solution -// ======================== -// -// Inspecting the data structures of Python 3.6, 3.7, 3.8 and 3.9 -// shows that concerning the here needed offset of nb_index, they -// are all compatible. -// That means: We can use the above definition for all these versions. -// -// From Python 3.10 on, the `PyType_GetSlot` function also works with -// non-heap types. That means this solution will always work. -// -// Note: When we have moved to Python 3.8 as the minimum version, -// this whole nonsense can be trashed. -// There is an automatic warning about this in parser.py . -// - -LIBSHIBOKEN_API int PepIndex_Check(PyObject *obj) -{ - static bool old_python_version = is_compatible_version(); - if (old_python_version) { - auto *type = reinterpret_cast<PyOldTypeObject *>(Py_TYPE(obj)); - return type->tp_as_number != nullptr && - type->tp_as_number->nb_index != nullptr; - } - // From Python 3.10 on, we can use PyType_GetSlot also with normal types! - unaryfunc nb_index = reinterpret_cast<unaryfunc>(PyType_GetSlot(Py_TYPE(obj), Py_nb_index)); - return nb_index != nullptr; -} - diff --git a/sources/shiboken6/libshiboken/pep384ext.h b/sources/shiboken6/libshiboken/pep384ext.h new file mode 100644 index 000000000..021c53d16 --- /dev/null +++ b/sources/shiboken6/libshiboken/pep384ext.h @@ -0,0 +1,89 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef PEP384EXT_H +#define PEP384EXT_H + +#include "pep384impl.h" + +/// Returns the allocator slot of the PyTypeObject. +inline allocfunc PepExt_Type_GetAllocSlot(PyTypeObject *t) +{ + return reinterpret_cast<allocfunc>(PepType_GetSlot(t, Py_tp_alloc)); +} + +/// Invokes the allocator slot of the PyTypeObject. +template <class Type> +inline Type *PepExt_TypeCallAlloc(PyTypeObject *t, Py_ssize_t nitems) +{ + PyObject *result = PepExt_Type_GetAllocSlot(t)(t, nitems); + return reinterpret_cast<Type *>(result); +} + +/// Returns the getattro slot of the PyTypeObject. +inline getattrofunc PepExt_Type_GetGetAttroSlot(PyTypeObject *t) +{ + return reinterpret_cast<getattrofunc>(PepType_GetSlot(t, Py_tp_getattro)); +} + +/// Returns the setattro slot of the PyTypeObject. +inline setattrofunc PepExt_Type_GetSetAttroSlot(PyTypeObject *t) +{ + return reinterpret_cast<setattrofunc>(PepType_GetSlot(t, Py_tp_setattro)); +} + +/// Returns the descr_get slot of the PyTypeObject. +inline descrgetfunc PepExt_Type_GetDescrGetSlot(PyTypeObject *t) +{ + return reinterpret_cast<descrgetfunc>(PepType_GetSlot(t, Py_tp_descr_get)); +} + +/// Invokes the descr_get slot of the PyTypeObject. +inline PyObject *PepExt_Type_CallDescrGet(PyObject *self, PyObject *obj, PyObject *type) +{ + return PepExt_Type_GetDescrGetSlot(Py_TYPE(self))(self, obj, type); +} + +/// Returns the descr_set slot of the PyTypeObject. +inline descrsetfunc PepExt_Type_GetDescrSetSlot(PyTypeObject *t) +{ + return reinterpret_cast<descrsetfunc>(PepType_GetSlot(t, Py_tp_descr_set)); +} + +/// Returns the call slot of the PyTypeObject. +inline ternaryfunc PepExt_Type_GetCallSlot(PyTypeObject *t) +{ + return reinterpret_cast<ternaryfunc>(PepType_GetSlot(t, Py_tp_call)); +} + +/// Returns the new slot of the PyTypeObject. +inline newfunc PepExt_Type_GetNewSlot(PyTypeObject *t) +{ + return reinterpret_cast<newfunc>(PepType_GetSlot(t, Py_tp_new)); +} + +/// Returns the init slot of the PyTypeObject. +inline initproc PepExt_Type_GetInitSlot(PyTypeObject *t) +{ + return reinterpret_cast<initproc>(PepType_GetSlot(t, Py_tp_init)); +} + +/// Returns the free slot of the PyTypeObject. +inline freefunc PepExt_Type_GetFreeSlot(PyTypeObject *t) +{ + return reinterpret_cast<freefunc>(PepType_GetSlot(t, Py_tp_free)); +} + +/// Invokes the free slot of the PyTypeObject. +inline void PepExt_TypeCallFree(PyTypeObject *t, void *object) +{ + PepExt_Type_GetFreeSlot(t)(object); +} + +/// Invokes the free slot of the PyTypeObject. +inline void PepExt_TypeCallFree(PyObject *object) +{ + PepExt_Type_GetFreeSlot(Py_TYPE(object))(object); +} + +#endif // PEP384EXT_H diff --git a/sources/shiboken6/libshiboken/pep384impl.cpp b/sources/shiboken6/libshiboken/pep384impl.cpp index cdf3d96dc..4b3759456 100644 --- a/sources/shiboken6/libshiboken/pep384impl.cpp +++ b/sources/shiboken6/libshiboken/pep384impl.cpp @@ -1,6 +1,8 @@ -// Copyright (C) 2018 The Qt Company Ltd. +// Copyright (C) 2023 The Qt Company Ltd. // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#define PEP384_INTERN + #include "sbkpython.h" #include "autodecref.h" #include "sbkstaticstrings.h" @@ -8,8 +10,6 @@ #include "basewrapper.h" #include "basewrapper_p.h" #include "sbkenum.h" -#include "sbkenum_p.h" -#include "sbkconverter.h" #include "voidptr.h" #include <cstdlib> @@ -19,7 +19,7 @@ extern "C" { /* - * The documentation is located in pep384impl_doc.rst + * The documentation is located in `sources/pyside6/doc/developer/limited_api.rst`. * Here is the verification code for PyTypeObject. * We create a type object and check if its fields @@ -37,16 +37,16 @@ dummy_func(PyObject * /* self */, PyObject * /* args */) } static struct PyMethodDef probe_methoddef[] = { - {"dummy", dummy_func, METH_NOARGS}, - {nullptr} + {"dummy", dummy_func, METH_NOARGS, nullptr}, + {nullptr, nullptr, 0, nullptr} }; static PyGetSetDef probe_getseters[] = { - {nullptr} /* Sentinel */ + {nullptr, nullptr, nullptr, nullptr, nullptr} /* Sentinel */ }; static PyMemberDef probe_members[] = { - {nullptr} /* Sentinel */ + {nullptr, 0, 0, 0, nullptr} /* Sentinel */ }; #define probe_tp_dealloc make_dummy(1) @@ -117,6 +117,8 @@ check_PyTypeObject_valid() PyObject *d = PyObject_GetAttr(obtype, Shiboken::PyMagicName::dictoffset()); long probe_tp_dictoffset = PyLong_AsLong(d); PyObject *probe_tp_mro = PyObject_GetAttr(obtype, Shiboken::PyMagicName::mro()); + Shiboken::AutoDecRef tpDict(PepType_GetDict(check)); + auto *checkDict = tpDict.object(); if (false || strcmp(probe_tp_name, check->tp_name) != 0 || probe_tp_basicsize != check->tp_basicsize @@ -133,8 +135,8 @@ check_PyTypeObject_valid() || probe_tp_methods != check->tp_methods || probe_tp_getset != check->tp_getset || probe_tp_base != typetype->tp_base - || !PyDict_Check(check->tp_dict) - || !PyDict_GetItemString(check->tp_dict, "dummy") + || !PyDict_Check(checkDict) + || !PyDict_GetItemString(checkDict, "dummy") || probe_tp_descr_get != check->tp_descr_get || probe_tp_descr_set != check->tp_descr_set || probe_tp_dictoffset != typetype->tp_dictoffset @@ -155,9 +157,6 @@ check_PyTypeObject_valid() Py_DECREF(probe_tp_mro); } -// PYSIDE-1797: This must be a runtime decision. -#include "pep384_issue33738.cpp" - #endif // Py_LIMITED_API /***************************************************************************** @@ -179,7 +178,7 @@ static PyObject * find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) { Py_ssize_t i, n; - PyObject *mro, *res, *base, *dict; + PyObject *mro, *res, *base; /* Look in tp_dict of types in MRO */ mro = type->tp_mro; @@ -193,9 +192,10 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) for (i = 0; i < n; i++) { base = PyTuple_GET_ITEM(mro, i); assert(PyType_Check(base)); - dict = ((PyTypeObject *)base)->tp_dict; - assert(dict && PyDict_Check(dict)); - res = PyDict_GetItem(dict, name); + auto *type = reinterpret_cast<PyTypeObject *>(base); + Shiboken::AutoDecRef dict(PepType_GetDict(type)); + assert(!dict.isNull() && PyDict_Check(dict.object())); + res = PyDict_GetItem(dict.object(), name); if (res != nullptr) break; if (PyErr_Occurred()) { @@ -251,7 +251,7 @@ _PepType_Lookup(PyTypeObject *type, PyObject *name) // structs and macros modelled after their equivalents in // cpython/Include/cpython/unicodeobject.h -struct PepASCIIObject +struct PepASCIIObject // since 3.12 { PyObject_HEAD Py_ssize_t length; /* Number of code points in the string */ @@ -264,18 +264,29 @@ struct PepASCIIObject unsigned int ready:1; unsigned int :24; } state; +}; + +struct PepASCIIObject_311 : public PepASCIIObject +{ wchar_t *wstr; /* wchar_t representation (null-terminated) */ }; -struct PepCompactUnicodeObject +struct PepCompactUnicodeObject // since 3.12 { PepASCIIObject _base; Py_ssize_t utf8_length; char *utf8; /* UTF-8 representation (null-terminated) */ +}; + +struct PepCompactUnicodeObject_311 // since 3.12 +{ + PepASCIIObject_311 _base; + Py_ssize_t utf8_length; + char *utf8; /* UTF-8 representation (null-terminated) */ Py_ssize_t wstr_length; /* Number of code points in wstr */ }; -struct PepUnicodeObject +struct PepUnicodeObject // since 3.12 { PepCompactUnicodeObject _base; union { @@ -286,6 +297,17 @@ struct PepUnicodeObject } data; /* Canonical, smallest-form Unicode buffer */ }; +struct PepUnicodeObject_311 +{ + PepCompactUnicodeObject_311 _base; + union { + void *any; + Py_UCS1 *latin1; + Py_UCS2 *ucs2; + Py_UCS4 *ucs4; + } data; /* Canonical, smallest-form Unicode buffer */ +}; + int _PepUnicode_KIND(PyObject *str) { return reinterpret_cast<PepASCIIObject *>(str)->state.kind; @@ -303,18 +325,33 @@ int _PepUnicode_IS_COMPACT(PyObject *str) return asciiObj->state.compact; } -static void *_PepUnicode_COMPACT_DATA(PyObject *str) +static void *_PepUnicode_ASCII_DATA(PyObject *str) { + if (_PepRuntimeVersion() < 0x030C00) { + auto *asciiObj_311 = reinterpret_cast<PepASCIIObject_311 *>(str); + return asciiObj_311 + 1; + } auto *asciiObj = reinterpret_cast<PepASCIIObject *>(str); - if (asciiObj->state.ascii) - return asciiObj + 1; + return asciiObj + 1; +} + +static void *_PepUnicode_COMPACT_DATA(PyObject *str) +{ + if (_PepUnicode_IS_ASCII(str) != 0) + return _PepUnicode_ASCII_DATA(str); + if (_PepRuntimeVersion() < 0x030C00) { + auto *compactObj_311 = reinterpret_cast<PepCompactUnicodeObject_311 *>(str); + return compactObj_311 + 1; + } auto *compactObj = reinterpret_cast<PepCompactUnicodeObject *>(str); return compactObj + 1; } static void *_PepUnicode_NONCOMPACT_DATA(PyObject *str) { - return reinterpret_cast<PepUnicodeObject *>(str)->data.any; + return _PepRuntimeVersion() < 0x030C00 + ? reinterpret_cast<PepUnicodeObject_311 *>(str)->data.any + : reinterpret_cast<PepUnicodeObject *>(str)->data.any; } void *_PepUnicode_DATA(PyObject *str) @@ -325,6 +362,23 @@ void *_PepUnicode_DATA(PyObject *str) // Fast path accessing UTF8 data without doing a conversion similar // to _PyUnicode_AsUTF8String +static const char *utf8FastPath_311(PyObject *str) +{ + if (PyUnicode_GetLength(str) == 0) + return ""; + auto *asciiObj = reinterpret_cast<PepASCIIObject_311 *>(str); + if (asciiObj->state.kind != PepUnicode_1BYTE_KIND || asciiObj->state.compact == 0) + return nullptr; // Empirical: PyCompactUnicodeObject.utf8 is only valid for 1 byte + if (asciiObj->state.ascii) { + auto *data = asciiObj + 1; + return reinterpret_cast<const char *>(data); + } + auto *compactObj = reinterpret_cast<PepCompactUnicodeObject_311 *>(str); + if (compactObj->utf8_length) + return compactObj->utf8; + return nullptr; +} + static const char *utf8FastPath(PyObject *str) { if (PyUnicode_GetLength(str) == 0) @@ -345,18 +399,21 @@ static const char *utf8FastPath(PyObject *str) const char *_PepUnicode_AsString(PyObject *str) { /* - * We need to keep the string alive but cannot borrow the Python object. - * Ugly easy way out: We re-code as an interned bytes string. This - * produces a pseudo-leak as long as there are new strings. - * Typically, this function is used for name strings, and the dict size - * will not grow so much. + * This function is the surrogate for PyUnicode_AsUTF8, which keeps the data + * in the unicode object as long as that object exists. + * + * The function does too much if not optimized by utf8, because it keeps the + * string alive, unconditionally. + * We should not rely on this behavior and think of PyUnicode_AsUTF8, only. */ #define STRINGIFY(x) #x #define TOSTRING(x) STRINGIFY(x) #define AT __FILE__ ":" TOSTRING(__LINE__) - if (const auto *utf8 = utf8FastPath(str)) + if (const auto *utf8 = _PepRuntimeVersion() < 0x030C00 + ? utf8FastPath_311(str) : utf8FastPath(str)) { return utf8; + } static PyObject *cstring_dict = nullptr; if (cstring_dict == nullptr) { @@ -447,8 +504,24 @@ PepCode_Get(PepCodeObject *co, const char *name) } return ret; } + +int PepCode_Check(PyObject *o) +{ + return o != nullptr && std::strcmp(Py_TYPE(o)->tp_name, "code") == 0 ? 1 : 0; +} + #endif // Py_LIMITED_API +#if defined(Py_LIMITED_API) || defined(PYPY_VERSION) +PyObject *PepFunction_GetDefaults(PyObject *function) +{ + auto *ob_ret = PyObject_GetAttrString(function, "__defaults__"); + Py_XDECREF(ob_ret); // returns borrowed ref + return ob_ret != Py_None ? ob_ret : nullptr; +} + +#endif // defined(Py_LIMITED_API) || defined(PYPY_VERSION) + /***************************************************************************** * * Support for datetime.h @@ -718,12 +791,37 @@ PepType_GetNameStr(PyTypeObject *type) return ret; } +// PYSIDE-2264: Find the _functools or functools module and retrieve the +// partial function. This can be tampered with, check carefully. +PyObject * +Pep_GetPartialFunction(void) +{ + static bool initialized = false; + static PyObject *result{}; + if (initialized) { + Py_INCREF(result); + return result; + } + auto *functools = PyImport_ImportModule("_functools"); + if (!functools) { + PyErr_Clear(); + functools = PyImport_ImportModule("functools"); + } + if (!functools) + Py_FatalError("functools cannot be found"); + result = PyObject_GetAttrString(functools, "partial"); + if (!result || !PyCallable_Check(result)) + Py_FatalError("partial not found or not a function"); + initialized = true; + return result; +} + /***************************************************************************** * * Newly introduced convenience functions * */ -#if PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) +#ifdef Py_LIMITED_API PyObject * PyImport_GetModule(PyObject *name) @@ -749,7 +847,7 @@ PyImport_GetModule(PyObject *name) return m; } -#endif // PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) +#endif // Py_LIMITED_API // 2020-06-16: For simplicity of creating arbitrary things, this function // is now made public. @@ -840,13 +938,13 @@ _Pep_PrivateMangle(PyObject *self, PyObject *name) wchar_t bigbuf[big_stack]; wchar_t *resbuf = amount <= big_stack ? bigbuf : (wchar_t *)malloc(sizeof(wchar_t) * amount); if (!resbuf) - return 0; + return nullptr; /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */ resbuf[0] = '_'; if (PyUnicode_AsWideChar(privateobj, resbuf + 1, ipriv + plen) < 0) - return 0; + return nullptr; if (PyUnicode_AsWideChar(name, resbuf + ipriv + plen + 1, nlen) < 0) - return 0; + return nullptr; PyObject *result = PyUnicode_FromWideChar(resbuf + ipriv, 1 + plen + nlen); if (amount > big_stack) free(resbuf); @@ -872,6 +970,21 @@ init_PepRuntime() PepRuntime_38_flag = 1; } +static long _GetPepRuntimeVersion() +{ + auto *version = PySys_GetObject("version_info"); + const auto major = PyLong_AsLong(PyTuple_GetItem(version, 0)); + const auto minor = PyLong_AsLong(PyTuple_GetItem(version, 1)); + const auto micro = PyLong_AsLong(PyTuple_GetItem(version, 2)); + return major << 16 | minor << 8 | micro; +} + +long _PepRuntimeVersion() +{ + static const auto number = _GetPepRuntimeVersion(); + return number; +} + /***************************************************************************** * * PYSIDE-535: Support for PyPy @@ -881,33 +994,104 @@ init_PepRuntime() * */ +/////////////////////////////////////////////////////////////////////// +// +// PEP 697: Support for embedded type structures. +// +// According to `https://docs.python.org/3/c-api/object.html?highlight=pyobject_gettypedata#c.PyObject_GetTypeData` +// the function `PyObject_GetTypeData` should belong to the Stable API +// since version 3.12.0, but it does not. We use instead some copies +// from Python source code. + +#if !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030C0000 + +# define PepObject_GetTypeData PyObject_GetTypeData + +SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *type) +{ + assert(SbkObjectType_Check(type)); + auto *obType = reinterpret_cast<PyObject *>(type); + void *data = PyObject_GetTypeData(obType, Py_TYPE(obType)); + return reinterpret_cast<SbkObjectTypePrivate *>(data); +} + +void PepType_SOTP_delete(PyTypeObject * /*type*/) +{ +} + +#else + +// The following comments are directly copied from Python 3.12 +// + +// Make sure we have maximum alignment, even if the current compiler +// does not support max_align_t. Note that: +// - Autoconf reports alignment of unknown types to 0. +// - 'long double' has maximum alignment on *most* platforms, +// looks like the best we can do for pre-C11 compilers. +// - The value is tested, see test_alignof_max_align_t +# if !defined(ALIGNOF_MAX_ALIGN_T) || ALIGNOF_MAX_ALIGN_T == 0 +# undef ALIGNOF_MAX_ALIGN_T +# define ALIGNOF_MAX_ALIGN_T alignof(long double) +# endif + +/* Align up to the nearest multiple of alignof(max_align_t) + * (like _Py_ALIGN_UP, but for a size rather than pointer) + */ +static Py_ssize_t _align_up(Py_ssize_t size) +{ + return (size + ALIGNOF_MAX_ALIGN_T - 1) & ~(ALIGNOF_MAX_ALIGN_T - 1); +} + +static void *PepObject_GetTypeData(PyObject *obj, PyTypeObject *cls) +{ + assert(PyObject_TypeCheck(obj, cls)); + return reinterpret_cast<char *>(obj) + _align_up(cls->tp_base->tp_basicsize); +} +// +/////////////////////////////////////////////////////////////////////// + /* * PyTypeObject extender */ + static std::unordered_map<PyTypeObject *, SbkObjectTypePrivate > SOTP_extender{}; static thread_local PyTypeObject *SOTP_key{}; static thread_local SbkObjectTypePrivate *SOTP_value{}; -SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *sbkType) +SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *type) { - if (sbkType == SOTP_key) + static bool use_312 = _PepRuntimeVersion() >= 0x030C00; + assert(SbkObjectType_Check(type)); + if (use_312) { + auto *obType = reinterpret_cast<PyObject *>(type); + void *data = PepObject_GetTypeData(obType, Py_TYPE(obType)); + return reinterpret_cast<SbkObjectTypePrivate *>(data); + } + if (type == SOTP_key) return SOTP_value; - auto it = SOTP_extender.find(sbkType); + auto it = SOTP_extender.find(type); if (it == SOTP_extender.end()) { - it = SOTP_extender.insert({sbkType, {}}).first; + it = SOTP_extender.insert({type, {}}).first; memset(&it->second, 0, sizeof(SbkObjectTypePrivate)); } - SOTP_key = sbkType; + SOTP_key = type; SOTP_value = &it->second; return SOTP_value; } -void PepType_SOTP_delete(PyTypeObject *sbkType) +void PepType_SOTP_delete(PyTypeObject *type) { - SOTP_extender.erase(sbkType); + static bool use_312 = _PepRuntimeVersion() >= 0x030C00; + assert(SbkObjectType_Check(type)); + if (use_312) + return; + SOTP_extender.erase(type); SOTP_key = nullptr; } +#endif // !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030C0000 + /* * SbkEnumType extender */ @@ -917,6 +1101,7 @@ static thread_local SbkEnumTypePrivate *SETP_value{}; SbkEnumTypePrivate *PepType_SETP(SbkEnumType *enumType) { + // PYSIDE-2230: This makes no sense at all for Enum types. if (enumType == SETP_key) return SETP_value; auto it = SETP_extender.find(enumType); @@ -935,38 +1120,76 @@ void PepType_SETP_delete(SbkEnumType *enumType) SETP_key = nullptr; } -/* - * PySideQFlagsType extender - */ -static std::unordered_map<PySideQFlagsType *, PySideQFlagsTypePrivate> PFTP_extender{}; -static thread_local PySideQFlagsType *PFTP_key{}; -static thread_local PySideQFlagsTypePrivate *PFTP_value{}; - -PySideQFlagsTypePrivate *PepType_PFTP(PySideQFlagsType *flagsType) -{ - static PyTypeObject *enumMeta = getPyEnumMeta(); - auto *mappedType = reinterpret_cast<PyTypeObject *>(flagsType); - auto *metaType = Py_TYPE(mappedType); - if (metaType == enumMeta) { - return reinterpret_cast<PySideQFlagsTypePrivate *>( - PepType_SETP(reinterpret_cast<SbkEnumType *>(flagsType))); - } - if (flagsType == PFTP_key) - return PFTP_value; - auto it = PFTP_extender.find(flagsType); - if (it == PFTP_extender.end()) { - it = PFTP_extender.insert({flagsType, {}}).first; - memset(&it->second, 0, sizeof(PySideQFlagsTypePrivate)); +#ifdef Py_LIMITED_API +static PyObject *emulatePyType_GetDict(PyTypeObject *type) +{ + if (_PepRuntimeVersion() < 0x030C00 || type->tp_dict) { + auto *res = type->tp_dict; + Py_XINCREF(res); + return res; } - PFTP_key = flagsType; - PFTP_value = &it->second; - return PFTP_value; + // PYSIDE-2230: Here we are really cheating. We don't know how to + // access an internal dict, and so we simply pretend + // it were an empty dict. This works great for our types. + // This was an unexpectedly simple solution :D + return PyDict_New(); } +#endif -void PepType_PFTP_delete(PySideQFlagsType *flagsType) +// PyType_GetDict: replacement for <static type>.tp_dict, which is +// zero for builtin types since 3.12. +PyObject *PepType_GetDict(PyTypeObject *type) { - PFTP_extender.erase(flagsType); - PFTP_key = nullptr; +#if !defined(Py_LIMITED_API) +# if PY_VERSION_HEX >= 0x030C0000 + return PyType_GetDict(type); +# else + // pre 3.12 fallback code, mimicking the addref-behavior. + Py_XINCREF(type->tp_dict); + return type->tp_dict; +# endif +#else + return emulatePyType_GetDict(type); +#endif // Py_LIMITED_API +} + +int PepType_SetDict(PyTypeObject *type, PyObject *dict) +{ + type->tp_dict = dict; + return 0; +} + +// Pre 3.10, PyType_GetSlot() would only work for heap types. +// FIXME: PyType_GetSlot() can be used unconditionally when the +// minimum limited API version is >= 3.10. +void *PepType_GetSlot(PyTypeObject *type, int aSlot) +{ + static const bool is310 = _PepRuntimeVersion() >= 0x030A00; + if (is310 || (type->tp_flags & Py_TPFLAGS_HEAPTYPE) != 0) + return PyType_GetSlot(type, aSlot); + + switch (aSlot) { + case Py_tp_alloc: + return reinterpret_cast<void *>(type->tp_alloc); + case Py_tp_getattro: + return reinterpret_cast<void *>(type->tp_getattro); + case Py_tp_setattro: + return reinterpret_cast<void *>(type->tp_setattro); + case Py_tp_descr_get: + return reinterpret_cast<void *>(type->tp_descr_get); + case Py_tp_descr_set: + return reinterpret_cast<void *>(type->tp_descr_set); + case Py_tp_call: + return reinterpret_cast<void *>(type->tp_call); + case Py_tp_new: + return reinterpret_cast<void *>(type->tp_new); + case Py_tp_init: + return reinterpret_cast<void *>(type->tp_init); + case Py_tp_free: + return reinterpret_cast<void *>(type->tp_free); + } + assert(false); + return nullptr; } /*************************************************************************** @@ -1000,16 +1223,16 @@ static inline void *PepType_ExTP(PyTypeObject *type, size_t size) static PyTypeObject *alias{}; const char *kind = size == sizeof(SbkObjectTypePrivate) ? "SOTP" : size == sizeof(SbkEnumTypePrivate) ? "SETP" : - size == sizeof(PySideQFlagsTypePrivate) ? "PFTP" : + size == sizeof(SbkQFlagsTypePrivate) ? "PFTP" : "unk."; fprintf(stderr, "%s:%d %p x %s s=%ld\n", __func__, __LINE__, type, kind, size); PyObject *kill{}; if (strlen(env_p) > 0) { - if (size == sizeof(PySideQFlagsTypePrivate)) { + if (size == sizeof(SbkQFlagsTypePrivate)) { if (alias == nullptr) alias = type; } - if (size != sizeof(PySideQFlagsTypePrivate)) { + if (size != sizeof(SbkQFlagsTypePrivate)) { if (type == alias) Py_INCREF(kill); } diff --git a/sources/shiboken6/libshiboken/pep384impl.h b/sources/shiboken6/libshiboken/pep384impl.h index a1ccad81f..ec58aac81 100644 --- a/sources/shiboken6/libshiboken/pep384impl.h +++ b/sources/shiboken6/libshiboken/pep384impl.h @@ -4,11 +4,6 @@ #ifndef PEP384IMPL_H #define PEP384IMPL_H -// PYSIDE-1436: Adapt to Python 3.10 -#if PY_VERSION_HEX < 0x030900A4 -# define Py_SET_REFCNT(obj, refcnt) ((Py_REFCNT(obj) = (refcnt)), (void)0) -#endif - extern "C" { @@ -55,42 +50,79 @@ typedef struct _typeobject { const char *tp_name; Py_ssize_t tp_basicsize; void *X03; // Py_ssize_t tp_itemsize; +#ifdef PEP384_INTERN destructor tp_dealloc; +#else + destructor X04; +#endif void *X05; // Py_ssize_t tp_vectorcall_offset; void *X06; // getattrfunc tp_getattr; void *X07; // setattrfunc tp_setattr; void *X08; // PyAsyncMethods *tp_as_async; +#ifdef PEP384_INTERN reprfunc tp_repr; +#else + reprfunc X09; +#endif void *X10; // PyNumberMethods *tp_as_number; void *X11; // PySequenceMethods *tp_as_sequence; void *X12; // PyMappingMethods *tp_as_mapping; void *X13; // hashfunc tp_hash; +#ifdef PEP384_INTERN ternaryfunc tp_call; - reprfunc tp_str; +#else + ternaryfunc X14; +#endif + reprfunc tp_str; // Only used for PEP384_INTERN and a shiboken test getattrofunc tp_getattro; setattrofunc tp_setattro; void *X18; // PyBufferProcs *tp_as_buffer; unsigned long tp_flags; void *X20; // const char *tp_doc; +#ifdef PEP384_INTERN traverseproc tp_traverse; inquiry tp_clear; +#else + traverseproc X21; + inquiry X22; +#endif void *X23; // richcmpfunc tp_richcompare; Py_ssize_t tp_weaklistoffset; void *X25; // getiterfunc tp_iter; +#ifdef PEP384_INTERN iternextfunc tp_iternext; +#else + iternextfunc X26; +#endif struct PyMethodDef *tp_methods; struct PyMemberDef *tp_members; struct PyGetSetDef *tp_getset; struct _typeobject *tp_base; +#ifdef PEP384_INTERN PyObject *tp_dict; descrgetfunc tp_descr_get; descrsetfunc tp_descr_set; +#else + void *X31; + descrgetfunc X32; + descrsetfunc X33; +#endif Py_ssize_t tp_dictoffset; +#ifdef PEP384_INTERN initproc tp_init; allocfunc tp_alloc; +#else + initproc X39; + allocfunc X40; +#endif newfunc tp_new; +#ifdef PEP384_INTERN freefunc tp_free; inquiry tp_is_gc; /* For PyObject_IS_GC */ +#else + freefunc X41; + inquiry X42; /* For PyObject_IS_GC */ +#endif PyObject *tp_bases; PyObject *tp_mro; /* method resolution order */ @@ -103,24 +135,16 @@ typedef struct _typeobject { && (Py_TYPE(o)->tp_is_gc == NULL || Py_TYPE(o)->tp_is_gc(o))) #endif -// This was a macro error in the limited API from the beginning. -// It was fixed in Python master, but did make it only into Python 3.8 . - -// PYSIDE-1797: This must be a runtime decision. -// Remove that when the minimum Python version is 3.8, -// because the macro PepIndex_Check bug was fixed then. -/// FIXME: Remove PepIndex_Check and pep384_issue33738.cpp when Python 3.7 is gone. -LIBSHIBOKEN_API int PepIndex_Check(PyObject *obj); - LIBSHIBOKEN_API PyObject *_PepType_Lookup(PyTypeObject *type, PyObject *name); #else // Py_LIMITED_API -#define PepIndex_Check(obj) PyIndex_Check(obj) #define _PepType_Lookup(type, name) _PyType_Lookup(type, name) #endif // Py_LIMITED_API +/// PYSIDE-939: We need the runtime version, given major << 16 + minor << 8 + micro +LIBSHIBOKEN_API long _PepRuntimeVersion(); /***************************************************************************** * * PYSIDE-535: Implement a clean type extension for PyPy @@ -139,16 +163,15 @@ LIBSHIBOKEN_API SbkEnumTypePrivate *PepType_SETP(SbkEnumType *type); LIBSHIBOKEN_API void PepType_SETP_delete(SbkEnumType *enumType); struct PySideQFlagsType; -struct PySideQFlagsTypePrivate; - -LIBSHIBOKEN_API PySideQFlagsTypePrivate *PepType_PFTP(PySideQFlagsType *type); -LIBSHIBOKEN_API void PepType_PFTP_delete(PySideQFlagsType *flagsType); +struct SbkQFlagsTypePrivate; /*****************************************************************************/ // functions used everywhere LIBSHIBOKEN_API const char *PepType_GetNameStr(PyTypeObject *type); +LIBSHIBOKEN_API PyObject *Pep_GetPartialFunction(void); + /***************************************************************************** * * RESOLVED: pydebug.h @@ -163,7 +186,6 @@ LIBSHIBOKEN_API const char *PepType_GetNameStr(PyTypeObject *type); */ LIBSHIBOKEN_API int Pep_GetFlag(const char *name); LIBSHIBOKEN_API int Pep_GetVerboseFlag(void); -#define Py_VerboseFlag Pep_GetVerboseFlag() #endif /***************************************************************************** @@ -193,12 +215,17 @@ LIBSHIBOKEN_API int Pep_GetVerboseFlag(void); // PyUnicode_GetSize is deprecated in favor of PyUnicode_GetLength. #define PepUnicode_GetLength(op) PyUnicode_GetLength((PyObject *)(op)) +// Unfortunately, we cannot ask this at runtime +// #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 +// FIXME: Python 3.10: Replace _PepUnicode_AsString by PyUnicode_AsUTF8 #ifdef Py_LIMITED_API LIBSHIBOKEN_API const char *_PepUnicode_AsString(PyObject *); enum PepUnicode_Kind { +#if PY_VERSION_HEX < 0x030C0000 PepUnicode_WCHAR_KIND = 0, +#endif PepUnicode_1BYTE_KIND = 1, PepUnicode_2BYTE_KIND = 2, PepUnicode_4BYTE_KIND = 4 @@ -213,7 +240,9 @@ LIBSHIBOKEN_API void *_PepUnicode_DATA(PyObject *str); #else enum PepUnicode_Kind { +#if PY_VERSION_HEX < 0x030C0000 PepUnicode_WCHAR_KIND = PyUnicode_WCHAR_KIND, +#endif PepUnicode_1BYTE_KIND = PyUnicode_1BYTE_KIND, PepUnicode_2BYTE_KIND = PyUnicode_2BYTE_KIND, PepUnicode_4BYTE_KIND = PyUnicode_4BYTE_KIND @@ -275,7 +304,7 @@ enum PepUnicode_Kind { #ifdef Py_LIMITED_API -typedef struct _pycfunc PyCFunctionObject; +using PyCFunctionObject = struct _pycfunc; #define PyCFunction_GET_FUNCTION(func) PyCFunction_GetFunction((PyObject *)func) #define PyCFunction_GET_SELF(func) PyCFunction_GetSelf((PyObject *)func) #define PyCFunction_GET_FLAGS(func) PyCFunction_GetFlags((PyObject *)func) @@ -305,11 +334,8 @@ LIBSHIBOKEN_API PyObject *PyRun_String(const char *, int, PyObject *, PyObject * // buffer functions. // But this is no problem as we check it's validity for every version. -#define PYTHON_BUFFER_VERSION_COMPATIBLE (PY_VERSION_HEX >= 0x03030000 && \ - PY_VERSION_HEX < 0x030AFFFF) -#if !PYTHON_BUFFER_VERSION_COMPATIBLE -# error Please check the buffer compatibility for this python version! -#endif +// PYSIDE-1960 The buffer interface is since Python 3.11 part of the stable +// API and we do not need to check the compatibility by hand anymore. typedef struct { getbufferproc bf_getbuffer; @@ -393,10 +419,13 @@ LIBSHIBOKEN_API PyObject *PyMethod_Self(PyObject *); typedef struct _code PepCodeObject; LIBSHIBOKEN_API int PepCode_Get(PepCodeObject *co, const char *name); +LIBSHIBOKEN_API int PepCode_Check(PyObject *o); # define PepCode_GET_FLAGS(o) PepCode_Get(o, "co_flags") # define PepCode_GET_ARGCOUNT(o) PepCode_Get(o, "co_argcount") +LIBSHIBOKEN_API PyObject *PepFunction_GetDefaults(PyObject *function); + /* Masks for co_flags above */ # define CO_OPTIMIZED 0x0001 # define CO_NEWLOCALS 0x0002 @@ -410,7 +439,15 @@ LIBSHIBOKEN_API int PepCode_Get(PepCodeObject *co, const char *name); # define PepCodeObject PyCodeObject # define PepCode_GET_FLAGS(o) ((o)->co_flags) # define PepCode_GET_ARGCOUNT(o) ((o)->co_argcount) +# define PepCode_Check PyCode_Check + +# ifdef PYPY_VERSION +LIBSHIBOKEN_API PyObject *PepFunction_GetDefaults(PyObject *function); + +# else +# define PepFunction_GetDefaults PyFunction_GetDefaults +# endif #endif /***************************************************************************** @@ -511,9 +548,9 @@ extern LIBSHIBOKEN_API PyTypeObject *PepBuiltinMethod_TypePtr; * * This is not defined if Py_LIMITED_API is defined. */ -#if PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) +#ifdef Py_LIMITED_API LIBSHIBOKEN_API PyObject *PyImport_GetModule(PyObject *name); -#endif // PY_VERSION_HEX < 0x03070000 || defined(Py_LIMITED_API) +#endif // Py_LIMITED_API // Evaluate a script and return the variable `result` LIBSHIBOKEN_API PyObject *PepRun_GetResult(const char *command); @@ -538,6 +575,20 @@ extern LIBSHIBOKEN_API int PepRuntime_38_flag; /***************************************************************************** * + * Runtime support for Python 3.12 incompatibility + * + */ + +LIBSHIBOKEN_API PyObject *PepType_GetDict(PyTypeObject *type); + +// This function does not exist as PyType_SetDict. But because tp_dict +// is no longer considered to be accessible, we treat it as such. +LIBSHIBOKEN_API int PepType_SetDict(PyTypeObject *type, PyObject *dict); + +LIBSHIBOKEN_API void *PepType_GetSlot(PyTypeObject *type, int aSlot); + +/***************************************************************************** + * * Module Initialization * */ diff --git a/sources/shiboken6/libshiboken/pep384impl_doc.rst b/sources/shiboken6/libshiboken/pep384impl_doc.rst deleted file mode 100644 index 9ee74a26c..000000000 --- a/sources/shiboken6/libshiboken/pep384impl_doc.rst +++ /dev/null @@ -1,704 +0,0 @@ -**************************************** -The Transition To The Limited Python API -**************************************** - - -Foreword -======== - -Python supports a limited API that restricts access to certain structures. -Besides eliminating whole modules and all functions and macros which names -start with an -underscore, the most drastic restriction is the removal of normal type object -declarations. - -For details about the eliminated modules and functions, please see the -`PEP 384`_ page for reference. - - -.. _`PEP 384`: https://www.python.org/dev/peps/pep-0384/ - - - -Changed Modules -=============== - -All changed module's include files are listed with the changed functions here. -As a general rule, it was tried to keep the changes to a minimum diff. -Macros which are not available were changed to functions with the same name -if possible. Completely removed names ``Py{name}`` were re-implemented as ``Pep{name}``. - - -memoryobject.h --------------- - -The buffer protocol was completely removed. We redefined all the structures -and methods, because PySide uses that. This is an exception to the limited API -that we have to check ourselves. The code is extracted in bufferprocs_py37.h . -This is related to the following: - - -abstract.h ----------- - -This belongs to the buffer protocol like memoryobject.h . -As replacement for ``Py_buffer`` we defined ``Pep_buffer`` and several other -internal macros. - -The version is checked by hand, and the version number must be updated only -if the implementation does not change. Otherwise, we need to write version -dependent code paths. - -It is questionable if it is worthwhile to continue using the buffer protocol -or if we should try to get rid of ``Pep_buffer``, completely. - - -pydebug.h ---------- - -We have no direct access to ``Py_VerboseFlag`` because debugging is not -supported. We redefined it as macro ``Py_VerboseFlag`` which calls ``Pep_VerboseFlag``. - - -unicodeobject.h ---------------- - -The macro ``PyUnicode_GET_SIZE`` was removed and replaced by ``PepUnicode_GetLength`` -which evaluates to ``PyUnicode_GetSize`` for Python 2 and ``PyUnicode_GetLength`` for Python 3. -Since Python 3.3, ``PyUnicode_GetSize`` would have the bad side effect of requiring the GIL! - -Function ``_PyUnicode_AsString`` is unavailable and was replaced by a macro -that calls ``_PepUnicode_AsString``. The implementation was a bit involved, -and it would be better to change the code and replace this function. - - -bytesobject.h -------------- - -The macros ``PyBytes_AS_STRING`` and ``PyBytes_GET_SIZE`` were redefined to call -the according functions. - - -floatobject.h -------------- - -``PyFloat_AS_DOUBLE`` now calls ``PyFloat_AsDouble``. - - -tupleobject.h -------------- - -``PyTuple_GET_ITEM``, ``PyTuple_SET_ITEM`` and ``PyTuple_GET_SIZE`` were redefined as -function calls. - - -listobject.h ------------- - -``PyList_GET_ITEM``, ``PyList_SET_ITEM`` and ``PyList_GET_SIZE`` were redefined as -function calls. - - -dictobject.h ------------- - -``PyDict_GetItem`` also exists in a ``PyDict_GetItemWithError`` version that does -not suppress errors. This suppression has the side effect of touching global -structures. This function exists in Python 2 only since Python 2.7.12 and has -a different name. We simply implemented the function. -Needed to avoid the GIL when accessing dictionaries. - - -methodobject.h --------------- - -``PyCFunction_GET_FUNCTION``, ``PyCFunction_GET_SELF`` and ``PyCFunction_GET_FLAGS`` -were redefined as function calls. - -Direct access to the methoddef structure is not available, and we defined -``PepCFunction_GET_NAMESTR`` as accessor for name strings. - - -pythonrun.h ------------ - -The simple function ``PyRun_String`` is not available. It was re-implemented -in a simplified version for the signature module. - - -funcobject.h ------------- - -The definitions of funcobject.h are completely missing, although there -are extra ``#ifdef`` conditional defines inside, too. This suggests that the exclusion -was unintended. - -We therefore redefined ``PyFunctionObject`` as an opaque type. - -The missing macro ``PyFunction_Check`` was defined, and the macro -``PyFunction_GET_CODE`` calls the according function. - -There is no equivalent for function name access, therefore we introduced -``PepFunction_GetName`` either as a function or as a macro. - -*TODO: We should fix funcobject.h* - - -classobject.h -------------- - -Classobject is also completely not imported, instead of defining an opaque type. - -We defined the missing functions ``PyMethod_New``, ``PyMethod_Function`` and -``PyMethod_Self`` and also redefined ``PyMethod_GET_SELF`` and -``PyMethod_GET_FUNCTION`` as calls to these functions. - -*TODO: We should fix classobject.h* - - -code.h ------- - -The whole code.c code is gone, although it may make sense to -define some minimum accessibility. This will be clarified on -`Python-Dev`_. We needed access to code objects and defined the missing -PepCode_GET_FLAGS and PepCode_GET_ARGCOUNT either as function or macro. -We further added the missing flags, although few are used: - -``CO_OPTIMIZED`` ``CO_NEWLOCALS`` ``CO_VARARGS`` ``CO_VARKEYWORDS`` ``CO_NESTED`` -``CO_GENERATOR`` - -*TODO: We should maybe fix code.h* - -.. _`Python-Dev`: https://mail.python.org/mailman/listinfo/python-dev - -datetime.h ----------- - -The DateTime module is explicitly not included in the limited API. -We defined all the needed functions but called them via Python instead -of direct call macros. This has a slight performance impact. - -The performance could be easily improved by providing an interface -that fetches all attributes at once, instead of going through the object -protocol every time. - -The re-defined macros and methods are:: - - PyDateTime_GET_YEAR - PyDateTime_GET_MONTH - PyDateTime_GET_DAY - PyDateTime_DATE_GET_HOUR - PyDateTime_DATE_GET_MINUTE - PyDateTime_DATE_GET_SECOND - PyDateTime_DATE_GET_MICROSECOND - PyDateTime_DATE_GET_FOLD - PyDateTime_TIME_GET_HOUR - PyDateTime_TIME_GET_MINUTE - PyDateTime_TIME_GET_SECOND - PyDateTime_TIME_GET_MICROSECOND - PyDateTime_TIME_GET_FOLD - - PyDate_Check - PyDateTime_Check - PyTime_Check - - PyDate_FromDate - PyDateTime_FromDateAndTime - PyTime_FromTime - -*XXX: We should maybe provide an optimized interface to datetime* - - -object.h --------- - -The file object.h contains the ``PyTypeObject`` structure, which is supposed -to be completely opaque. All access to types should be done through -``PyType_GetSlot`` calls. Due to bugs and deficiencies in the limited API -implementation, it was not possible to do that. Instead, we have defined -a simplified structure for ``PyTypeObject`` that has only the fields that -are used in PySide. - -We will explain later why and how this was done. Here is the reduced -structure:: - - typedef struct _typeobject { - PyVarObject ob_base; - const char *tp_name; - Py_ssize_t tp_basicsize; - void *X03; // Py_ssize_t tp_itemsize; - void *X04; // destructor tp_dealloc; - void *X05; // printfunc tp_print; - void *X06; // getattrfunc tp_getattr; - void *X07; // setattrfunc tp_setattr; - void *X08; // PyAsyncMethods *tp_as_async; - void *X09; // reprfunc tp_repr; - void *X10; // PyNumberMethods *tp_as_number; - void *X11; // PySequenceMethods *tp_as_sequence; - void *X12; // PyMappingMethods *tp_as_mapping; - void *X13; // hashfunc tp_hash; - ternaryfunc tp_call; - reprfunc tp_str; - void *X16; // getattrofunc tp_getattro; - void *X17; // setattrofunc tp_setattro; - void *X18; // PyBufferProcs *tp_as_buffer; - void *X19; // unsigned long tp_flags; - void *X20; // const char *tp_doc; - traverseproc tp_traverse; - inquiry tp_clear; - void *X23; // richcmpfunc tp_richcompare; - Py_ssize_t tp_weaklistoffset; - void *X25; // getiterfunc tp_iter; - void *X26; // iternextfunc tp_iternext; - struct PyMethodDef *tp_methods; - void *X28; // struct PyMemberDef *tp_members; - void *X29; // struct PyGetSetDef *tp_getset; - struct _typeobject *tp_base; - PyObject *tp_dict; - descrgetfunc tp_descr_get; - void *X33; // descrsetfunc tp_descr_set; - Py_ssize_t tp_dictoffset; - initproc tp_init; - allocfunc tp_alloc; - newfunc tp_new; - freefunc tp_free; - inquiry tp_is_gc; /* For PyObject_IS_GC */ - PyObject *tp_bases; - PyObject *tp_mro; /* method resolution order */ - } PyTypeObject; - -Function ``PyIndex_Check`` had to be defined in an unwanted way due to -a Python issue. See file pep384_issue33738.cpp . - -There are extension structures which have been isolated as special macros that -dynamically compute the right offsets of the extended type structures: - -* ``PepType_SOTP`` for ``SbkObjectTypePrivate`` -* ``PepType_SETP`` for ``SbkEnumTypePrivate`` -* ``PepType_PFTP`` for ``PySideQFlagsTypePrivate`` - -How these extension structures are used can best be seen by searching -``PepType_{four}`` in the source. - -Due to the new heaptype interface, the names of certain types contain -now the module name in the ``tp_name`` field. To have a compatible way -to access simple type names as C string, ``PepType_GetNameStr`` has been -written that skips over dotted name parts. - -Finally, the function ``_PyObject_Dump`` was excluded from the limited API. -This is a useful debugging aid that we always want to have available, -so it is added back, again. Anyway, we did not reimplement it, and so -Windows is not supported. -Therefore, a forgotten debugging call of this functions will break COIN. :-) - - -Using The New Type API -====================== - -After converting everything but the object.h file, we were a little -bit shocked: it suddenly was clear that we would have no more -access to type objects, and even more scary that all types which we -use have to be heap types, only! - -For PySide with its intense use of heap type extensions in various -flavors, the situation looked quite unsolvable. In the end, it was -nicely solved, but it took almost 3.5 months to get that right. - -Before we see how this is done, we will explain the differences -between the APIs and their consequences. - - -The Interface -------------- - -The old type API of Python knows static types and heap types. -Static types are written down as a declaration of a ``PyTypeObject`` -structure with all its fields filled in. Here is for example -the definition of the Python type ``object`` (Python 3.6):: - - PyTypeObject PyBaseObject_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "object", /* tp_name */ - sizeof(PyObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - object_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - object_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - (hashfunc)_Py_HashPointer, /* tp_hash */ - 0, /* tp_call */ - object_str, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - PyObject_GenericSetAttr, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("object()\n--\n\nThe most base type"), /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - object_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - object_methods, /* tp_methods */ - 0, /* tp_members */ - object_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - object_init, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - object_new, /* tp_new */ - PyObject_Del, /* tp_free */ - }; - -We can write the same structure in form of a ``PyType_Spec`` structure, -and there is even an incomplete tool *abitype.py* that does this conversion -for us. With a few corrections, the result looks like this:: - - static PyType_Slot PyBaseObject_Type_slots[] = { - {Py_tp_dealloc, (void *)object_dealloc}, - {Py_tp_repr, (void *)object_repr}, - {Py_tp_hash, (void *)_Py_HashPointer}, - {Py_tp_str, (void *)object_str}, - {Py_tp_getattro, (void *)PyObject_GenericGetAttr}, - {Py_tp_setattro, (void *)PyObject_GenericSetAttr}, - {Py_tp_richcompare, (void *)object_richcompare}, - {Py_tp_methods, (void *)object_methods}, - {Py_tp_getset, (void *)object_getsets}, - {Py_tp_init, (void *)object_init}, - {Py_tp_alloc, (void *)PyType_GenericAlloc}, - {Py_tp_new, (void *)object_new}, - {Py_tp_free, (void *)PyObject_Del}, - {0, 0}, - }; - static PyType_Spec PyBaseObject_Type_spec = { - "object", - sizeof(PyObject), - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - PyBaseObject_Type_slots, - }; - -This new structure is almost compatible with the old one, but there -are some subtle differences. - -* The new types are generated in one step - -This seems to be no problem, but it was very much, due to the way the -types were built in PySide. Types were assembled piece by piece, and -finally the ``PyType_Ready`` function was called. - -With the new API, ``PyType_Ready`` is called already at the end of -``PyType_FromSpec``, and that meant that the logic of type creation became -completely turned upside down. - -* The new types are always heaptypes - -With the new type creation functions, it is no longer possible to -create "normal" types. Instead, they all have to be allocated on the -heap and garbage collected. The user should normally not recognize this. -But type creation is more constrained, and you cannot create a subtype -if the ``Py_TPFLAGS_BASETYPE`` is not set. This constraint was already -violated by PySide and needed a quite profound fix. - -* The new types always need a module - -While this is not a problem per se, the above new type spec will not create -a usable new type, but complain with:: - - DeprecationWarning: builtin type object has no __module__ attribute - -But there are more problems: - -* The new types have unexpected defaults - -When fields are empty, you would usually assume that they stay empty. -There are just a few corrections that ``PyType_Ready`` will do to a type. - -But there is the following clause in ``PyType_FromSpec`` that can give you -many headaches:: - - if (type->tp_dealloc == NULL) { - /* It's a heap type, so needs the heap types' dealloc. - subtype_dealloc will call the base type's tp_dealloc, if - necessary. */ - type->tp_dealloc = subtype_dealloc; - } - -In fact, before the move to the new API, the ``PyType_Ready`` function -filled empty ``tp_dealloc`` fields with ``object_dealloc``. And the code -that has been written with that in mind now becomes pretty wrong if suddenly -``subtype_dealloc`` is used. - -The way out was to explicitly provide an ``object_dealloc`` function. -This would then again impose a problem, because ``object_dealloc`` is not -public. Writing our own version is easy, but it again needs access to -type objects. But fortunately, we have broken this rule, already... - - -* The new types are only partially allocated - -The structures used in ``PyType_FromSpec`` are almost all allocated, -only the name field is static. This is no problem for types which are -statically created once. But if you want to parameterize things and -create multiple types with a single slots and spec definition, the name -field that is used for tp_name must be allocated dynamically. -This is misleading, since all the slots already are copies. - -* The new types don't support special offsets - -The special fields ``tp_weaklistoffset`` and ``tp_dictoffset`` are not supported -by ``PyType_FromSpec``. Unfortunately the documentation does not tell you -if you are allowed to set these fields manually after creating the type or not. -We finally did it and it worked, but we are not sure about correctness. - -See basewrapper.cpp function ``SbkObject_TypeF()`` as the only reference to -these fields in PySide. This single reference is absolutely necessary and -very important, since all derived types invisibly inherit these two fields. - - -Future Versions Of The Limited API -================================== - -As we have seen, the current version of the limited API does a bit of -cheating, because it uses parts of the data structure that should be -an opaque type. At the moment, this works fine because the data is -still way more compatible as it could be. - -But what if this is changed in the future? - -We know that the data structures are stable until Python 3.8 comes out. -Until then, the small bugs and omissions will hopefully all be solved. -Then it will be possible to replace the current small tricks by calls -to ``PyType_GetSlot`` in the way things should be. - -At the very moment when the current assumptions about the data structure -are no longer true, we will rewrite the direct attribute access with -calls to ``PyType_GetSlot``. After that, no more changes will be necessary. - - -Appendix A: The Transition To Simpler Types -=========================================== - -After all code had been converted to the limited API, there was a -remaining problem with the ``PyHeapTypeObject``. - -Why a problem? Well, all the type structures in shiboken use -special extra fields at the end of the heap type object. This -currently enforces extra knowledge at compile time about how large the -heap type object is. In a clean implementation, we would only use -the ``PyTypeObject`` itself and access the fields *behind* the type -by a pointer that is computed at runtime. - - -Restricted PyTypeObject ------------------------ - -Before we are going into details, let us motivate the existence of -the restricted ``PyTypeObject``: - -Originally, we wanted to use ``PyTypeObject`` as an opaque type and -restrict ourselves to only use the access function ``PyType_GetSlot``. -This function allows access to all fields which are supported by -the limited API. - -But this is a restriction, because we get no access to ``tp_dict``, -which we need to support the signature extension. But we can work -around that. - -The real restriction is that ``PyType_GetSlot`` only works for heap -types. This makes the function quite useless, because we have -no access to ``PyType_Type``, which is the most important type ``type`` -in Python. We need that for instance to compute the size of -``PyHeapTypeObject`` dynamically. - -With much effort, it is possible to clone ``PyType_Type`` as a heap -type. But due to a bug in the Pep 384 support, we need -access to the ``nb_index`` field of a normal type. Cloning does not -help because ``PyNumberMethods`` fields are *not* inherited. - -After we realized this dead end, we changed concept and did not -use ``PyType_GetSlot`` at all (except in function ``copyNumberMethods``), -but created a restricted ``PyTypeObject`` with only those fields -defined that are needed in PySide. - -Is this breakage of the limited API? I don't think so. A special -function runs on program startup that checks the correct position -of the fields of ``PyTypeObject``, although a change in those fields is -more than unlikely. -The really crucial thing is to no longer use ``PyHeapTypeObject`` -explicitly because that *does* change its layout over time. - - -Diversification ---------------- - -There were multiple ``Sbk{something}`` structures which all used a "d" field -for their private data. This made it not easy to find the right -fields when switching between objects and types:: - - struct LIBSHIBOKEN_API SbkObject - { - PyObject_HEAD - PyObject *ob_dict; - PyObject *weakreflist; - SbkObjectPrivate *d; - }; - - struct LIBSHIBOKEN_API SbkObjectType - { - PyHeapTypeObject super; - SbkObjectTypePrivate *d; - }; - -The first step was to rename the SbkObjectTypePrivate part from "d" to -"sotp". It was chosen to be short but easy to remember as abbreviation -of "SbkObjectTypePrivate", leading to:: - - struct LIBSHIBOKEN_API SbkObjectType - { - PyHeapTypeObject super; - SbkObjectTypePrivate *sotp; - }; - -After renaming, it was easier to do the following transformations. - - -Abstraction ------------ - -After renaming the type extension pointers to ``sotp``, I replaced -them by function-like macros which did the special access *behind* -the types, instead of those explicit fields. For instance, the -expression:: - - type->sotp->converter - -became:: - - PepType_SOTP(type)->converter - -The macro expansion can be seen here:: - - #define PepHeapType_SIZE \ - (reinterpret_cast<PyTypeObject *>(&PyType_Type)->tp_basicsize) - - #define _genericTypeExtender(etype) \ - (reinterpret_cast<char *>(etype) + PepHeapType_SIZE) - - #define PepType_SOTP(etype) \ - (*reinterpret_cast<SbkObjectTypePrivate **>(_genericTypeExtender(etype))) - -This looks complicated, but in the end there is only a single new -indirection via ``PyType_Type``, which happens at runtime. This is the -key to fulfil what Pep 384 wants to achieve: *No more version-dependent fields*. - - -Simplification --------------- - -After all type extension fields were replaced by macro calls, we -could remove the following version dependent re-definition of ``PyHeapTypeObject`` -:: - - typedef struct _pyheaptypeobject { - union { - PyTypeObject ht_type; - void *opaque[PY_HEAPTYPE_SIZE]; - }; - } PyHeapTypeObject; - -, and the version dependent structure:: - - struct LIBSHIBOKEN_API SbkObjectType - { - PyHeapTypeObject super; - SbkObjectTypePrivate *sotp; - }; - -could be removed. SbkObjectType remains as a (deprecated) -type alias to PyTypeObject. - - -Appendix B: Verification Of PyTypeObject -======================================== - -We have introduced a limited PyTypeObject in the same place -as the original PyTypeObject, and now we need to prove that -we are allowed to do so. - -When using the limited API as intended, then types are completely -opaque, and access is only through ``PyType_FromSpec`` and (from -version 3.5 upwards) through ``PyType_GetSlot``. - -Python then uses all the slot definitions in the type description -and produces a regular heap type object. - - -Unused Information ------------------- - -We know many things about types that are not explicitly said, -but they are inherently clear: - -(a) The basic structure of a type is always the same, regardless - if it is a static type or a heap type. - -(b) types are evolving very slowly, and a field is never replaced - by another field with different semantics. - -Inherent rule (a) gives us the following information: If we calculate -the offsets of the basic fields, then this info is also usable for non-heap -types. - -The validation checks if rule (b) is still valid. - - -How it Works ------------- - -The basic idea of the validation is to produce a new type using -``PyType_FromSpec`` and to see where in the type structure these fields -show up. So we build a ``PyType_Slot`` structure with all the fields we -are using and make sure that these values are all unique in the -type. - -Most fields are not interrogated by ``PyType_FromSpec``, and so we -simply used some numeric value. Some fields are interpreted, like -``tp_members``. This field must really be a ``PyMemberDef``. And there are -``tp_base`` and ``tp_bases`` which have to be type objects and lists -thereof. It was easiest to not produce these fields from scratch -but use them from the ``type`` object ``PyType_Type``. - -Then one would think to write a function that searches the known -values in the opaque type structure. - -But we can do better and use optimistically the observation (b): -We simply use the restricted ``PyTypeObject`` structure and assume that -every field lands exactly where we are awaiting it. - -And that is the whole proof: If we find all the disjoint values at -the places where we expect them, then verification is done. - - -About ``tp_dict`` ------------------ - -One word about the ``tp_dict`` field: This field is a bit special in -the proof, since it does not appear in the spec and cannot easily -be checked by ``type.__dict__`` because that creates a *dictproxy* -object. So how do we prove that is really the right dict? - -We have to create that ``PyMethodDef`` structure anyway, and instead of -leaving it empty, we insert a dummy function. Then we ask the -``tp_dict`` field if it has the awaited object in it, and that's it! - -#EOT diff --git a/sources/shiboken6/libshiboken/pyobjectholder.h b/sources/shiboken6/libshiboken/pyobjectholder.h new file mode 100644 index 000000000..857748c2f --- /dev/null +++ b/sources/shiboken6/libshiboken/pyobjectholder.h @@ -0,0 +1,86 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef PYOBJECTHOLDER_H +#define PYOBJECTHOLDER_H + +#include "sbkpython.h" + +#include <cassert> +#include <utility> + +namespace Shiboken +{ + +/// PyObjectHolder holds a PyObject pointer, keeping a reference decrementing +/// its reference counter when destroyed. It makes sure to hold the GIL when +/// releasing. It implements copy/move semantics and is mainly intended as a +/// base class for functors holding a callable which can be passed around and +/// stored in containers or moved from freely. +/// For one-shot functors, release() can be invoked after the call. +class PyObjectHolder +{ +public: + PyObjectHolder() noexcept = default; + + /// PyObjectHolder constructor. + /// \param pyobj A reference to a Python object + explicit PyObjectHolder(PyObject *pyObj) noexcept : m_pyObj(pyObj) + { + assert(pyObj != nullptr); + Py_INCREF(m_pyObj); + } + + PyObjectHolder(const PyObjectHolder &o) noexcept : m_pyObj(o.m_pyObj) + { + Py_XINCREF(m_pyObj); + } + + PyObjectHolder &operator=(const PyObjectHolder &o) noexcept + { + if (this != &o) { + m_pyObj = o.m_pyObj; + Py_XINCREF(m_pyObj); + } + return *this; + } + + PyObjectHolder(PyObjectHolder &&o) noexcept : m_pyObj{std::exchange(o.m_pyObj, nullptr)} {} + + PyObjectHolder &operator=(PyObjectHolder &&o) noexcept + { + m_pyObj = std::exchange(o.m_pyObj, nullptr); + return *this; + } + + /// Decref the python reference + ~PyObjectHolder() { release(); } + + [[nodiscard]] bool isNull() const { return m_pyObj == nullptr; } + [[nodiscard]] operator bool() const { return m_pyObj != nullptr; } + + /// Returns the pointer of the Python object being held. + [[nodiscard]] PyObject *object() const { return m_pyObj; } + [[nodiscard]] operator PyObject *() const { return m_pyObj; } + + [[nodiscard]] PyObject *operator->() { return m_pyObj; } + +protected: + void release() + { + if (m_pyObj != nullptr) { + assert(Py_IsInitialized()); + auto gstate = PyGILState_Ensure(); + Py_DECREF(m_pyObj); + PyGILState_Release(gstate); + m_pyObj = nullptr; + } + } + +private: + PyObject *m_pyObj = nullptr; +}; + +} // namespace Shiboken + +#endif // PYOBJECTHOLDER_H diff --git a/sources/shiboken6/libshiboken/sbkarrayconverter.cpp b/sources/shiboken6/libshiboken/sbkarrayconverter.cpp index 8af310a53..bcc3fb767 100644 --- a/sources/shiboken6/libshiboken/sbkarrayconverter.cpp +++ b/sources/shiboken6/libshiboken/sbkarrayconverter.cpp @@ -14,8 +14,7 @@ static SbkArrayConverter *ArrayTypeConverters[Shiboken::Conversions::SBK_ARRAY_IDX_SIZE] [2] = {}; -namespace Shiboken { -namespace Conversions { +namespace Shiboken::Conversions { // Check whether Predicate is true for all elements of a sequence template <class Predicate> @@ -244,5 +243,4 @@ void setArrayTypeConverter(int index, int dimension, SbkArrayConverter *c) ArrayTypeConverters[index][dimension - 1] = c; } -} // namespace Conversions -} // namespace Shiboken +} // namespace Shiboken::Conversions diff --git a/sources/shiboken6/libshiboken/sbkarrayconverter.h b/sources/shiboken6/libshiboken/sbkarrayconverter.h index 97bd8ac6f..f07cb1d70 100644 --- a/sources/shiboken6/libshiboken/sbkarrayconverter.h +++ b/sources/shiboken6/libshiboken/sbkarrayconverter.h @@ -11,8 +11,7 @@ extern "C" { struct SbkArrayConverter; } -namespace Shiboken { -namespace Conversions { +namespace Shiboken::Conversions { enum : int { SBK_UNIMPLEMENTED_ARRAY_IDX, @@ -132,7 +131,6 @@ void ArrayHandle<T>::destroy() m_owned = false; } -} // namespace Conversions -} // namespace Shiboken +} // namespace Shiboken::Conversions #endif // SBKARRAYCONVERTERS_H diff --git a/sources/shiboken6/libshiboken/sbkarrayconverter_p.h b/sources/shiboken6/libshiboken/sbkarrayconverter_p.h index db92e56af..63d03fb12 100644 --- a/sources/shiboken6/libshiboken/sbkarrayconverter_p.h +++ b/sources/shiboken6/libshiboken/sbkarrayconverter_p.h @@ -10,7 +10,7 @@ extern "C" { -typedef PythonToCppFunc (*IsArrayConvertibleToCppFunc)(PyObject *, int dim1, int dim2); +using IsArrayConvertibleToCppFunc = PythonToCppFunc (*)(PyObject *, int dim1, int dim2); /** * \internal * Private structure of SbkArrayConverter. diff --git a/sources/shiboken6/libshiboken/sbkcontainer.cpp b/sources/shiboken6/libshiboken/sbkcontainer.cpp index 13c9f1a29..7de1f03e6 100644 --- a/sources/shiboken6/libshiboken/sbkcontainer.cpp +++ b/sources/shiboken6/libshiboken/sbkcontainer.cpp @@ -3,14 +3,17 @@ #include "sbkcontainer.h" #include "sbkstaticstrings.h" +#include "autodecref.h" namespace Shiboken { bool isOpaqueContainer(PyObject *o) { + if (!o) + return false; + Shiboken::AutoDecRef tpDict(PepType_GetDict(o->ob_type)); return o != nullptr && o != Py_None - && PyDict_Contains(o->ob_type->tp_dict, - Shiboken::PyMagicName::opaque_container()) == 1; + && PyDict_Contains(tpDict.object(), Shiboken::PyMagicName::opaque_container()) == 1; } } // Shiboken diff --git a/sources/shiboken6/libshiboken/sbkcontainer.h b/sources/shiboken6/libshiboken/sbkcontainer.h index 22513d3a0..240c772a9 100644 --- a/sources/shiboken6/libshiboken/sbkcontainer.h +++ b/sources/shiboken6/libshiboken/sbkcontainer.h @@ -63,7 +63,8 @@ public: static PyObject *tpNew(PyTypeObject *subtype, PyObject * /* args */, PyObject * /* kwds */) { - auto *me = reinterpret_cast<ShibokenContainer *>(subtype->tp_alloc(subtype, 0)); + allocfunc allocFunc = reinterpret_cast<allocfunc>(PepType_GetSlot(subtype, Py_tp_alloc)); + auto *me = reinterpret_cast<ShibokenContainer *>(allocFunc(subtype, 0)); auto *d = new ShibokenSequenceContainerPrivate; d->m_list = new SequenceContainer; d->m_ownsList = true; @@ -71,6 +72,14 @@ public: return reinterpret_cast<PyObject *>(me); } + static PyObject *tpNewInvalid(PyTypeObject * /* subtype */, PyObject * /* args */, PyObject * /* kwds */) + { + PyErr_Format(PyExc_NotImplementedError, + "Opaque containers of type '%s' cannot be instantiated.", + typeid(SequenceContainer).name()); + return nullptr; + } + static int tpInit(PyObject * /* self */, PyObject * /* args */, PyObject * /* kwds */) { return 0; @@ -83,7 +92,9 @@ public: if (d->m_ownsList) delete d->m_list; delete d; - Py_TYPE(pySelf)->tp_base->tp_free(self); + auto freeFunc = reinterpret_cast<freefunc>(PepType_GetSlot(Py_TYPE(pySelf)->tp_base, + Py_tp_free)); + freeFunc(self); } static Py_ssize_t sqLen(PyObject *self) @@ -98,7 +109,7 @@ public: PyErr_SetString(PyExc_IndexError, "index out of bounds"); return nullptr; } - auto it = d->m_list->cbegin(); + auto it = std::cbegin(*d->m_list); std::advance(it, i); return ShibokenContainerValueConverter<value_type>::convertValueToPython(*it); } @@ -110,7 +121,7 @@ public: PyErr_SetString(PyExc_IndexError, "index out of bounds"); return -1; } - auto it = d->m_list->begin(); + auto it = std::begin(*d->m_list); std::advance(it, i); OptionalValue value = ShibokenContainerValueConverter<value_type>::convertValueToCpp(pyArg); if (!value.has_value()) diff --git a/sources/shiboken6/libshiboken/sbkconverter.cpp b/sources/shiboken6/libshiboken/sbkconverter.cpp index 309810290..358827aa8 100644 --- a/sources/shiboken6/libshiboken/sbkconverter.cpp +++ b/sources/shiboken6/libshiboken/sbkconverter.cpp @@ -4,6 +4,7 @@ #include "sbkconverter.h" #include "sbkconverter_p.h" #include "sbkarrayconverter_p.h" +#include "sbkmodule.h" #include "basewrapper_p.h" #include "bindingmanager.h" #include "autodecref.h" @@ -18,8 +19,7 @@ static SbkConverter **PrimitiveTypeConverters; using ConvertersMap = std::unordered_map<std::string, SbkConverter *>; static ConvertersMap converters; -namespace Shiboken { -namespace Conversions { +namespace Shiboken::Conversions { void initArrayConverters(); @@ -147,6 +147,13 @@ void addPythonToCppValueConversion(PyTypeObject *type, addPythonToCppValueConversion(sotp->converter, pythonToCppFunc, isConvertibleToCppFunc); } +void addPythonToCppValueConversion(Shiboken::Module::TypeInitStruct typeStruct, + PythonToCppFunc pythonToCppFunc, + IsConvertibleToCppFunc isConvertibleToCppFunc) +{ + addPythonToCppValueConversion(typeStruct.type, pythonToCppFunc, isConvertibleToCppFunc); +} + PyObject *pointerToPython(PyTypeObject *type, const void *cppIn) { auto *sotp = PepType_SOTP(type); @@ -228,6 +235,11 @@ PythonToCppConversion pythonToCppPointerConversion(PyTypeObject *type, PyObject return {}; } +PythonToCppConversion pythonToCppPointerConversion(Module::TypeInitStruct typeStruct, PyObject *pyIn) +{ + return pythonToCppPointerConversion(typeStruct.type, pyIn); +} + static inline PythonToCppFunc IsPythonToCppConvertible(const SbkConverter *converter, PyObject *pyIn) { assert(pyIn); @@ -403,19 +415,34 @@ bool isImplicitConversion(PyTypeObject *type, PythonToCppFunc toCppFunc) return toCppFunc != (*conv).second; } -void registerConverterName(SbkConverter *converter , const char *typeName) +void registerConverterName(SbkConverter *converter, const char *typeName) { auto iter = converters.find(typeName); if (iter == converters.end()) converters.insert(std::make_pair(typeName, converter)); } +static std::string getRealTypeName(const char *name) +{ + std::string typeName(name); + auto size = typeName.size(); + if (std::isalnum(typeName[size - 1]) == 0) + return typeName.substr(0, size - 1); + return typeName; +} + SbkConverter *getConverter(const char *typeName) { - ConvertersMap::const_iterator it = converters.find(typeName); + auto it = converters.find(typeName); if (it != converters.end()) return it->second; - if (Py_VerboseFlag > 0) { + // PYSIDE-2404: Did not find the name. Load the lazy classes + // which have this name and try again. + Shiboken::Module::loadLazyClassesWithName(getRealTypeName(typeName).c_str()); + it = converters.find(typeName); + if (it != converters.end()) + return it->second; + if (Shiboken::pyVerbose() > 0) { const std::string message = std::string("Can't find type resolver for type '") + typeName + "'."; PyErr_WarnEx(PyExc_RuntimeWarning, message.c_str(), 0); @@ -677,7 +704,14 @@ PyTypeObject *getPythonTypeObject(const SbkConverter *converter) PyTypeObject *getPythonTypeObject(const char *typeName) { - return getPythonTypeObject(getConverter(typeName)); + auto *type = getPythonTypeObject(getConverter(typeName)); + if (type == nullptr) { + // PYSIDE-2404: Did not find the name. Load the lazy classes + // which have this name and try again. + Shiboken::Module::loadLazyClassesWithName(getRealTypeName(typeName).c_str()); + type = getPythonTypeObject(getConverter(typeName)); + } + return type; } bool pythonTypeIsValueType(const SbkConverter *converter) @@ -748,4 +782,4 @@ void SpecificConverter::toCpp(PyObject *pyIn, void *cppOut) } } -} } // namespace Shiboken::Conversions +} // namespace Shiboken::Conversions diff --git a/sources/shiboken6/libshiboken/sbkconverter.h b/sources/shiboken6/libshiboken/sbkconverter.h index 1428b90e2..0d68f3faf 100644 --- a/sources/shiboken6/libshiboken/sbkconverter.h +++ b/sources/shiboken6/libshiboken/sbkconverter.h @@ -5,9 +5,9 @@ #define SBK_CONVERTER_H #include "sbkpython.h" +#include "sbkmodule.h" #include "shibokenmacros.h" #include "sbkenum.h" -#include "sbkenum_p.h" #include "basewrapper_p.h" #include <limits> @@ -43,7 +43,7 @@ struct SbkArrayConverter; * * C++ -> Python */ -typedef PyObject *(*CppToPythonFunc)(const void *); +using CppToPythonFunc = PyObject *(*)(const void *); /** * This function converts a Python object to a C++ value, it may be @@ -56,7 +56,7 @@ typedef PyObject *(*CppToPythonFunc)(const void *); * * Python -> C++ */ -typedef void (*PythonToCppFunc)(PyObject *,void *); +using PythonToCppFunc = void (*)(PyObject *,void *); /** * Checks if the Python object passed in the argument is convertible to a @@ -67,7 +67,7 @@ typedef void (*PythonToCppFunc)(PyObject *,void *); * * Python -> C++ ? */ -typedef PythonToCppFunc (*IsConvertibleToCppFunc)(PyObject *); +using IsConvertibleToCppFunc = PythonToCppFunc (*)(PyObject *); } // extern "C" @@ -147,6 +147,9 @@ LIBSHIBOKEN_API void addPythonToCppValueConversion(SbkConverter *converter, LIBSHIBOKEN_API void addPythonToCppValueConversion(PyTypeObject *type, PythonToCppFunc pythonToCppFunc, IsConvertibleToCppFunc isConvertibleToCppFunc); +LIBSHIBOKEN_API void addPythonToCppValueConversion(Shiboken::Module::TypeInitStruct typeStruct, + PythonToCppFunc pythonToCppFunc, + IsConvertibleToCppFunc isConvertibleToCppFunc); // C++ -> Python --------------------------------------------------------------------------- @@ -204,6 +207,7 @@ struct PythonToCppConversion */ LIBSHIBOKEN_API PythonToCppFunc isPythonToCppPointerConvertible(PyTypeObject *type, PyObject *pyIn); LIBSHIBOKEN_API PythonToCppConversion pythonToCppPointerConversion(PyTypeObject *type, PyObject *pyIn); +LIBSHIBOKEN_API PythonToCppConversion pythonToCppPointerConversion(Module::TypeInitStruct typeStruct, PyObject *pyIn); /** * Returns a Python to C++ conversion function if the Python object is convertible to a C++ value. @@ -410,7 +414,7 @@ template<> inline PyTypeObject *SbkType<std::nullptr_t>() { return Py_TYPE(&_Py_ #define SbkChar_Check(X) (PyNumber_Check(X) || Shiboken::String::checkChar(X)) struct PySideQFlagsType; -struct PySideQFlagsTypePrivate +struct SbkQFlagsTypePrivate { SbkConverter *converter; }; diff --git a/sources/shiboken6/libshiboken/sbkconverter_p.h b/sources/shiboken6/libshiboken/sbkconverter_p.h index 27126fbb1..c886c9b9f 100644 --- a/sources/shiboken6/libshiboken/sbkconverter_p.h +++ b/sources/shiboken6/libshiboken/sbkconverter_p.h @@ -278,7 +278,7 @@ struct Primitive<PY_LONG_LONG> : OnePrimitive<PY_LONG_LONG> { PY_LONG_LONG result = PyLong_AsLongLong(pyIn); if (OverFlowChecker<PY_LONG_LONG>::check(result, pyIn)) - PyErr_SetObject(PyExc_OverflowError, 0); + PyErr_SetObject(PyExc_OverflowError, nullptr); *reinterpret_cast<PY_LONG_LONG * >(cppOut) = result; } static PythonToCppFunc isConvertible(PyObject *pyIn) @@ -327,7 +327,7 @@ struct FloatPrimitive : TwoPrimitive<FLOAT> } static void toCpp(PyObject *pyIn, void *cppOut) { - *reinterpret_cast<FLOAT *>(cppOut) = FLOAT(PyLong_AsLong(pyIn)); + *reinterpret_cast<FLOAT *>(cppOut) = FLOAT(PyLong_AsDouble(pyIn)); } static PythonToCppFunc isConvertible(PyObject *pyIn) { @@ -524,13 +524,13 @@ struct Primitive<std::nullptr_t> : OnePrimitive<std::nullptr_t> } }; -namespace Shiboken { -namespace Conversions { +namespace Shiboken::Conversions { + SbkConverter *createConverterObject(PyTypeObject *type, PythonToCppFunc toCppPointerConvFunc, IsConvertibleToCppFunc toCppPointerCheckFunc, CppToPythonFunc pointerToPythonFunc, CppToPythonFunc copyToPythonFunc); -} // namespace Conversions -} // namespace Shiboken +} // namespace Shiboken::Conversions + #endif // SBK_CONVERTER_P_H diff --git a/sources/shiboken6/libshiboken/sbkcppstring.cpp b/sources/shiboken6/libshiboken/sbkcppstring.cpp index 42b09111c..8e8324f5e 100644 --- a/sources/shiboken6/libshiboken/sbkcppstring.cpp +++ b/sources/shiboken6/libshiboken/sbkcppstring.cpp @@ -12,6 +12,11 @@ PyObject *fromCppString(const std::string &value) return PyUnicode_FromStringAndSize(value.data(), value.size()); } +PyObject *fromCppStringView(std::string_view value) +{ + return PyUnicode_FromStringAndSize(value.data(), value.size()); +} + PyObject *fromCppWString(const std::wstring &value) { return PyUnicode_FromWideChar(value.data(), value.size()); diff --git a/sources/shiboken6/libshiboken/sbkcppstring.h b/sources/shiboken6/libshiboken/sbkcppstring.h index f418ea8dd..7ffe11c75 100644 --- a/sources/shiboken6/libshiboken/sbkcppstring.h +++ b/sources/shiboken6/libshiboken/sbkcppstring.h @@ -8,10 +8,12 @@ #include "shibokenmacros.h" #include <string> +#include <string_view> namespace Shiboken::String { LIBSHIBOKEN_API PyObject *fromCppString(const std::string &value); + LIBSHIBOKEN_API PyObject *fromCppStringView(std::string_view value); LIBSHIBOKEN_API PyObject *fromCppWString(const std::wstring &value); LIBSHIBOKEN_API void toCppString(PyObject *str, std::string *value); LIBSHIBOKEN_API void toCppWString(PyObject *str, std::wstring *value); diff --git a/sources/shiboken6/libshiboken/sbkcpptonumpy.cpp b/sources/shiboken6/libshiboken/sbkcpptonumpy.cpp new file mode 100644 index 000000000..44e900f01 --- /dev/null +++ b/sources/shiboken6/libshiboken/sbkcpptonumpy.cpp @@ -0,0 +1,67 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +// included by sbknumpy.cpp + +namespace Shiboken::Numpy { + +#ifdef HAVE_NUMPY + +// Helper to create a 1-dimensional numpy array +template <class Type> +static PyObject *_createArray1(Py_ssize_t size, int numpyType, const Type *data) +{ + const npy_intp dims[1] = {size}; + PyObject *result = PyArray_EMPTY(1, dims, numpyType, 0); + auto *array = reinterpret_cast<PyArrayObject *>(result); + auto *rawTargetData = PyArray_DATA(array); + auto *targetData = reinterpret_cast<Type *>(rawTargetData); + std::copy(data, data + size, targetData); + return result; +} + +PyObject *createByteArray1(Py_ssize_t size, const uint8_t *data) +{ + return _createArray1(size, NPY_BYTE, data); +} + +PyObject *createDoubleArray1(Py_ssize_t size, const double *data) +{ + return _createArray1(size, NPY_DOUBLE, data); +} + +PyObject *createFloatArray1(Py_ssize_t size, const float *data) +{ + return _createArray1(size, NPY_FLOAT, data); +} + +PyObject *createIntArray1(Py_ssize_t size, const int *data) +{ + return _createArray1(size, NPY_INT, data); +} + +#else // HAVE_NUMPY + +PyObject *createByteArray1(Py_ssize_t, const uint8_t *) +{ + return Py_None; +} + +PyObject *createDoubleArray1(Py_ssize_t, const double *) +{ + return Py_None; +} + +PyObject *createFloatArray1(Py_ssize_t, const float *) +{ + return Py_None; +} + +PyObject *createIntArray1(Py_ssize_t, const int *) +{ + return Py_None; +} + +#endif // !HAVE_NUMPY + +} //namespace Shiboken::Numpy diff --git a/sources/shiboken6/libshiboken/sbkcpptonumpy.h b/sources/shiboken6/libshiboken/sbkcpptonumpy.h new file mode 100644 index 000000000..8b9b0cfd2 --- /dev/null +++ b/sources/shiboken6/libshiboken/sbkcpptonumpy.h @@ -0,0 +1,41 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef SBKCPPTONUMPY_H +#define SBKCPPTONUMPY_H + +#include <sbkpython.h> +#include <shibokenmacros.h> + +#include <cstdint> + +namespace Shiboken::Numpy +{ + +/// Create a one-dimensional numpy array of type uint8/NPY_BYTE +/// \param size Size +/// \param data Data +/// \return PyArrayObject +LIBSHIBOKEN_API PyObject *createByteArray1(Py_ssize_t size, const uint8_t *data); + +/// Create a one-dimensional numpy array of type double/NPY_DOUBLE +/// \param size Size +/// \param data Data +/// \return PyArrayObject +LIBSHIBOKEN_API PyObject *createDoubleArray1(Py_ssize_t size, const double *data); + +/// Create a one-dimensional numpy array of type float/NPY_FLOAT +/// \param size Size +/// \param data Data +/// \return PyArrayObject +LIBSHIBOKEN_API PyObject *createFloatArray1(Py_ssize_t size, const float *data); + +/// Create a one-dimensional numpy array of type int/NPY_INT +/// \param size Size +/// \param data Data +/// \return PyArrayObject +LIBSHIBOKEN_API PyObject *createIntArray1(Py_ssize_t size, const int *data); + +} //namespace Shiboken::Numpy + +#endif // SBKCPPTONUMPY_H diff --git a/sources/shiboken6/libshiboken/sbkenum.cpp b/sources/shiboken6/libshiboken/sbkenum.cpp index 3f432dcb2..d39369979 100644 --- a/sources/shiboken6/libshiboken/sbkenum.cpp +++ b/sources/shiboken6/libshiboken/sbkenum.cpp @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "sbkenum.h" -#include "sbkenum_p.h" #include "sbkstring.h" #include "sbkstaticstrings.h" #include "sbkstaticstrings_p.h" @@ -16,389 +15,35 @@ #include <vector> #include <sstream> -#define SbkEnumType_Check(o) (Py_TYPE(Py_TYPE(o)) == SbkEnumType_TypeF()) -using enum_func = PyObject *(*)(PyObject *, PyObject *); - using namespace Shiboken; extern "C" { -// forward -struct lastEnumCreated; - -// forward -static PyTypeObject *recordCurrentEnum(PyObject *scopeOrModule, - const char *name, - PyTypeObject *enumType, - PyTypeObject *flagsType); - struct SbkEnumType { PyTypeObject type; }; -static void cleanupEnumTypes(); - -struct SbkEnumObject -{ - PyObject_HEAD - long ob_value; - PyObject *ob_name; -}; - -static PyTypeObject *SbkEnum_TypeF(); // forward - -static PyObject *SbkEnumObject_repr(PyObject *self) -{ - const SbkEnumObject *enumObj = reinterpret_cast<SbkEnumObject *>(self); - auto name = Py_TYPE(self)->tp_name; - if (enumObj->ob_name) { - return String::fromFormat("%s.%s", name, PyBytes_AS_STRING(enumObj->ob_name)); - } - return String::fromFormat("%s(%ld)", name, enumObj->ob_value); -} - -static PyObject *SbkEnumObject_name(PyObject *self, void *) -{ - auto *enum_self = reinterpret_cast<SbkEnumObject *>(self); - - if (enum_self->ob_name == nullptr) - Py_RETURN_NONE; - - Py_INCREF(enum_self->ob_name); - return enum_self->ob_name; -} - -static PyObject *SbkEnum_tp_new(PyTypeObject *type, PyObject *args, PyObject *) -{ - long itemValue = 0; - if (!PyArg_ParseTuple(args, "|l:__new__", &itemValue)) - return nullptr; - - if (type == SbkEnum_TypeF()) { - PyErr_Format(PyExc_TypeError, "You cannot use %s directly", type->tp_name); - return nullptr; - } - - SbkEnumObject *self = PyObject_New(SbkEnumObject, type); - if (!self) - return nullptr; - self->ob_value = itemValue; - AutoDecRef item(Enum::getEnumItemFromValue(type, itemValue)); - self->ob_name = item.object() ? SbkEnumObject_name(item, nullptr) : nullptr; - return reinterpret_cast<PyObject *>(self); -} - -static const char *SbkEnum_SignatureStrings[] = { - "Shiboken.Enum(self,itemValue:int=0)", - nullptr}; // Sentinel - -static void enum_object_dealloc(PyObject *ob) -{ - auto *self = reinterpret_cast<SbkEnumObject *>(ob); - Py_XDECREF(self->ob_name); - Sbk_object_dealloc(ob); -} - -static PyObject *_enum_op(enum_func f, PyObject *a, PyObject *b) { - PyObject *valA = a; - PyObject *valB = b; - PyObject *result = nullptr; - bool enumA = false; - bool enumB = false; - - // We are not allowing floats - if (!PyFloat_Check(valA) && !PyFloat_Check(valB)) { - // Check if both variables are SbkEnumObject - if (SbkEnumType_Check(valA)) { - valA = PyLong_FromLong(reinterpret_cast<SbkEnumObject *>(valA)->ob_value); - enumA = true; - } - if (SbkEnumType_Check(valB)) { - valB = PyLong_FromLong(reinterpret_cast<SbkEnumObject *>(valB)->ob_value); - enumB = true; - } - } - - // Without an enum we are not supporting the operation - if (!(enumA || enumB)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - - result = f(valA, valB); - - // Decreasing the reference of the used variables a and b. - if (enumA) - Py_DECREF(valA); - if (enumB) - Py_DECREF(valB); - return result; -} - -/* Notes: - * On Py3k land we use long type when using integer numbers. However, on older - * versions of Python (version 2) we need to convert it to int type, - * respectively. - * - * Thus calling PyLong_FromLong() will result in calling PyLong_FromLong in - * Py3k. - */ -static PyObject *enum_int(PyObject *v) -{ - return PyLong_FromLong(reinterpret_cast<SbkEnumObject *>(v)->ob_value); -} - -static PyObject *enum_and(PyObject *self, PyObject *b) -{ - return _enum_op(PyNumber_And, self, b); -} - -static PyObject *enum_or(PyObject *self, PyObject *b) -{ - return _enum_op(PyNumber_Or, self, b); -} - -static PyObject *enum_xor(PyObject *self, PyObject *b) -{ - return _enum_op(PyNumber_Xor, self, b); -} - -static int enum_bool(PyObject *v) -{ - return (reinterpret_cast<SbkEnumObject *>(v)->ob_value > 0); -} - -static PyObject *enum_add(PyObject *self, PyObject *v) -{ - return _enum_op(PyNumber_Add, self, v); -} - -static PyObject *enum_subtract(PyObject *self, PyObject *v) -{ - return _enum_op(PyNumber_Subtract, self, v); -} - -static PyObject *enum_multiply(PyObject *self, PyObject *v) -{ - return _enum_op(PyNumber_Multiply, self, v); -} - -static PyObject *enum_richcompare(PyObject *self, PyObject *other, int op) -{ - PyObject *valA = self; - PyObject *valB = other; - PyObject *result = nullptr; - bool enumA = false; - bool enumB = false; - - // We are not allowing floats - if (!PyFloat_Check(valA) && !PyFloat_Check(valB)) { - - // Check if both variables are SbkEnumObject - if (SbkEnumType_Check(valA)) { - valA = PyLong_FromLong(reinterpret_cast<SbkEnumObject *>(valA)->ob_value); - enumA = true; - } - if (SbkEnumType_Check(valB)) { - valB = PyLong_FromLong(reinterpret_cast<SbkEnumObject *>(valB)->ob_value); - enumB =true; - } - } - - // Without an enum we are not supporting the operation - if (!(enumA || enumB)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - result = PyObject_RichCompare(valA, valB, op); - - // Decreasing the reference of the used variables a and b. - if (enumA) - Py_DECREF(valA); - if (enumB) - Py_DECREF(valB); - - return result; -} - -static Py_hash_t enum_hash(PyObject *pyObj) -{ - Py_hash_t val = reinterpret_cast<SbkEnumObject *>(pyObj)->ob_value; - if (val == -1) - val = -2; - return val; -} - -static PyGetSetDef SbkEnumGetSetList[] = { - {const_cast<char *>("name"), SbkEnumObject_name, nullptr, nullptr, nullptr}, - {nullptr, nullptr, nullptr, nullptr, nullptr} // Sentinel -}; - -static void SbkEnumTypeDealloc(PyObject *pyObj); -static PyTypeObject *SbkEnumTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds); - -static PyGetSetDef SbkEnumType_getsetlist[] = { - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(Sbk_TypeGet___signature__), - nullptr, nullptr, nullptr}, - {nullptr, nullptr, nullptr, nullptr, nullptr} // Sentinel -}; - -static PyType_Slot SbkEnumType_Type_slots[] = { - {Py_tp_dealloc, reinterpret_cast<void *>(SbkEnumTypeDealloc)}, - {Py_tp_base, reinterpret_cast<void *>(&PyType_Type)}, - {Py_tp_alloc, reinterpret_cast<void *>(PyType_GenericAlloc)}, - {Py_tp_new, reinterpret_cast<void *>(SbkEnumTypeTpNew)}, - {Py_tp_free, reinterpret_cast<void *>(PyObject_GC_Del)}, - {Py_tp_getset, reinterpret_cast<void *>(SbkEnumType_getsetlist)}, - {0, nullptr} -}; - -// PYSIDE-535: The tp_itemsize field is inherited and does not need to be set. -// In PyPy, it _must_ not be set, because it would have the meaning that a -// `__len__` field must be defined. Not doing so creates a hard-to-find crash. -static PyType_Spec SbkEnumType_Type_spec = { - "1:Shiboken.EnumMeta", - 0, - 0, // sizeof(PyMemberDef), not for PyPy without a __len__ defined - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, - SbkEnumType_Type_slots, -}; - -PyTypeObject *SbkEnumType_TypeF(void) -{ - static auto *type = SbkType_FromSpec(&SbkEnumType_Type_spec); - return type; -} - -static void SbkEnumTypeDealloc(PyObject *pyObj) -{ - auto *enumType = reinterpret_cast<SbkEnumType *>(pyObj); - auto *setp = PepType_SETP(enumType); - - PyObject_GC_UnTrack(pyObj); -#ifndef Py_LIMITED_API -# if PY_VERSION_HEX >= 0x030A0000 - Py_TRASHCAN_BEGIN(pyObj, 1); -# else - Py_TRASHCAN_SAFE_BEGIN(pyObj); -# endif -#endif - if (setp->converter) - Conversions::deleteConverter(setp->converter); - PepType_SETP_delete(enumType); -#ifndef Py_LIMITED_API -# if PY_VERSION_HEX >= 0x030A0000 - Py_TRASHCAN_END; -# else - Py_TRASHCAN_SAFE_END(pyObj); -# endif -#endif - if (PepRuntime_38_flag) { - // PYSIDE-939: Handling references correctly. - // This was not needed before Python 3.8 (Python issue 35810) - Py_DECREF(Py_TYPE(pyObj)); - } -} - -PyTypeObject *SbkEnumTypeTpNew(PyTypeObject *metatype, PyObject *args, PyObject *kwds) -{ - init_enum(); - return PepType_Type_tp_new(metatype, args, kwds); -} - -} // extern "C" - -/////////////////////////////////////////////////////////////// -// -// PYSIDE-15: Pickling Support for Qt Enum objects -// This works very well and fixes the issue. -// -extern "C" { - -static PyObject *enum_unpickler = nullptr; - -// Pickling: reduce the Qt Enum object -static PyObject *enum___reduce__(PyObject *obj) -{ - init_enum(); - return Py_BuildValue("O(Ni)", - enum_unpickler, - Py_BuildValue("s", Py_TYPE(obj)->tp_name), - PyLong_AS_LONG(obj)); -} - -} // extern "C" - -namespace Shiboken { namespace Enum { - -// Unpickling: rebuild the Qt Enum object -PyObject *unpickleEnum(PyObject *enum_class_name, PyObject *value) -{ - AutoDecRef parts(PyObject_CallMethod(enum_class_name, - "split", "s", ".")); - if (parts.isNull()) - return nullptr; - PyObject *top_name = PyList_GetItem(parts, 0); // borrowed ref - if (top_name == nullptr) - return nullptr; - PyObject *module = PyImport_GetModule(top_name); - if (module == nullptr) { - PyErr_Format(PyExc_ImportError, "could not import module %.200s", - String::toCString(top_name)); - return nullptr; - } - AutoDecRef cur_thing(module); - int len = PyList_Size(parts); - for (int idx = 1; idx < len; ++idx) { - PyObject *name = PyList_GetItem(parts, idx); // borrowed ref - PyObject *thing = PyObject_GetAttr(cur_thing, name); - if (thing == nullptr) { - PyErr_Format(PyExc_ImportError, "could not import Qt Enum type %.200s", - String::toCString(enum_class_name)); - return nullptr; - } - cur_thing.reset(thing); - } - PyObject *klass = cur_thing; - return PyObject_CallFunctionObjArgs(klass, value, nullptr); -} - -} // namespace Enum -} // namespace Shiboken - -extern "C" { - // Initialization static bool _init_enum() { AutoDecRef shibo(PyImport_ImportModule("shiboken6.Shiboken")); - auto mod = shibo.object(); - // publish Shiboken.Enum so that the signature gets initialized - if (PyObject_SetAttrString(mod, "Enum", reinterpret_cast<PyObject *>(SbkEnum_TypeF())) < 0) - return false; - if (InitSignatureStrings(SbkEnum_TypeF(), SbkEnum_SignatureStrings) < 0) - return false; - enum_unpickler = PyObject_GetAttrString(mod, "_unpickle_enum"); - if (enum_unpickler == nullptr) - return false; - return true; + return !shibo.isNull(); } -static int useOldEnum = -1; - -static PyMethodDef SbkEnumObject_Methods[] = { - {"__reduce__", reinterpret_cast<PyCFunction>(enum___reduce__), - METH_NOARGS, nullptr}, - {nullptr, nullptr, 0, nullptr} // Sentinel -}; - +static PyObject *PyEnumModule{}; static PyObject *PyEnumMeta{}; static PyObject *PyEnum{}; static PyObject *PyIntEnum{}; static PyObject *PyFlag{}; static PyObject *PyIntFlag{}; +static PyObject *PyFlag_KEEP{}; + +bool PyEnumMeta_Check(PyObject *ob) +{ + return Py_TYPE(ob) == reinterpret_cast<PyTypeObject *>(PyEnumMeta); +} PyTypeObject *getPyEnumMeta() { @@ -407,6 +52,7 @@ PyTypeObject *getPyEnumMeta() static auto *mod = PyImport_ImportModule("enum"); if (mod) { + PyEnumModule = mod; PyEnumMeta = PyObject_GetAttrString(mod, "EnumMeta"); if (PyEnumMeta && PyType_Check(PyEnumMeta)) PyEnum = PyObject_GetAttrString(mod, "Enum"); @@ -416,8 +62,12 @@ PyTypeObject *getPyEnumMeta() PyFlag = PyObject_GetAttrString(mod, "Flag"); if (PyFlag && PyType_Check(PyFlag)) PyIntFlag = PyObject_GetAttrString(mod, "IntFlag"); - if (PyIntFlag && PyType_Check(PyIntFlag)) + if (PyIntFlag && PyType_Check(PyIntFlag)) { + // KEEP is defined from Python 3.11 on. + PyFlag_KEEP = PyObject_GetAttrString(mod, "KEEP"); + PyErr_Clear(); return reinterpret_cast<PyTypeObject *>(PyEnumMeta); + } } Py_FatalError("Python module 'enum' not found"); return nullptr; @@ -425,23 +75,22 @@ PyTypeObject *getPyEnumMeta() void init_enum() { - static bool is_initialized = false; - if (is_initialized) + static bool isInitialized = false; + if (isInitialized) return; - if (!(is_initialized || enum_unpickler || _init_enum())) - Py_FatalError("could not load enum pickling helper function"); - Py_AtExit(cleanupEnumTypes); + if (!(isInitialized || _init_enum())) + Py_FatalError("could not init enum"); // PYSIDE-1735: Determine whether we should use the old or the new enum implementation. - static const char *envname = "PYSIDE63_OPTION_PYTHON_ENUM"; - const char *envsetting = getenv(envname); - // I tried to use the save version getenv_s instead, but this function does not - // exist on macOS. But this does no harm: - // This variable has been set already by parser.py initialization. - assert(envsetting); - useOldEnum = strncmp(envsetting, "0", 10) == 0; + static PyObject *option = PySys_GetObject("pyside6_option_python_enum"); + if (!option || !PyLong_Check(option)) { + PyErr_Clear(); + option = PyLong_FromLong(1); + } + int ignoreOver{}; + Enum::enumOption = PyLong_AsLongAndOverflow(option, &ignoreOver); getPyEnumMeta(); - is_initialized = true; + isInitialized = true; } // PYSIDE-1735: Helper function supporting QEnum @@ -453,8 +102,8 @@ int enumIsFlag(PyObject *ob_type) if (metatype != reinterpret_cast<PyTypeObject *>(PyEnumMeta)) return -1; auto *mro = reinterpret_cast<PyTypeObject *>(ob_type)->tp_mro; - Py_ssize_t idx, n = PyTuple_GET_SIZE(mro); - for (idx = 0; idx < n; idx++) { + const Py_ssize_t n = PyTuple_GET_SIZE(mro); + for (Py_ssize_t idx = 0; idx < n; ++idx) { auto *sub_type = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(mro, idx)); if (sub_type == reinterpret_cast<PyTypeObject *>(PyFlag)) return 1; @@ -462,84 +111,121 @@ int enumIsFlag(PyObject *ob_type) return 0; } -} // extern "C" - +/////////////////////////////////////////////////////////////////////// +// +// Support for Missing Values +// ========================== +// +// Qt enums sometimes use undefined values in enums. +// The enum module handles this by the option "KEEP" for Flag and +// IntFlag. The handling of missing enum values is still strict. +// +// We changed that (also for compatibility with some competitor) +// and provide a `_missing_` function that creates the missing value. +// +// The idea: +// --------- +// We cannot modify the already created class. +// But we can create a one-element class with the new value and +// pretend that this is the already existing class. +// +// We create each constant only once and keep the result in a dict +// "_sbk_missing_". This is similar to a competitor's "_sip_missing_". // -/////////////////////////////////////////////////////////////// - -namespace Shiboken { -class DeclaredEnumTypes +static PyObject *missing_func(PyObject * /* self */ , PyObject *args) { -public: - struct EnumEntry - { - char *name; // full name as allocated. type->tp_name might be a substring. - PyTypeObject *type; - }; - - DeclaredEnumTypes(const DeclaredEnumTypes &) = delete; - DeclaredEnumTypes(DeclaredEnumTypes &&) = delete; - DeclaredEnumTypes &operator=(const DeclaredEnumTypes &) = delete; - DeclaredEnumTypes &operator=(DeclaredEnumTypes &&) = delete; - - DeclaredEnumTypes(); - ~DeclaredEnumTypes(); - static DeclaredEnumTypes &instance(); - void addEnumType(const EnumEntry &e) { m_enumTypes.push_back(e); } - - void cleanup(); - -private: - std::vector<EnumEntry> m_enumTypes; + // In order to relax matters to be more compatible with C++, we need + // to create a pseudo-member with that value. + static auto *const _sbk_missing = Shiboken::String::createStaticString("_sbk_missing_"); + static auto *const _name = Shiboken::String::createStaticString("__name__"); + static auto *const _mro = Shiboken::String::createStaticString("__mro__"); + static auto *const _class = Shiboken::String::createStaticString("__class__"); + + PyObject *klass{}, *value{}; + if (!PyArg_UnpackTuple(args, "missing", 2, 2, &klass, &value)) + Py_RETURN_NONE; + if (!PyLong_Check(value)) + Py_RETURN_NONE; + auto *type = reinterpret_cast<PyTypeObject *>(klass); + AutoDecRef tpDict(PepType_GetDict(type)); + auto *sbk_missing = PyDict_GetItem(tpDict.object(), _sbk_missing); + if (!sbk_missing) { + sbk_missing = PyDict_New(); + PyDict_SetItem(tpDict.object(), _sbk_missing, sbk_missing); + } + // See if the value is already in the dict. + AutoDecRef val_str(PyObject_CallMethod(value, "__str__", nullptr)); + auto *ret = PyDict_GetItem(sbk_missing, val_str); + if (ret) { + Py_INCREF(ret); + return ret; + } + // No, we must create a new object and insert it into the dict. + AutoDecRef cls_name(PyObject_GetAttr(klass, _name)); + AutoDecRef mro(PyObject_GetAttr(klass, _mro)); + auto *baseClass(PyTuple_GetItem(mro, 1)); + AutoDecRef param(PyDict_New()); + PyDict_SetItem(param, val_str, value); + AutoDecRef fake(PyObject_CallFunctionObjArgs(baseClass, cls_name.object(), param.object(), + nullptr)); + ret = PyObject_GetAttr(fake, val_str); + PyDict_SetItem(sbk_missing, val_str, ret); + // Now the real fake: Pretend that the type is our original type! + PyObject_SetAttr(ret, _class, klass); + return ret; +} + +static struct PyMethodDef dummy_methods[] = { + {"_missing_", reinterpret_cast<PyCFunction>(missing_func), METH_VARARGS|METH_STATIC, nullptr}, + {nullptr, nullptr, 0, nullptr} +}; + +static PyType_Slot dummy_slots[] = { + {Py_tp_base, reinterpret_cast<void *>(&PyType_Type)}, + {Py_tp_methods, reinterpret_cast<void *>(dummy_methods)}, + {0, nullptr} }; -namespace Enum { +static PyType_Spec dummy_spec = { + "1:builtins.EnumType", + 0, + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + dummy_slots, +}; -// forward -static PyObject *newItemOld(PyTypeObject *enumType, long itemValue, const char *itemName); +static PyObject *create_missing_func(PyObject *klass) +{ + // When creating the class, memorize it in the missing function by + // a partial function argument. + static auto *const type = SbkType_FromSpec(&dummy_spec); + static auto *const obType = reinterpret_cast<PyObject *>(type); + static auto *const _missing = Shiboken::String::createStaticString("_missing_"); + static auto *const func = PyObject_GetAttr(obType, _missing); + static auto *const partial = Pep_GetPartialFunction(); + return PyObject_CallFunctionObjArgs(partial, func, klass, nullptr); +} +// +//////////////////////////////////////////////////////////////////////// -// forward -static PyTypeObject * newTypeWithNameOld(const char *name, - const char *cppName, - PyTypeObject *numbers_fromFlag); +} // extern "C" + +namespace Shiboken::Enum { + +int enumOption{}; bool check(PyObject *pyObj) { init_enum(); - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return Py_TYPE(Py_TYPE(pyObj)) == SbkEnumType_TypeF(); - static PyTypeObject *meta = getPyEnumMeta(); return Py_TYPE(Py_TYPE(pyObj)) == reinterpret_cast<PyTypeObject *>(meta); } -static PyObject *getEnumItemFromValueOld(PyTypeObject *enumType, long itemValue) -{ - PyObject *key, *value; - Py_ssize_t pos = 0; - PyObject *values = PyDict_GetItem(enumType->tp_dict, PyName::values()); - if (values == nullptr) - return nullptr; - - while (PyDict_Next(values, &pos, &key, &value)) { - auto *obj = reinterpret_cast<SbkEnumObject *>(value); - if (obj->ob_value == itemValue) { - Py_INCREF(value); - return value; - } - } - return nullptr; -} - -PyObject *getEnumItemFromValue(PyTypeObject *enumType, long itemValue) +PyObject *getEnumItemFromValue(PyTypeObject *enumType, EnumValueType itemValue) { init_enum(); - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return getEnumItemFromValueOld(enumType, itemValue); auto *obEnumType = reinterpret_cast<PyObject *>(enumType); AutoDecRef val2members(PyObject_GetAttrString(obEnumType, "_value2member_map_")); @@ -547,467 +233,220 @@ PyObject *getEnumItemFromValue(PyTypeObject *enumType, long itemValue) PyErr_Clear(); return nullptr; } - AutoDecRef ob_value(PyLong_FromLong(itemValue)); + AutoDecRef ob_value(PyLong_FromLongLong(itemValue)); auto *result = PyDict_GetItem(val2members, ob_value); Py_XINCREF(result); return result; } -static PyTypeObject *createEnum(const char *fullName, const char *cppName, - PyTypeObject *flagsType) +PyObject *newItem(PyTypeObject *enumType, EnumValueType itemValue, + const char *itemName) { init_enum(); - PyTypeObject *enumType = newTypeWithNameOld(fullName, cppName, flagsType); - if (PyType_Ready(enumType) < 0) { - Py_XDECREF(enumType); - return nullptr; - } - return enumType; -} -PyTypeObject *createGlobalEnum(PyObject *module, const char *name, const char *fullName, - const char *cppName, PyTypeObject *flagsType) -{ - PyTypeObject *enumType = createEnum(fullName, cppName, flagsType); - if (enumType && PyModule_AddObject(module, name, reinterpret_cast<PyObject *>(enumType)) < 0) { - Py_DECREF(enumType); - return nullptr; - } - flagsType = recordCurrentEnum(module, name, enumType, flagsType); - if (flagsType && PyModule_AddObject(module, PepType_GetNameStr(flagsType), - reinterpret_cast<PyObject *>(flagsType)) < 0) { - Py_DECREF(enumType); - return nullptr; - } - return enumType; -} + auto *obEnumType = reinterpret_cast<PyObject *>(enumType); + if (!itemName) + return PyObject_CallFunction(obEnumType, "L", itemValue); -PyTypeObject *createScopedEnum(PyTypeObject *scope, const char *name, const char *fullName, - const char *cppName, PyTypeObject *flagsType) -{ - PyTypeObject *enumType = createEnum(fullName, cppName, flagsType); - if (enumType && PyDict_SetItemString(scope->tp_dict, name, - reinterpret_cast<PyObject *>(enumType)) < 0) { - Py_DECREF(enumType); - return nullptr; - } - auto *obScope = reinterpret_cast<PyObject *>(scope); - flagsType = recordCurrentEnum(obScope, name, enumType, flagsType); - if (flagsType && PyDict_SetItemString(scope->tp_dict, - PepType_GetNameStr(flagsType), - reinterpret_cast<PyObject *>(flagsType)) < 0) { - Py_DECREF(enumType); + static PyObject *const _member_map_ = String::createStaticString("_member_map_"); + AutoDecRef tpDict(PepType_GetDict(enumType)); + auto *member_map = PyDict_GetItem(tpDict.object(), _member_map_); + if (!(member_map && PyDict_Check(member_map))) return nullptr; - } - return enumType; + auto *result = PyDict_GetItemString(member_map, itemName); + Py_XINCREF(result); + return result; } -static PyObject *createEnumItem(PyTypeObject *enumType, const char *itemName, long itemValue) +EnumValueType getValue(PyObject *enumItem) { init_enum(); - PyObject *enumItem = newItemOld(enumType, itemValue, itemName); - if (PyDict_SetItemString(enumType->tp_dict, itemName, enumItem) < 0) { - Py_DECREF(enumItem); - return nullptr; - } - return enumItem; -} -bool createGlobalEnumItem(PyTypeObject *enumType, PyObject *module, const char *itemName, long itemValue) -{ - PyObject *enumItem = createEnumItem(enumType, itemName, itemValue); - if (!enumItem) - return false; - int ok = useOldEnum ? PyModule_AddObject(module, itemName, enumItem) : true; - Py_DECREF(enumItem); - return ok >= 0; + assert(Enum::check(enumItem)); + + AutoDecRef pyValue(PyObject_GetAttrString(enumItem, "value")); + return PyLong_AsLongLong(pyValue); } -bool createScopedEnumItem(PyTypeObject *enumType, PyTypeObject *scope, - const char *itemName, long itemValue) +void setTypeConverter(PyTypeObject *type, SbkConverter *converter) { - PyObject *enumItem = createEnumItem(enumType, itemName, itemValue); - if (!enumItem) - return false; - int ok = useOldEnum ? PyDict_SetItemString(scope->tp_dict, itemName, enumItem) : true; - Py_DECREF(enumItem); - return ok >= 0; + auto *enumType = reinterpret_cast<SbkEnumType *>(type); + PepType_SETP(enumType)->converter = converter; } -// This exists temporary as the old way to create an enum item. -// For the public interface, we use a new function -static PyObject * -newItemOld(PyTypeObject *enumType, long itemValue, const char *itemName) +static PyTypeObject *createEnumForPython(PyObject *scopeOrModule, + const char *fullName, + PyObject *pyEnumItems) { - bool newValue = true; - SbkEnumObject *enumObj; - if (!itemName) { - enumObj = reinterpret_cast<SbkEnumObject *>( - getEnumItemFromValue(enumType, itemValue)); - if (enumObj) - return reinterpret_cast<PyObject *>(enumObj); - - newValue = false; - } - - enumObj = PyObject_New(SbkEnumObject, enumType); - if (!enumObj) - return nullptr; + const char *colon = strchr(fullName, ':'); + assert(colon); + int package_level = atoi(fullName); + const char *mod = colon + 1; - enumObj->ob_name = itemName ? PyBytes_FromString(itemName) : nullptr; - enumObj->ob_value = itemValue; - - if (newValue) { - auto dict = enumType->tp_dict; // Note: 'values' is borrowed - PyObject *values = PyDict_GetItemWithError(dict, PyName::values()); - if (values == nullptr) { - if (PyErr_Occurred()) - return nullptr; - AutoDecRef new_values(values = PyDict_New()); - if (values == nullptr) - return nullptr; - if (PyDict_SetItem(dict, PyName::values(), values) < 0) - return nullptr; - } - PyDict_SetItemString(values, itemName, reinterpret_cast<PyObject *>(enumObj)); + const char *qual = mod; + for (int idx = package_level; idx > 0; --idx) { + const char *dot = strchr(qual, '.'); + if (!dot) + break; + qual = dot + 1; + } + int mlen = qual - mod - 1; + AutoDecRef module(Shiboken::String::fromCString(mod, mlen)); + AutoDecRef qualname(Shiboken::String::fromCString(qual)); + const char *dot = strrchr(qual, '.'); + AutoDecRef name(Shiboken::String::fromCString(dot ? dot + 1 : qual)); + + static PyObject *enumName = String::createStaticString("IntEnum"); + if (PyType_Check(scopeOrModule)) { + // For global objects, we have no good solution, yet where to put the int info. + auto type = reinterpret_cast<PyTypeObject *>(scopeOrModule); + auto *sotp = PepType_SOTP(type); + if (!sotp->enumFlagsDict) + initEnumFlagsDict(type); + enumName = PyDict_GetItem(sotp->enumTypeDict, name); } - return reinterpret_cast<PyObject *>(enumObj); -} + AutoDecRef PyEnumType(PyObject_GetAttr(PyEnumModule, enumName)); + assert(PyEnumType.object()); + bool isFlag = PyObject_IsSubclass(PyEnumType, PyFlag); -PyObject * -newItem(PyTypeObject *enumType, long itemValue, const char *itemName) -{ - init_enum(); - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return newItemOld(enumType, itemValue, itemName); - - if (!itemName) { - //PyObject *enumObj = getEnumItemFromValue(enumType, itemValue); - PyObject *enumObj = PyObject_CallFunction(reinterpret_cast<PyObject *>(enumType), "i", itemValue); - //if (enumObj) - return enumObj; + // See if we should use the Int versions of the types, again + bool useIntInheritance = Enum::enumOption & Enum::ENOPT_INHERIT_INT; + if (useIntInheritance) { + auto *surrogate = PyObject_IsSubclass(PyEnumType, PyFlag) ? PyIntFlag : PyIntEnum; + Py_INCREF(surrogate); + PyEnumType.reset(surrogate); } - return PyObject_GetAttrString(reinterpret_cast<PyObject *>(enumType), itemName); -} -} // namespace Shiboken -} // namespace Enum - -static PyType_Slot SbkNewEnum_slots[] = { - {Py_tp_repr, reinterpret_cast<void *>(SbkEnumObject_repr)}, - {Py_tp_str, reinterpret_cast<void *>(SbkEnumObject_repr)}, - {Py_tp_getset, reinterpret_cast<void *>(SbkEnumGetSetList)}, - {Py_tp_methods, reinterpret_cast<void *>(SbkEnumObject_Methods)}, - {Py_tp_new, reinterpret_cast<void *>(SbkEnum_tp_new)}, - {Py_nb_add, reinterpret_cast<void *>(enum_add)}, - {Py_nb_subtract, reinterpret_cast<void *>(enum_subtract)}, - {Py_nb_multiply, reinterpret_cast<void *>(enum_multiply)}, - {Py_nb_positive, reinterpret_cast<void *>(enum_int)}, - {Py_nb_bool, reinterpret_cast<void *>(enum_bool)}, - {Py_nb_and, reinterpret_cast<void *>(enum_and)}, - {Py_nb_xor, reinterpret_cast<void *>(enum_xor)}, - {Py_nb_or, reinterpret_cast<void *>(enum_or)}, - {Py_nb_int, reinterpret_cast<void *>(enum_int)}, - {Py_nb_index, reinterpret_cast<void *>(enum_int)}, - {Py_tp_richcompare, reinterpret_cast<void *>(enum_richcompare)}, - {Py_tp_hash, reinterpret_cast<void *>(enum_hash)}, - {Py_tp_dealloc, reinterpret_cast<void *>(enum_object_dealloc)}, - {0, nullptr} -}; -static PyType_Spec SbkNewEnum_spec = { - "1:Shiboken.Enum", - sizeof(SbkEnumObject), - 0, - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, - SbkNewEnum_slots, -}; + // Walk the enumItemStrings and create a Python enum type. + auto *pyName = name.object(); -static PyTypeObject *SbkEnum_TypeF() -{ - static auto type = SbkType_FromSpecWithMeta(&SbkNewEnum_spec, SbkEnumType_TypeF()); - return type; -} + // We now create the new type. Since Python 3.11, we need to pass in + // `boundary=KEEP` because the default STRICT crashes on us. + // See QDir.Filter.Drives | QDir.Filter.Files + AutoDecRef callArgs(Py_BuildValue("(OO)", pyName, pyEnumItems)); + AutoDecRef callDict(PyDict_New()); + static PyObject *boundary = String::createStaticString("boundary"); + if (PyFlag_KEEP) + PyDict_SetItem(callDict, boundary, PyFlag_KEEP); + auto *obNewType = PyObject_Call(PyEnumType, callArgs, callDict); + if (!obNewType || PyObject_SetAttr(scopeOrModule, pyName, obNewType) < 0) + return nullptr; -namespace Shiboken { namespace Enum { + // For compatibility with Qt enums, provide a permissive missing method for (Int)?Enum. + if (!isFlag) { + bool supportMissing = !(Enum::enumOption & Enum::ENOPT_NO_MISSING); + if (supportMissing) { + AutoDecRef enum_missing(create_missing_func(obNewType)); + PyObject_SetAttrString(obNewType, "_missing_", enum_missing); + } + } -static void -copyNumberMethods(PyTypeObject *flagsType, - PyType_Slot number_slots[], - int *pidx) -{ - int idx = *pidx; -#define PUT_SLOT(name) \ - number_slots[idx].slot = (name); \ - number_slots[idx].pfunc = PyType_GetSlot(flagsType, (name)); \ - ++idx; - - PUT_SLOT(Py_nb_absolute); - PUT_SLOT(Py_nb_add); - PUT_SLOT(Py_nb_and); - PUT_SLOT(Py_nb_bool); - PUT_SLOT(Py_nb_divmod); - PUT_SLOT(Py_nb_float); - PUT_SLOT(Py_nb_floor_divide); - PUT_SLOT(Py_nb_index); - PUT_SLOT(Py_nb_inplace_add); - PUT_SLOT(Py_nb_inplace_and); - PUT_SLOT(Py_nb_inplace_floor_divide); - PUT_SLOT(Py_nb_inplace_lshift); - PUT_SLOT(Py_nb_inplace_multiply); - PUT_SLOT(Py_nb_inplace_or); - PUT_SLOT(Py_nb_inplace_power); - PUT_SLOT(Py_nb_inplace_remainder); - PUT_SLOT(Py_nb_inplace_rshift); - PUT_SLOT(Py_nb_inplace_subtract); - PUT_SLOT(Py_nb_inplace_true_divide); - PUT_SLOT(Py_nb_inplace_xor); - PUT_SLOT(Py_nb_int); - PUT_SLOT(Py_nb_invert); - PUT_SLOT(Py_nb_lshift); - PUT_SLOT(Py_nb_multiply); - PUT_SLOT(Py_nb_negative); - PUT_SLOT(Py_nb_or); - PUT_SLOT(Py_nb_positive); - PUT_SLOT(Py_nb_power); - PUT_SLOT(Py_nb_remainder); - PUT_SLOT(Py_nb_rshift); - PUT_SLOT(Py_nb_subtract); - PUT_SLOT(Py_nb_true_divide); - PUT_SLOT(Py_nb_xor); -#undef PUT_SLOT - *pidx = idx; -} + auto *newType = reinterpret_cast<PyTypeObject *>(obNewType); + PyObject_SetAttr(obNewType, PyMagicName::qualname(), qualname); + PyObject_SetAttr(obNewType, PyMagicName::module(), module); -static PyTypeObject * newTypeWithNameOld(const char *name, - const char *cppName, - PyTypeObject *numbers_fromFlag) -{ - // Careful: SbkType_FromSpec does not allocate the string. - PyType_Slot newslots[99] = {}; // enough but not too big for the stack - PyType_Spec newspec; - DeclaredEnumTypes::EnumEntry entry{strdup(name), nullptr}; - newspec.name = entry.name; // Note that SbkType_FromSpec might use a substring. - newspec.basicsize = SbkNewEnum_spec.basicsize; - newspec.itemsize = SbkNewEnum_spec.itemsize; - newspec.flags = SbkNewEnum_spec.flags; - // we must append all the number methods, so rebuild everything: - int idx = 0; - while (SbkNewEnum_slots[idx].slot) { - newslots[idx].slot = SbkNewEnum_slots[idx].slot; - newslots[idx].pfunc = SbkNewEnum_slots[idx].pfunc; - ++idx; + // See if we should re-introduce shortcuts in the enclosing object. + const bool useGlobalShortcut = (Enum::enumOption & Enum::ENOPT_GLOBAL_SHORTCUT) != 0; + const bool useScopedShortcut = (Enum::enumOption & Enum::ENOPT_SCOPED_SHORTCUT) != 0; + if (useGlobalShortcut || useScopedShortcut) { + // We have to use the iterator protokol because the values dict is a mappingproxy. + AutoDecRef values(PyObject_GetAttr(obNewType, PyMagicName::members())); + AutoDecRef mapIterator(PyObject_GetIter(values)); + AutoDecRef mapKey{}; + bool isModule = PyModule_Check(scopeOrModule); + while ((mapKey.reset(PyIter_Next(mapIterator))), mapKey.object()) { + if ((useGlobalShortcut && isModule) || (useScopedShortcut && !isModule)) { + AutoDecRef value(PyObject_GetItem(values, mapKey)); + if (PyObject_SetAttr(scopeOrModule, mapKey, value) < 0) + return nullptr; + } + } } - if (numbers_fromFlag) - copyNumberMethods(numbers_fromFlag, newslots, &idx); - newspec.slots = newslots; - AutoDecRef bases(PyTuple_New(1)); - static auto basetype = reinterpret_cast<PyObject *>(SbkEnum_TypeF()); - Py_INCREF(basetype); - PyTuple_SetItem(bases, 0, basetype); - auto *type = SbkType_FromSpecBasesMeta(&newspec, bases, SbkEnumType_TypeF()); - entry.type = type; - auto *enumType = reinterpret_cast<SbkEnumType *>(type); - auto *setp = PepType_SETP(enumType); - setp->cppName = cppName; - DeclaredEnumTypes::instance().addEnumType(entry); - return entry.type; + return newType; } -// PySIDE-1735: This function is in the API and should be removed in 6.4 . -// Python enums are created differently. -PyTypeObject *newTypeWithName(const char *name, - const char *cppName, - PyTypeObject *numbers_fromFlag) +template <typename IntT> +static PyObject *toPyObject(IntT v) { - if (!useOldEnum) - PyErr_Format(PyExc_RuntimeError, "function `%s` can no longer be used when the Python " - "Enum's have been selected", __FUNCTION__); - return newTypeWithNameOld(name, cppName, numbers_fromFlag); + if constexpr (sizeof(IntT) == 8) { + if constexpr (std::is_unsigned_v<IntT>) + return PyLong_FromUnsignedLongLong(v); + return PyLong_FromLongLong(v); + } + if constexpr (std::is_unsigned_v<IntT>) + return PyLong_FromUnsignedLong(v); + return PyLong_FromLong(v); } -const char *getCppName(PyTypeObject *enumType) +template <typename IntT> +static PyTypeObject *createPythonEnumHelper(PyObject *module, + const char *fullName, const char *enumItemStrings[], const IntT enumValues[]) { - assert(Py_TYPE(enumType) == SbkEnumType_TypeF()); - auto *type = reinterpret_cast<SbkEnumType *>(enumType); - auto *setp = PepType_SETP(type); - return setp->cppName; + AutoDecRef args(PyList_New(0)); + auto *pyEnumItems = args.object(); + for (size_t idx = 0; enumItemStrings[idx] != nullptr; ++idx) { + const char *kv = enumItemStrings[idx]; + auto *key = PyUnicode_FromString(kv); + auto *value = toPyObject(enumValues[idx]); + auto *key_value = PyTuple_New(2); + PyTuple_SET_ITEM(key_value, 0, key); + PyTuple_SET_ITEM(key_value, 1, value); + PyList_Append(pyEnumItems, key_value); + } + return createEnumForPython(module, fullName, pyEnumItems); } -long int getValue(PyObject *enumItem) -{ - init_enum(); - - assert(Enum::check(enumItem)); - - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return reinterpret_cast<SbkEnumObject *>(enumItem)->ob_value; - - AutoDecRef pyValue(PyObject_GetAttrString(enumItem, "value")); - return PyLong_AsLong(pyValue); -} +// Now we have to concretize these functions explicitly, +// otherwise templates will not work across modules. -void setTypeConverter(PyTypeObject *type, SbkConverter *converter, bool isFlag) +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int64_t enumValues[]) { - if (isFlag) { - auto *flagsType = reinterpret_cast<PySideQFlagsType *>(type); - PepType_PFTP(flagsType)->converter = converter; - } - else { - auto *enumType = reinterpret_cast<SbkEnumType *>(type); - PepType_SETP(enumType)->converter = converter; - } + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); } -} // namespace Enum - -DeclaredEnumTypes &DeclaredEnumTypes::instance() +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint64_t enumValues[]) { - static DeclaredEnumTypes me; - return me; + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); } -DeclaredEnumTypes::DeclaredEnumTypes() = default; - -DeclaredEnumTypes::~DeclaredEnumTypes() +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int32_t enumValues[]) { - cleanup(); + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); } -void DeclaredEnumTypes::cleanup() +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint32_t enumValues[]) { - static bool was_called = false; - if (was_called) - return; - - for (const auto &e : m_enumTypes) { - std::free(e.name); - } - m_enumTypes.clear(); - was_called = true; + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); } -} // namespace Shiboken - -static void cleanupEnumTypes() +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int16_t enumValues[]) { - DeclaredEnumTypes::instance().cleanup(); + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); } -/////////////////////////////////////////////////////////////////////// -// -// PYSIDE-1735: Re-implementation of Enums using Python -// ==================================================== -// -// This is a very simple, first implementation of a replacement -// for the Qt-like Enums using the Python Enum module. -// -// The basic idea: -// --------------- -// * We create the Enums as always -// * After creation of each enum, a special function is called that -// * grabs the last generated enum -// * reads all Enum items -// * generates a class statement for the Python Enum -// * creates a new Python Enum class -// * replaces the already inserted Enum with the new one. -// -// There are lots of ways to optimize that. Will be added later. -// -extern "C" { - -struct lastEnumCreated { - PyObject *scopeOrModule; - const char *name; - PyTypeObject *enumType; - PyTypeObject *flagsType; -}; - -static lastEnumCreated lec{}; - -static PyTypeObject *recordCurrentEnum(PyObject *scopeOrModule, - const char *name, - PyTypeObject *enumType, - PyTypeObject *flagsType) +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint16_t enumValues[]) { - lec.scopeOrModule = scopeOrModule; - lec.name = name; - lec.enumType = enumType; - lec.flagsType = flagsType; - - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return flagsType; - - // We return nullptr as flagsType to disable flag creation. - return nullptr; + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); } -PyTypeObject *morphLastEnumToPython() +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int8_t enumValues[]) { - /// The Python Enum internal structure is way too complicated. - /// It is much easier to generate Python code and execute it. - - // Pick up the last generated Enum and convert it into a PyEnum - auto *enumType = lec.enumType; - // This is temporary; SbkEnumType will be removed, soon. - - // PYSIDE-1735: Decide dynamically if new or old enums will be used. - if (useOldEnum) - return enumType; - - auto *setp = PepType_SETP(reinterpret_cast<SbkEnumType *>(enumType)); - if (setp->replacementType) { - // For some (yet to fix) reason, initialization of the enums can happen twice. - // If that happens, use the existing new type to keep type checks correct. - return setp->replacementType; - } - PyObject *key, *value; - Py_ssize_t pos = 0; - PyObject *values = PyDict_GetItem(enumType->tp_dict, PyName::values()); - if (!values) - return nullptr; - - // Walk the values dict and create a Python enum type. - auto *PyEnumType = lec.flagsType ? PyIntFlag : PyIntEnum; - AutoDecRef name(PyUnicode_FromString(lec.name)); - AutoDecRef args(PyList_New(0)); - auto *pyName = name.object(); - auto *pyArgs = args.object(); - while (PyDict_Next(values, &pos, &key, &value)) { - auto *key_value = PyTuple_New(2); - PyTuple_SET_ITEM(key_value, 0, key); - Py_INCREF(key); - auto *obj = reinterpret_cast<SbkEnumObject *>(value); - auto *num = PyLong_FromLong(obj->ob_value); - PyTuple_SET_ITEM(key_value, 1, num); - PyList_Append(pyArgs, key_value); - } - auto *obNewType = PyObject_CallFunctionObjArgs(PyEnumType, pyName, pyArgs, nullptr); - if (!obNewType || PyObject_SetAttr(lec.scopeOrModule, pyName, obNewType) < 0) - return nullptr; - auto *newType = reinterpret_cast<PyTypeObject *>(obNewType); - auto *obEnumType = reinterpret_cast<PyObject *>(enumType); - AutoDecRef qual_name(PyObject_GetAttr(obEnumType, PyMagicName::qualname())); - PyObject_SetAttr(obNewType, PyMagicName::qualname(), qual_name); - AutoDecRef module(PyObject_GetAttr(obEnumType, PyMagicName::module())); - PyObject_SetAttr(obNewType, PyMagicName::module(), module); - // Protect against double initialization - setp->replacementType = newType; -#if PY_VERSION_HEX < 0x03080000 - // PYSIDE-1735: Old Python versions can't stand the early enum deallocation. - Py_INCREF(enumType); -#endif - return newType; + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); } -PyTypeObject *mapFlagsToSameEnum(PyTypeObject *FType, PyTypeObject *EType) +PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint8_t enumValues[]) { - // this will be switchable... - return useOldEnum ? FType : EType; + return createPythonEnumHelper(module, fullName, enumItemStrings, enumValues); } -} // extern "C" +} // namespace Shiboken::Enum diff --git a/sources/shiboken6/libshiboken/sbkenum.h b/sources/shiboken6/libshiboken/sbkenum.h index 8b266710e..e19ca4b4c 100644 --- a/sources/shiboken6/libshiboken/sbkenum.h +++ b/sources/shiboken6/libshiboken/sbkenum.h @@ -10,77 +10,93 @@ extern "C" { +LIBSHIBOKEN_API bool PyEnumMeta_Check(PyObject *ob); + /// exposed for the signature module LIBSHIBOKEN_API void init_enum(); -extern LIBSHIBOKEN_API PyTypeObject *SbkEnumType_TypeF(void); struct SbkConverter; struct SbkEnumType; -struct SbkEnumTypePrivate; - -} // extern "C" -namespace Shiboken +struct SbkEnumTypePrivate { + SbkConverter *converter; +}; + +/// PYSIDE-1735: Pass on the Python enum/flag information. +LIBSHIBOKEN_API void initEnumFlagsDict(PyTypeObject *type); + +/// PYSIDE-1735: Make sure that we can import the Python enum implementation. +LIBSHIBOKEN_API PyTypeObject *getPyEnumMeta(); +/// PYSIDE-1735: Helper function supporting QEnum +LIBSHIBOKEN_API int enumIsFlag(PyObject *ob_enum); -inline bool isShibokenEnum(PyObject *pyObj) -{ - return Py_TYPE(Py_TYPE(pyObj)) == SbkEnumType_TypeF(); } -namespace Enum +namespace Shiboken::Enum { + +enum : int { + ENOPT_OLD_ENUM = 0x00, // PySide 6.6: no longer supported + ENOPT_NEW_ENUM = 0x01, + ENOPT_INHERIT_INT = 0x02, + ENOPT_GLOBAL_SHORTCUT = 0x04, + ENOPT_SCOPED_SHORTCUT = 0x08, + ENOPT_NO_FAKESHORTCUT = 0x10, + ENOPT_NO_FAKERENAMES = 0x20, + ENOPT_NO_ZERODEFAULT = 0x40, + ENOPT_NO_MISSING = 0x80, +}; + +LIBSHIBOKEN_API extern int enumOption; + +using EnumValueType = long long; + +LIBSHIBOKEN_API bool check(PyObject *obj); + +LIBSHIBOKEN_API PyObject *newItem(PyTypeObject *enumType, EnumValueType itemValue, + const char *itemName = nullptr); + +LIBSHIBOKEN_API EnumValueType getValue(PyObject *enumItem); +LIBSHIBOKEN_API PyObject *getEnumItemFromValue(PyTypeObject *enumType, + EnumValueType itemValue); + +/// Sets the enum/flag's type converter. +LIBSHIBOKEN_API void setTypeConverter(PyTypeObject *type, SbkConverter *converter); + +/// Creating Python enums for different types. +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int64_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint64_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int32_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint32_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int16_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint16_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const int8_t enumValues[]); + +LIBSHIBOKEN_API PyTypeObject *createPythonEnum(PyObject *module, + const char *fullName, const char *enumItemStrings[], const uint8_t enumValues[]); + +/// This template removes duplication by inlining necessary type casts. +template <typename IntT> +inline PyTypeObject *createPythonEnum(PyTypeObject *scope, + const char *fullName, const char *enumItemStrings[], const IntT enumValues[]) { - LIBSHIBOKEN_API bool check(PyObject *obj); - /** - * Creates a new enum type (and its flags type, if any is given) - * and registers it to Python and adds it to \p module. - * \param module Module to where the new enum type will be added. - * \param name Name of the enum. - * \param fullName Name of the enum that includes all scope information (e.g.: "module.Enum"). - * \param cppName Full qualified C++ name of the enum. - * \param flagsType Optional Python type for the flags associated with the enum. - * \return The new enum type or NULL if it fails. - */ - LIBSHIBOKEN_API PyTypeObject *createGlobalEnum(PyObject *module, - const char *name, - const char *fullName, - const char *cppName, - PyTypeObject *flagsType = nullptr); - /// This function does the same as createGlobalEnum, but adds the enum to a Shiboken type or namespace. - LIBSHIBOKEN_API PyTypeObject *createScopedEnum(PyTypeObject *scope, - const char *name, - const char *fullName, - const char *cppName, - PyTypeObject *flagsType = nullptr); - - /** - * Creates a new enum item for a given enum type and adds it to \p module. - * \param enumType Enum type to where the new enum item will be added. - * \param module Module to where the enum type of the new enum item belongs. - * \param itemName Name of the enum item. - * \param itemValue Numerical value of the enum item. - * \return true if everything goes fine, false if it fails. - */ - LIBSHIBOKEN_API bool createGlobalEnumItem(PyTypeObject *enumType, PyObject *module, const char *itemName, long itemValue); - /// This function does the same as createGlobalEnumItem, but adds the enum to a Shiboken type or namespace. - LIBSHIBOKEN_API bool createScopedEnumItem(PyTypeObject *enumType, PyTypeObject *scope, - const char *itemName, long itemValue); - - LIBSHIBOKEN_API PyObject *newItem(PyTypeObject *enumType, long itemValue, const char *itemName = nullptr); - - LIBSHIBOKEN_API PyTypeObject *newTypeWithName(const char *name, const char *cppName, - PyTypeObject *numbers_fromFlag=nullptr); - LIBSHIBOKEN_API const char *getCppName(PyTypeObject *type); - - LIBSHIBOKEN_API long getValue(PyObject *enumItem); - LIBSHIBOKEN_API PyObject *getEnumItemFromValue(PyTypeObject *enumType, long itemValue); - - /// Sets the enum/flag's type converter. - LIBSHIBOKEN_API void setTypeConverter(PyTypeObject *type, SbkConverter *converter, bool isFlag); - - LIBSHIBOKEN_API PyObject *unpickleEnum(PyObject *, PyObject *); + auto *obScope = reinterpret_cast<PyObject *>(scope); + return createPythonEnum(obScope, fullName, enumItemStrings, enumValues); } -} // namespace Shiboken +} // namespace Shiboken::Enum #endif // SKB_PYENUM_H diff --git a/sources/shiboken6/libshiboken/sbkenum_p.h b/sources/shiboken6/libshiboken/sbkenum_p.h deleted file mode 100644 index 9e6fd6a61..000000000 --- a/sources/shiboken6/libshiboken/sbkenum_p.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (C) 2021 The Qt Company Ltd. -// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only - -#ifndef SBKENUM_P_H -#define SBKENUM_P_H - -#include "sbkpython.h" -#include "shibokenmacros.h" - -struct SbkEnumTypePrivate -{ - SbkConverter *converter; - const char *cppName; - PyTypeObject *replacementType; -}; - -extern "C" { - -/// PYSIDE-1735: Patching the Enum / Flags implementation. Remove in 6.4 -LIBSHIBOKEN_API PyTypeObject *morphLastEnumToPython(); -LIBSHIBOKEN_API PyTypeObject *mapFlagsToSameEnum(PyTypeObject *FType, PyTypeObject *EType); - -/// PYSIDE-1735: Make sure that we can import the Python enum implementation. -LIBSHIBOKEN_API PyTypeObject *getPyEnumMeta(); -/// PYSIDE-1735: Helper function supporting QEnum -LIBSHIBOKEN_API int enumIsFlag(PyObject *ob_enum); - -} - -#endif // SBKENUM_P_H diff --git a/sources/shiboken6/libshiboken/sbkerrors.cpp b/sources/shiboken6/libshiboken/sbkerrors.cpp index 38da36b16..1832624d5 100644 --- a/sources/shiboken6/libshiboken/sbkerrors.cpp +++ b/sources/shiboken6/libshiboken/sbkerrors.cpp @@ -2,10 +2,37 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "sbkerrors.h" +#include "sbkstring.h" #include "helper.h" +#include "gilstate.h" namespace Shiboken { + +// PYSIDE-2335: Track down if we can reach a Python error handler. +// _pythonContextStack has always the current state of handler status +// in its lowest bit. +// Blocking calls like exec or run need to use `setBlocking`. +static thread_local std::size_t _pythonContextStack{}; + +PythonContextMarker::PythonContextMarker() +{ + // Shift history up and set lowest bit. + _pythonContextStack = (_pythonContextStack * 2) + 1; +} + +PythonContextMarker::~PythonContextMarker() +{ + // Shift history down. + _pythonContextStack /= 2; +} + +void PythonContextMarker::setBlocking() +{ + // Clear lowest bit. + _pythonContextStack = _pythonContextStack / 2 * 2; +} + namespace Errors { @@ -66,6 +93,34 @@ void setWrongContainerType() PyErr_SetString(PyExc_TypeError, "Wrong type passed to container conversion."); } +struct ErrorStore { + PyObject *type; + PyObject *exc; + PyObject *traceback; +}; + +static thread_local ErrorStore savedError{}; + +void storeErrorOrPrint() +{ + // This error happened in a function with no way to return an error state. + // Therefore, we handle the error when we are error checking, anyway. + // But we do that only when we know that an error handler can pick it up. + if (_pythonContextStack & 1) + PyErr_Fetch(&savedError.type, &savedError.exc, &savedError.traceback); + else + PyErr_Print(); +} + +PyObject *occurred() +{ + if (savedError.type) { + PyErr_Restore(savedError.type, savedError.exc, savedError.traceback); + savedError.type = nullptr; + } + return PyErr_Occurred(); +} + } // namespace Errors namespace Warnings @@ -94,5 +149,22 @@ void warnDeprecated(const char *className, const char *functionName) className, functionName); } +void warnDeprecatedEnum(const char *enumName) +{ + Shiboken::warning(PyExc_DeprecationWarning, 1, + "Enum: '%s' is marked as deprecated, please check " + "the documentation for more information.", + enumName); +} + +void warnDeprecatedEnumValue(const char *enumName, const char *valueName) +{ + Shiboken::warning(PyExc_DeprecationWarning, 1, + "Enum value '%s.%s' is marked as deprecated, please check " + "the documentation for more information.", + enumName, valueName); + +} + } // namespace Warnings } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/sbkerrors.h b/sources/shiboken6/libshiboken/sbkerrors.h index cc86c10b9..6ff85f8e1 100644 --- a/sources/shiboken6/libshiboken/sbkerrors.h +++ b/sources/shiboken6/libshiboken/sbkerrors.h @@ -7,8 +7,31 @@ #include "sbkpython.h" #include "shibokenmacros.h" +/// Craving for C++20 and std::source_location::current() +#if defined(_MSC_VER) +# define SBK_FUNC_INFO __FUNCSIG__ +#elif defined(__GNUC__) +# define SBK_FUNC_INFO __PRETTY_FUNCTION__ +#else +# define SBK_FUNC_INFO __FUNCTION__ +#endif + namespace Shiboken { + +struct LIBSHIBOKEN_API PythonContextMarker +{ +public: + PythonContextMarker(const PythonContextMarker &) = delete; + PythonContextMarker(PythonContextMarker &&) = delete; + PythonContextMarker &operator=(const PythonContextMarker &) = delete; + PythonContextMarker &operator=(PythonContextMarker &&) = delete; + + explicit PythonContextMarker(); + ~PythonContextMarker(); + void setBlocking(); +}; + namespace Errors { @@ -23,6 +46,15 @@ LIBSHIBOKEN_API void setSequenceTypeError(const char *expectedType); LIBSHIBOKEN_API void setSetterTypeError(const char *name, const char *expectedType); LIBSHIBOKEN_API void setWrongContainerType(); +/// Report an error ASAP: Instead of printing, store for later re-raise. +/// This replaces `PyErr_Print`, which cannot report errors as exception. +/// To be used in contexts where raising errors is impossible. +LIBSHIBOKEN_API void storeErrorOrPrint(); +/// Handle an error as in PyErr_Occurred(), but also check for errors which +/// were captured by `storeErrorOrPrint`. +/// To be used in normal error checks. +LIBSHIBOKEN_API PyObject *occurred(); + } // namespace Errors namespace Warnings @@ -32,6 +64,8 @@ LIBSHIBOKEN_API void warnInvalidReturnValue(const char *className, const char *f const char *expectedType, const char *actualType); LIBSHIBOKEN_API void warnDeprecated(const char *functionName); LIBSHIBOKEN_API void warnDeprecated(const char *className, const char *functionName); +LIBSHIBOKEN_API void warnDeprecatedEnum(const char *enumName); +LIBSHIBOKEN_API void warnDeprecatedEnumValue(const char *enumName, const char *valueName); } // namespace Warnings } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/sbkfeature_base.cpp b/sources/shiboken6/libshiboken/sbkfeature_base.cpp index e191204e4..f31b8f4f7 100644 --- a/sources/shiboken6/libshiboken/sbkfeature_base.cpp +++ b/sources/shiboken6/libshiboken/sbkfeature_base.cpp @@ -4,12 +4,16 @@ #include "basewrapper.h" #include "basewrapper_p.h" #include "autodecref.h" -#include "sbkenum_p.h" +#include "pep384ext.h" +#include "sbkenum.h" #include "sbkstring.h" #include "sbkstaticstrings.h" #include "sbkstaticstrings_p.h" #include "signature.h" #include "sbkfeature_base.h" +#include "gilstate.h" + +#include <cctype> using namespace Shiboken; @@ -18,64 +22,293 @@ extern "C" //////////////////////////////////////////////////////////////////////////// // -// getFeatureSelectId +// Minimal __feature__ support in Shiboken // -// This function is needed here already for signature handling. -// Maybe the same function from feature_select.cpp will be replaced. +int currentSelectId(PyTypeObject *type) +{ + AutoDecRef tpDict(PepType_GetDict(type)); + PyObject *PyId = PyObject_GetAttr(tpDict.object(), PyName::select_id()); + if (PyId == nullptr) { + PyErr_Clear(); + return 0x00; + } + int sel = PyLong_AsLong(PyId); + Py_DECREF(PyId); + return sel; +} + +static SelectableFeatureHook SelectFeatureSet = nullptr; +static SelectableFeatureCallback featureCb = nullptr; + +void setSelectableFeatureCallback(SelectableFeatureCallback func) +{ + featureCb = func; +} + +SelectableFeatureHook initSelectableFeature(SelectableFeatureHook func) +{ + auto ret = SelectFeatureSet; + SelectFeatureSet = func; + if (featureCb) + featureCb(SelectFeatureSet != nullptr); + return ret; +} // +//////////////////////////////////////////////////////////////////////////// -static PyObject *cached_globals{}; -static PyObject *last_select_id{}; +// This useful function is for debugging +void disassembleFrame(const char *marker) +{ + Shiboken::GilState gil; + PyObject *error_type, *error_value, *error_traceback; + PyErr_Fetch(&error_type, &error_value, &error_traceback); + static PyObject *dismodule = PyImport_ImportModule("dis"); + static PyObject *disco = PyObject_GetAttrString(dismodule, "disco"); + static PyObject *const _f_lasti = Shiboken::String::createStaticString("f_lasti"); + static PyObject *const _f_lineno = Shiboken::String::createStaticString("f_lineno"); + static PyObject *const _f_code = Shiboken::String::createStaticString("f_code"); + static PyObject *const _co_filename = Shiboken::String::createStaticString("co_filename"); + AutoDecRef ignore{}; + auto *frame = reinterpret_cast<PyObject *>(PyEval_GetFrame()); + if (frame == nullptr) { + fprintf(stdout, "\n%s BEGIN no frame END\n\n", marker); + } else { + AutoDecRef f_lasti(PyObject_GetAttr(frame, _f_lasti)); + AutoDecRef f_lineno(PyObject_GetAttr(frame, _f_lineno)); + AutoDecRef f_code(PyObject_GetAttr(frame, _f_code)); + AutoDecRef co_filename(PyObject_GetAttr(f_code, _co_filename)); + long line = PyLong_AsLong(f_lineno); + const char *fname = String::toCString(co_filename); + fprintf(stdout, "\n%s BEGIN line=%ld %s\n", marker, line, fname); + ignore.reset(PyObject_CallFunctionObjArgs(disco, f_code.object(), f_lasti.object(), nullptr)); + fprintf(stdout, "%s END line=%ld %s\n\n", marker, line, fname); + } +#if PY_VERSION_HEX >= 0x030C0000 && !Py_LIMITED_API + if (error_type) + PyErr_DisplayException(error_value); +#endif + static PyObject *stdout_file = PySys_GetObject("stdout"); + ignore.reset(PyObject_CallMethod(stdout_file, "flush", nullptr)); + PyErr_Restore(error_type, error_value, error_traceback); +} + +// python 3.12 +static int const CALL = 171; +// Python 3.11 +static int const PRECALL = 166; +// we have "big instructions" with gaps after them +static int const LOAD_METHOD_GAP_311 = 10 * 2; +static int const LOAD_ATTR_GAP_311 = 4 * 2; +static int const LOAD_ATTR_GAP = 9 * 2; +// Python 3.7 - 3.10 +static int const LOAD_METHOD = 160; +static int const CALL_METHOD = 161; +// Python 3.6 +static int const CALL_FUNCTION = 131; +static int const LOAD_ATTR = 106; +// NoGil (how long will this exist in this form?) +static int const LOAD_METHOD_NOGIL = 55; +static int const CALL_METHOD_NOGIL = 72; -PyObject *getFeatureSelectId() +static bool currentOpcode_Is_CallMethNoArgs() { - static PyObject *undef = PyLong_FromLong(-1); - static PyObject *feature_dict = GetFeatureDict(); - // these things are all borrowed - PyObject *globals = PyEval_GetGlobals(); - if (globals == nullptr - || globals == cached_globals) - return last_select_id; + // PYSIDE-2221: Special case for the NoGil version: + // Find out if we have such a version. + // We could also ask the variable `Py_NOGIL`. + static PyObject *flags = PySys_GetObject("flags"); + static bool isNoGil = PyObject_HasAttrString(flags, "nogil"); + // We look into the currently active operation if we are going to call + // a method with zero arguments. + auto *frame = PyEval_GetFrame(); +#if !Py_LIMITED_API && !defined(PYPY_VERSION) + auto *f_code = PyFrame_GetCode(frame); +#else + static PyObject *const _f_code = Shiboken::String::createStaticString("f_code"); + AutoDecRef dec_f_code(PyObject_GetAttr(reinterpret_cast<PyObject *>(frame), _f_code)); + auto *f_code = dec_f_code.object(); +#endif +#if PY_VERSION_HEX >= 0x030B0000 && !Py_LIMITED_API + AutoDecRef dec_co_code(PyCode_GetCode(f_code)); + Py_ssize_t f_lasti = PyFrame_GetLasti(frame); +#else + static PyObject *const _f_lasti = Shiboken::String::createStaticString("f_lasti"); + static PyObject *const _co_code = Shiboken::String::createStaticString("co_code"); + AutoDecRef dec_co_code(PyObject_GetAttr(reinterpret_cast<PyObject *>(f_code), _co_code)); + AutoDecRef dec_f_lasti(PyObject_GetAttr(reinterpret_cast<PyObject *>(frame), _f_lasti)); + Py_ssize_t f_lasti = PyLong_AsSsize_t(dec_f_lasti); +#endif + Py_ssize_t code_len; + char *co_code{}; + PyBytes_AsStringAndSize(dec_co_code, &co_code, &code_len); + uint8_t opcode1 = co_code[f_lasti]; + if (isNoGil) { + uint8_t opcode2 = co_code[f_lasti + 4]; + uint8_t oparg2 = co_code[f_lasti + 6]; + return opcode1 == LOAD_METHOD_NOGIL && opcode2 == CALL_METHOD_NOGIL && oparg2 == 1; + } + uint8_t opcode2 = co_code[f_lasti + 2]; + uint8_t oparg2 = co_code[f_lasti + 3]; + static auto number = _PepRuntimeVersion(); + if (number < 0x030B00) + return opcode1 == LOAD_METHOD && opcode2 == CALL_METHOD && oparg2 == 0; + + if (number < 0x030C00) { + // With Python 3.11, the opcodes get bigger and change a bit. + // Note: The new adaptive opcodes are elegantly hidden and we + // don't need to take care of them. + if (opcode1 == LOAD_METHOD) + f_lasti += LOAD_METHOD_GAP_311; + else if (opcode1 == LOAD_ATTR) + f_lasti += LOAD_ATTR_GAP_311; + else + return false; + + opcode2 = co_code[f_lasti + 2]; + oparg2 = co_code[f_lasti + 3]; - PyObject *modname = PyDict_GetItem(globals, PyMagicName::name()); - if (modname == nullptr) - return last_select_id; + return opcode2 == PRECALL && oparg2 == 0; + } + // With Python 3.12, the opcodes get again bigger and change a bit. + // Note: The new adaptive opcodes are elegantly hidden and we + // don't need to take care of them. + if (opcode1 == LOAD_ATTR) + f_lasti += LOAD_ATTR_GAP; + else + return false; - PyObject *select_id = PyDict_GetItem(feature_dict, modname); - if (select_id == nullptr - || !PyLong_Check(select_id) // int/long cheating - || select_id == undef) - return last_select_id; + opcode2 = co_code[f_lasti + 2]; + oparg2 = co_code[f_lasti + 3]; - cached_globals = globals; - last_select_id = select_id; - assert(PyLong_AsSsize_t(select_id) >= 0); - return select_id; + return opcode2 == CALL && oparg2 == 0; } -int currentSelectId(PyTypeObject *type) +void initEnumFlagsDict(PyTypeObject *type) { - int sel = SbkObjectType_GetReserved(type); - // This could theoretically be -1 if used too early. - assert(sel >= 0); - return sel; + // We create a dict for all flag enums that holds the original C++ name + // and a dict that gives every enum/flag type name. + static PyObject *const split = Shiboken::String::createStaticString("split"); + static PyObject *const colon = Shiboken::String::createStaticString(":"); + auto sotp = PepType_SOTP(type); + auto **enumFlagInfo = sotp->enumFlagInfo; + auto *dict = PyDict_New(); + auto *typeDict = PyDict_New(); + for (; *enumFlagInfo; ++enumFlagInfo) { + AutoDecRef line(PyUnicode_FromString(*enumFlagInfo)); + AutoDecRef parts(PyObject_CallMethodObjArgs(line, split, colon, nullptr)); + auto *name = PyList_GetItem(parts, 0); + if (PyList_Size(parts) == 3) { + auto *key = PyList_GetItem(parts, 2); + auto *value = name; + PyDict_SetItem(dict, key, value); + } + auto *typeName = PyList_GetItem(parts, 1); + PyDict_SetItem(typeDict, name, typeName); + } + sotp->enumFlagsDict = dict; + sotp->enumTypeDict = typeDict; } -void initFeatureShibokenPart() +static PyObject *replaceNoArgWithZero(PyObject *callable) { - static PyObject *no_sel = PyLong_FromLong(0); - last_select_id = no_sel; - // Reset the cache. This is called at any "from __feature__ import". - cached_globals = nullptr; + static auto *partial = Pep_GetPartialFunction(); + static auto *zero = PyLong_FromLong(0); + return PyObject_CallFunctionObjArgs(partial, callable, zero, nullptr); } -static SelectableFeatureHook SelectFeatureSet = nullptr; - -SelectableFeatureHook initSelectableFeature(SelectableFeatureHook func) +static PyObject *lookupUnqualifiedOrOldEnum(PyTypeObject *type, PyObject *name) { - auto ret = SelectFeatureSet; - SelectFeatureSet = func; - return ret; + // MRO has been observed to be 0 in case of errors with QML decorators + if (type == nullptr || type->tp_mro == nullptr) + return nullptr; + // Quick Check: Avoid "__..", "_slots", etc. + if (std::isalpha(Shiboken::String::toCString(name)[0]) == 0) + return nullptr; + static PyTypeObject *const EnumMeta = getPyEnumMeta(); + static PyObject *const _member_map_ = String::createStaticString("_member_map_"); + // This is similar to `find_name_in_mro`, but instead of looking directly into + // tp_dict, we also search for the attribute in local classes of that dict (Part 2). + PyObject *mro = type->tp_mro; + PyObject *result{}; + assert(PyTuple_Check(mro)); + Py_ssize_t idx, n = PyTuple_GET_SIZE(mro); + for (idx = 0; idx < n; ++idx) { + auto *base = PyTuple_GET_ITEM(mro, idx); + auto *type_base = reinterpret_cast<PyTypeObject *>(base); + if (!SbkObjectType_Check(type_base)) + continue; + auto sotp = PepType_SOTP(type_base); + // The EnumFlagInfo structure tells us if there are Enums at all. + const char **enumFlagInfo = sotp->enumFlagInfo; + if (!(enumFlagInfo)) + continue; + if (!sotp->enumFlagsDict) + initEnumFlagsDict(type_base); + bool useFakeRenames = !(Enum::enumOption & Enum::ENOPT_NO_FAKERENAMES); + if (useFakeRenames) { + auto *rename = PyDict_GetItem(sotp->enumFlagsDict, name); + if (rename) { + /* + * Part 1: Look into the enumFlagsDict if we have an old flags name. + * ------------------------------------------------------------- + * We need to replace the parameterless + + QtCore.Qt.Alignment() + + * by the one-parameter call + + QtCore.Qt.AlignmentFlag(0) + + * That means: We need to bind the zero as default into a wrapper and + * return that to be called. + * + * Addendum: + * --------- + * We first need to look into the current opcode of the bytecode to find + * out if we have a call like above or just a type lookup. + */ + AutoDecRef tpDict(PepType_GetDict(type_base)); + auto *flagType = PyDict_GetItem(tpDict.object(), rename); + if (currentOpcode_Is_CallMethNoArgs()) + return replaceNoArgWithZero(flagType); + Py_INCREF(flagType); + return flagType; + } + } + bool useFakeShortcuts = !(Enum::enumOption & Enum::ENOPT_NO_FAKESHORTCUT); + if (useFakeShortcuts) { + AutoDecRef tpDict(PepType_GetDict(type_base)); + auto *dict = tpDict.object(); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(dict, &pos, &key, &value)) { + /* + * Part 2: Check for a duplication into outer scope. + * ------------------------------------------------- + * We need to replace the shortcut + + QtCore.Qt.AlignLeft + + * by the correct call + + QtCore.Qt.AlignmentFlag.AlignLeft + + * That means: We need to search all Enums of the class. + */ + if (Py_TYPE(value) == EnumMeta) { + auto *valtype = reinterpret_cast<PyTypeObject *>(value); + AutoDecRef valtypeDict(PepType_GetDict(valtype)); + auto *member_map = PyDict_GetItem(valtypeDict.object(), _member_map_); + if (member_map && PyDict_Check(member_map)) { + result = PyDict_GetItem(member_map, name); + Py_XINCREF(result); + if (result) + return result; + } + } + } + } + } + return nullptr; } PyObject *mangled_type_getattro(PyTypeObject *type, PyObject *name) @@ -86,67 +319,64 @@ PyObject *mangled_type_getattro(PyTypeObject *type, PyObject *name) * with the complex `tp_getattro` of `QObject` and other instances. * What we change here is the meta class of `QObject`. */ - static getattrofunc type_getattro = PyType_Type.tp_getattro; - static PyObject *ignAttr1 = PyName::qtStaticMetaObject(); - static PyObject *ignAttr2 = PyMagicName::get(); + static getattrofunc const type_getattro = PepExt_Type_GetGetAttroSlot(&PyType_Type); + static PyObject *const ignAttr1 = PyName::qtStaticMetaObject(); + static PyObject *const ignAttr2 = PyMagicName::get(); + static PyTypeObject *const EnumMeta = getPyEnumMeta(); + if (SelectFeatureSet != nullptr) - type->tp_dict = SelectFeatureSet(type); + SelectFeatureSet(type); auto *ret = type_getattro(reinterpret_cast<PyObject *>(type), name); // PYSIDE-1735: Be forgiving with strict enums and fetch the enum, silently. // The PYI files now look correct, but the old duplication is // emulated here. This should be removed in Qt 7, see `parser.py`. // - // FIXME PYSIDE7 should remove this forgivingness: + // FIXME PYSIDE7 should remove this forgiveness: // // The duplication of enum values into the enclosing scope, allowing to write // Qt.AlignLeft instead of Qt.Alignment.AlignLeft, is still implemented but // no longer advertized in PYI files or line completion. + if (ret && Py_TYPE(ret) == EnumMeta && currentOpcode_Is_CallMethNoArgs()) { + bool useZeroDefault = !(Enum::enumOption & Enum::ENOPT_NO_ZERODEFAULT); + if (useZeroDefault) { + // We provide a zero argument for compatibility if it is a call with no args. + auto *hold = replaceNoArgWithZero(ret); + Py_DECREF(ret); + ret = hold; + } + } + if (!ret && name != ignAttr1 && name != ignAttr2) { - PyObject *error_type, *error_value, *error_traceback; + PyObject *error_type{}, *error_value{}, *error_traceback{}; PyErr_Fetch(&error_type, &error_value, &error_traceback); - - // This is similar to `find_name_in_mro`, but instead of looking directly into - // tp_dict, we search for the attribute in local classes of that dict. - PyObject *mro = type->tp_mro; - assert(PyTuple_Check(mro)); - size_t idx, n = PyTuple_GET_SIZE(mro); - for (idx = 0; idx < n; ++idx) { - // FIXME This loop should further be optimized by installing an extra - // <classname>_EnumInfo structure. This comes with the next compatibility patch. - auto *base = PyTuple_GET_ITEM(mro, idx); - auto *type_base = reinterpret_cast<PyTypeObject *>(base); - auto *dict = type_base->tp_dict; - PyObject *key, *value; - Py_ssize_t pos = 0; - while (PyDict_Next(dict, &pos, &key, &value)) { - static auto *EnumMeta = getPyEnumMeta(); - if (Py_TYPE(value) == EnumMeta) { - auto *valtype = reinterpret_cast<PyTypeObject *>(value); - auto *result = PyDict_GetItem(valtype->tp_dict, name); - if (result) { - Py_INCREF(result); - return result; - } - } - } + ret = lookupUnqualifiedOrOldEnum(type, name); + if (ret) { + Py_DECREF(error_type); + Py_XDECREF(error_value); + Py_XDECREF(error_traceback); + } else { + PyErr_Restore(error_type, error_value, error_traceback); } - PyErr_Restore(error_type, error_value, error_traceback); } return ret; } -PyObject *Sbk_TypeGet___dict__(PyTypeObject *type, void *context) +PyObject *Sbk_TypeGet___dict__(PyTypeObject *type, void * /* context */) { /* * This is the override for getting a dict. */ - auto dict = type->tp_dict; + AutoDecRef tpDict(PepType_GetDict(type)); + auto *dict = tpDict.object();; if (dict == nullptr) Py_RETURN_NONE; - if (SelectFeatureSet != nullptr) - dict = SelectFeatureSet(type); + if (SelectFeatureSet != nullptr) { + SelectFeatureSet(type); + tpDict.reset(PepType_GetDict(type)); + dict = tpDict.object(); + } return PyDictProxy_New(dict); } @@ -157,7 +387,7 @@ PyObject *SbkObject_GenericGetAttr(PyObject *obj, PyObject *name) { auto type = Py_TYPE(obj); if (SelectFeatureSet != nullptr) - type->tp_dict = SelectFeatureSet(type); + SelectFeatureSet(type); return PyObject_GenericGetAttr(obj, name); } @@ -165,21 +395,10 @@ int SbkObject_GenericSetAttr(PyObject *obj, PyObject *name, PyObject *value) { auto type = Py_TYPE(obj); if (SelectFeatureSet != nullptr) - type->tp_dict = SelectFeatureSet(type); + SelectFeatureSet(type); return PyObject_GenericSetAttr(obj, name, value); } -// Caching the select Id. -int SbkObjectType_GetReserved(PyTypeObject *type) -{ - return PepType_SOTP(type)->pyside_reserved_bits; -} - -void SbkObjectType_SetReserved(PyTypeObject *type, int value) -{ - PepType_SOTP(type)->pyside_reserved_bits = value; -} - const char **SbkObjectType_GetPropertyStrings(PyTypeObject *type) { return PepType_SOTP(type)->propertyStrings; @@ -190,11 +409,16 @@ void SbkObjectType_SetPropertyStrings(PyTypeObject *type, const char **strings) PepType_SOTP(type)->propertyStrings = strings; } +void SbkObjectType_SetEnumFlagInfo(PyTypeObject *type, const char **strings) +{ + PepType_SOTP(type)->enumFlagInfo = strings; +} + // PYSIDE-1626: Enforcing a context switch without further action. void SbkObjectType_UpdateFeature(PyTypeObject *type) { if (SelectFeatureSet != nullptr) - type->tp_dict = SelectFeatureSet(type); + SelectFeatureSet(type); } } // extern "C" diff --git a/sources/shiboken6/libshiboken/sbkfeature_base.h b/sources/shiboken6/libshiboken/sbkfeature_base.h index 8d55297d7..290884062 100644 --- a/sources/shiboken6/libshiboken/sbkfeature_base.h +++ b/sources/shiboken6/libshiboken/sbkfeature_base.h @@ -7,9 +7,7 @@ extern "C" { -LIBSHIBOKEN_API PyObject *getFeatureSelectId(); LIBSHIBOKEN_API int currentSelectId(PyTypeObject *type); -LIBSHIBOKEN_API void initFeatureShibokenPart(); LIBSHIBOKEN_API PyObject *mangled_type_getattro(PyTypeObject *type, PyObject *name); LIBSHIBOKEN_API PyObject *Sbk_TypeGet___dict__(PyTypeObject *type, void *context); LIBSHIBOKEN_API PyObject *SbkObject_GenericGetAttr(PyObject *obj, PyObject *name); diff --git a/sources/shiboken6/libshiboken/sbkmodule.cpp b/sources/shiboken6/libshiboken/sbkmodule.cpp index 51ab4311b..4153df27f 100644 --- a/sources/shiboken6/libshiboken/sbkmodule.cpp +++ b/sources/shiboken6/libshiboken/sbkmodule.cpp @@ -2,54 +2,470 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "sbkmodule.h" +#include "autodecref.h" #include "basewrapper.h" #include "bindingmanager.h" +#include "sbkstring.h" +#include "sbkcppstring.h" + #include <unordered_map> +#include <unordered_set> +#include <cstring> /// This hash maps module objects to arrays of Python types. -using ModuleTypesMap = std::unordered_map<PyObject *, PyTypeObject **> ; +using ModuleTypesMap = std::unordered_map<PyObject *, Shiboken::Module::TypeInitStruct *> ; /// This hash maps module objects to arrays of converters. using ModuleConvertersMap = std::unordered_map<PyObject *, SbkConverter **>; +/// This hash maps type names to type creation functions. +using TypeCreationFunctionModulePair = + std::pair<Shiboken::Module::TypeCreationFunction, PyObject *>; +using NameToTypeFunctionMap = std::unordered_map<std::string, TypeCreationFunctionModulePair>; + +/// This hash maps module objects to maps of names to functions. +using ModuleToFuncsMap = std::unordered_map<PyObject *, NameToTypeFunctionMap> ; + /// All types produced in imported modules are mapped here. static ModuleTypesMap moduleTypes; static ModuleConvertersMap moduleConverters; +static ModuleToFuncsMap moduleToFuncs; namespace Shiboken { namespace Module { +// PYSIDE-2404: Replacing the arguments generated by cpythonTypeNameExt +// by a function call. +LIBSHIBOKEN_API PyTypeObject *get(TypeInitStruct &typeStruct) +{ + if (typeStruct.type != nullptr) + return typeStruct.type; + + static PyObject *sysModules = PyImport_GetModuleDict(); + + // The slow path for initialization. + // We get the type by following the chain from the module. + // As soon as types[index] gets filled, we can stop. + + std::string_view names(typeStruct.fullName); + const bool usePySide = names.compare(0, 8, "PySide6.") == 0; + auto dotPos = usePySide ? names.find('.', 8) : names.find('.'); + auto startPos = dotPos + 1; + AutoDecRef modName(String::fromCppStringView(names.substr(0, dotPos))); + auto *modOrType = PyDict_GetItem(sysModules, modName); + if (modOrType == nullptr) { + PyErr_Format(PyExc_SystemError, "Module %s should already be in sys.modules", + PyModule_GetName(modOrType)); + return nullptr; + } + + do { + dotPos = names.find('.', startPos); + auto typeName = dotPos != std::string::npos + ? names.substr(startPos, dotPos - startPos) + : names.substr(startPos); + startPos = dotPos + 1; + AutoDecRef obTypeName(String::fromCppStringView(typeName)); + modOrType = PyObject_GetAttr(modOrType, obTypeName); + } while (typeStruct.type == nullptr && dotPos != std::string::npos); + + return typeStruct.type; +} + +static PyTypeObject *incarnateType(PyObject *module, const char *name, + NameToTypeFunctionMap &nameToFunc) +{ + // - locate the name and retrieve the generating function + auto funcIter = nameToFunc.find(name); + if (funcIter == nameToFunc.end()) { + // attribute does really not exist. + PyErr_SetNone(PyExc_AttributeError); + return nullptr; + } + // - call this function that returns a PyTypeObject + auto pair = funcIter->second; + auto initFunc = pair.first; + auto *modOrType = pair.second; + + // PYSIDE-2404: Make sure that no switching happens during type creation. + auto saveFeature = initSelectableFeature(nullptr); + PyTypeObject *type = initFunc(modOrType); + initSelectableFeature(saveFeature); + + // - assign this object to the name in the module + auto *res = reinterpret_cast<PyObject *>(type); + Py_INCREF(res); + PyModule_AddObject(module, name, res); // steals reference + // - remove the entry, if not by something cleared. + if (!nameToFunc.empty()) + nameToFunc.erase(funcIter); + // - return the PyTypeObject. + return type; +} + +// PYSIDE-2404: Make sure that the mentioned classes really exist. +// Used in `Pyside::typeName`. Because the result will be cached by +// the creation of the type(s), this is efficient. +void loadLazyClassesWithName(const char *name) +{ + for (auto const & tableIter : moduleToFuncs) { + auto nameToFunc = tableIter.second; + auto funcIter = nameToFunc.find(name); + if (funcIter != nameToFunc.end()) { + // attribute exists in the lazy types. + auto *module = tableIter.first; + incarnateType(module, name, nameToFunc); + } + } +} + +// PYSIDE-2404: Completely load all not yet loaded classes. +// This is needed to resolve a star import. +void resolveLazyClasses(PyObject *module) +{ + // - locate the module in the moduleTofuncs mapping + auto tableIter = moduleToFuncs.find(module); + if (tableIter == moduleToFuncs.end()) + return; + + // - see if there are still unloaded elements + auto &nameToFunc = tableIter->second; + + // - incarnate all types. + while (!nameToFunc.empty()) { + auto it = nameToFunc.begin(); + auto attrNameStr = it->first; + incarnateType(module, attrNameStr.c_str(), nameToFunc); + } +} + +// PYSIDE-2404: Override the gettattr function of modules. +static getattrofunc origModuleGetattro{}; + +// PYSIDE-2404: Use the patched module getattr to do on-demand initialization. +// This modifies _all_ modules but should have no impact. +static PyObject *PyModule_lazyGetAttro(PyObject *module, PyObject *name) +{ + // - check if the attribute is present and return it. + auto *attr = PyObject_GenericGetAttr(module, name); + // - we handle AttributeError, only. + if (!(attr == nullptr && PyErr_ExceptionMatches(PyExc_AttributeError))) + return attr; + + PyErr_Clear(); + // - locate the module in the moduleTofuncs mapping + auto tableIter = moduleToFuncs.find(module); + // - if this is not our module, use the original + if (tableIter == moduleToFuncs.end()) + return origModuleGetattro(module, name); + + // - locate the name and retrieve the generating function + const char *attrNameStr = Shiboken::String::toCString(name); + auto &nameToFunc = tableIter->second; + // - create the real type (incarnateType checks this) + auto *type = incarnateType(module, attrNameStr, nameToFunc); + auto *ret = reinterpret_cast<PyObject *>(type); + // - if attribute does really not exist use the original + if (ret == nullptr && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + return origModuleGetattro(module, name); + } + + return ret; +} + +// PYSIDE-2404: Supply a new module dir for not yet visible entries. +// This modification is only for "our" modules. +static PyObject *_module_dir_template(PyObject * /* self */, PyObject *args) +{ + static PyObject *const _dict = Shiboken::String::createStaticString("__dict__"); + // The dir function must replace all of the builtin function. + PyObject *module{}; + if (!PyArg_ParseTuple(args, "O", &module)) + return nullptr; + + auto tableIter = moduleToFuncs.find(module); + assert(tableIter != moduleToFuncs.end()); + Shiboken::AutoDecRef dict(PyObject_GetAttr(module, _dict)); + auto *ret = PyDict_Keys(dict); + // Now add all elements that were not yet in the dict. + auto &nameToFunc = tableIter->second; + for (const auto &funcIter : nameToFunc) { + const char *name = funcIter.first.c_str(); + Shiboken::AutoDecRef pyName(PyUnicode_FromString(name)); + PyList_Append(ret, pyName); + } + return ret; +} + +static PyMethodDef module_methods[] = { + {"__dir__", (PyCFunction)_module_dir_template, METH_VARARGS, nullptr}, + {nullptr, nullptr, 0, nullptr} +}; + +// Python 3.8 - 3.12 +static int const LOAD_CONST_312 = 100; +static int const IMPORT_NAME_312 = 108; + +static bool isImportStar(PyObject *module) +{ + // Find out whether we have a star import. This must work even + // when we have no import support from feature. + static PyObject *const _f_code = Shiboken::String::createStaticString("f_code"); + static PyObject *const _f_lasti = Shiboken::String::createStaticString("f_lasti"); + static PyObject *const _f_back = Shiboken::String::createStaticString("f_back"); + static PyObject *const _co_code = Shiboken::String::createStaticString("co_code"); + static PyObject *const _co_consts = Shiboken::String::createStaticString("co_consts"); + static PyObject *const _co_names = Shiboken::String::createStaticString("co_names"); + + auto *obFrame = reinterpret_cast<PyObject *>(PyEval_GetFrame()); + if (obFrame == nullptr) + return true; // better assume worst-case. + + Py_INCREF(obFrame); + AutoDecRef dec_frame(obFrame); + + // Calculate the offset of the running import_name opcode on the stack. + // Right before that there must be a load_const with the tuple `("*",)`. + while (dec_frame.object() != Py_None) { + AutoDecRef dec_f_code(PyObject_GetAttr(dec_frame, _f_code)); + AutoDecRef dec_co_code(PyObject_GetAttr(dec_f_code, _co_code)); + AutoDecRef dec_f_lasti(PyObject_GetAttr(dec_frame, _f_lasti)); + Py_ssize_t f_lasti = PyLong_AsSsize_t(dec_f_lasti); + Py_ssize_t code_len; + char *co_code{}; + PyBytes_AsStringAndSize(dec_co_code, &co_code, &code_len); + uint8_t opcode2 = co_code[f_lasti]; + uint8_t opcode1 = co_code[f_lasti - 2]; + if (opcode1 == LOAD_CONST_312 && opcode2 == IMPORT_NAME_312) { + uint8_t oparg1 = co_code[f_lasti - 1]; + uint8_t oparg2 = co_code[f_lasti + 1]; + AutoDecRef dec_co_consts(PyObject_GetAttr(dec_f_code, _co_consts)); + auto *fromlist = PyTuple_GetItem(dec_co_consts, oparg1); + if (PyTuple_Check(fromlist) && PyTuple_Size(fromlist) == 1 + && Shiboken::String::toCString(PyTuple_GetItem(fromlist, 0))[0] == '*') { + AutoDecRef dec_co_names(PyObject_GetAttr(dec_f_code, _co_names)); + const char *name = String::toCString(PyTuple_GetItem(dec_co_names, oparg2)); + const char *modName = PyModule_GetName(module); + if (std::strcmp(name, modName) == 0) + return true; + } + } + dec_frame.reset(PyObject_GetAttr(dec_frame, _f_back)); + } + return false; +} + +// PYSIDE-2404: These modules produce ambiguous names which we cannot handle, yet. +static std::unordered_set<std::string> dontLazyLoad{ + "sample", + "smart", + "testbinding" +}; + +static const std::unordered_set<std::string> knownModules{ + "shiboken6.Shiboken", + "minimal", + "other", + "sample", + "smart", + "scriptableapplication", + "testbinding" +}; + +static bool canNotLazyLoad(PyObject *module) +{ + const char *modName = PyModule_GetName(module); + + // There are no more things that must be disabled :-D + return dontLazyLoad.find(modName) != dontLazyLoad.end(); +} + +static bool shouldLazyLoad(PyObject *module) +{ + const char *modName = PyModule_GetName(module); + + if (knownModules.find(modName) != knownModules.end()) + return true; + return std::strncmp(modName, "PySide6.", 8) == 0; +} + +void AddTypeCreationFunction(PyObject *module, + const char *name, + TypeCreationFunction func) +{ + static const char *flag = getenv("PYSIDE6_OPTION_LAZY"); + static const int value = flag != nullptr ? std::atoi(flag) : 1; + + // - locate the module in the moduleTofuncs mapping + auto tableIter = moduleToFuncs.find(module); + assert(tableIter != moduleToFuncs.end()); + // - Assign the name/generating function pair. + auto &nameToFunc = tableIter->second; + TypeCreationFunctionModulePair pair{func, module}; + auto nit = nameToFunc.find(name); + if (nit == nameToFunc.end()) + nameToFunc.insert(std::make_pair(name, pair)); + else + nit->second = pair; + + // PYSIDE-2404: Lazy Loading + // + // Options: + // 0 - switch lazy loading off. + // 1 - lazy loading for all known modules. + // 3 - lazy loading for any module. + // + // By default we lazy load all known modules (option = 1). + + if (value == 0 // completely disabled + || canNotLazyLoad(module) // for some reason we cannot lazy load + || (value == 1 && !shouldLazyLoad(module)) // not a known module + ) { + PyTypeObject *type = func(module); + PyModule_AddObject(module, name, reinterpret_cast<PyObject *>(type)); // steals reference + } +} + +void AddTypeCreationFunction(PyObject *module, + const char *name, + TypeCreationFunction func, + const char *containerName) +{ + // This version could be delayed as well, but for the few cases + // we simply fetch the container type and insert directly. + AutoDecRef obContainerType(PyObject_GetAttrString(module, containerName)); + PyTypeObject *type = func(obContainerType); + PyObject_SetAttrString(obContainerType, name, reinterpret_cast<PyObject *>(type)); // steals reference +} + +void AddTypeCreationFunction(PyObject *module, + const char *name, + TypeCreationFunction func, + const char *outerContainerName, + const char *innerContainerName) +{ + // This version has even more indirection. It is very rare, and + // we handle it directly. + AutoDecRef obOuterType(PyObject_GetAttrString(module, outerContainerName)); + AutoDecRef obInnerType(PyObject_GetAttrString(obOuterType, innerContainerName)); + PyTypeObject *type = func(obInnerType); + PyObject_SetAttrString(obInnerType, name, reinterpret_cast<PyObject *>(type)); // steals reference +} + +void AddTypeCreationFunction(PyObject *module, + const char *name, + TypeCreationFunction func, + const char *containerName3, + const char *containerName2, + const char *containerName) +{ + // This version has even mode indirection. It is very rare, and + // we handle it directly. + AutoDecRef obContainerType3(PyObject_GetAttrString(module, containerName3)); + AutoDecRef obContainerType2(PyObject_GetAttrString(obContainerType3, containerName2)); + AutoDecRef obContainerType(PyObject_GetAttrString(obContainerType2, containerName)); + PyTypeObject *type = func(obContainerType); + PyObject_SetAttrString(obContainerType, name, reinterpret_cast<PyObject *>(type)); // steals reference +} + PyObject *import(const char *moduleName) { PyObject *sysModules = PyImport_GetModuleDict(); PyObject *module = PyDict_GetItemString(sysModules, moduleName); - if (module) + if (module != nullptr) Py_INCREF(module); else module = PyImport_ImportModule(moduleName); - if (!module) - PyErr_Format(PyExc_ImportError,"could not import module '%s'", moduleName); + if (module == nullptr) + PyErr_Format(PyExc_ImportError, "could not import module '%s'", moduleName); return module; } -PyObject *create(const char *moduleName, void *moduleData) +// PYSIDE-2404: Redirecting import for "import *" support. +// +// The first import will be handled by the isImportStar function. +// But the same module might be imported twice, which would give no +// introspection due to module caching. + +static PyObject *origImportFunc{}; + +static PyObject *lazy_import(PyObject * /* self */, PyObject *args, PyObject *kwds) +{ + auto *ret = PyObject_Call(origImportFunc, args, kwds); + if (ret != nullptr) { + // PYSIDE-2404: Support star import when lazy loading. + if (PyTuple_Size(args) >= 4) { + auto *fromlist = PyTuple_GetItem(args, 3); + if (PyTuple_Check(fromlist) && PyTuple_Size(fromlist) == 1 + && Shiboken::String::toCString(PyTuple_GetItem(fromlist, 0))[0] == '*') + Shiboken::Module::resolveLazyClasses(ret); + } + } + return ret; +} + +static PyMethodDef lazy_methods[] = { + {"__lazy_import__", (PyCFunction)lazy_import, METH_VARARGS | METH_KEYWORDS, nullptr}, + {nullptr, nullptr, 0, nullptr} +}; + +PyObject *create(const char * /* modName */, void *moduleData) { + static auto *sysModules = PyImport_GetModuleDict(); + static auto *builtins = PyEval_GetBuiltins(); + static auto *partial = Pep_GetPartialFunction(); + static bool lazy_init{}; + Shiboken::init(); - return PyModule_Create(reinterpret_cast<PyModuleDef *>(moduleData)); + auto *module = PyModule_Create(reinterpret_cast<PyModuleDef *>(moduleData)); + + // Setup of a dir function for "missing" classes. + auto *moduleDirTemplate = PyCFunction_NewEx(module_methods, nullptr, nullptr); + // Turn this function into a bound object, so we have access to the module. + auto *moduleDir = PyObject_CallFunctionObjArgs(partial, moduleDirTemplate, module, nullptr); + PyModule_AddObject(module, module_methods->ml_name, moduleDir); // steals reference + // Insert an initial empty table for the module. + NameToTypeFunctionMap empty; + moduleToFuncs.insert(std::make_pair(module, empty)); + + // A star import must be done unconditionally. Use the complete name. + if (isImportStar(module)) + dontLazyLoad.insert(PyModule_GetName(module)); + + if (!lazy_init) { + // Install the getattr patch. + origModuleGetattro = PyModule_Type.tp_getattro; + PyModule_Type.tp_getattro = PyModule_lazyGetAttro; + // Add the lazy import redirection. + origImportFunc = PyDict_GetItemString(builtins, "__import__"); + auto *func = PyCFunction_NewEx(lazy_methods, nullptr, nullptr); + PyDict_SetItemString(builtins, "__import__", func); + // Everything is set. + lazy_init = true; + } + // PYSIDE-2404: Nuitka inserts some additional code in standalone mode + // in an invisible virtual module (i.e. `QtCore-postLoad`) + // that gets imported before the running import can call + // `_PyImport_FixupExtensionObject` which does the insertion + // into `sys.modules`. This can cause a race condition. + // Insert the module early into the module dict to prevend recursion. + PyDict_SetItemString(sysModules, PyModule_GetName(module), module); + return module; } -void registerTypes(PyObject *module, PyTypeObject **types) +void registerTypes(PyObject *module, TypeInitStruct *types) { auto iter = moduleTypes.find(module); if (iter == moduleTypes.end()) moduleTypes.insert(std::make_pair(module, types)); } -PyTypeObject **getTypes(PyObject *module) +TypeInitStruct *getTypes(PyObject *module) { auto iter = moduleTypes.find(module); return (iter == moduleTypes.end()) ? 0 : iter->second; diff --git a/sources/shiboken6/libshiboken/sbkmodule.h b/sources/shiboken6/libshiboken/sbkmodule.h index a4f7837f5..1b3de33b7 100644 --- a/sources/shiboken6/libshiboken/sbkmodule.h +++ b/sources/shiboken6/libshiboken/sbkmodule.h @@ -12,8 +12,22 @@ extern "C" struct SbkConverter; } -namespace Shiboken { -namespace Module { +namespace Shiboken::Module { + +struct TypeInitStruct +{ + PyTypeObject *type; + const char *fullName; +}; + +/// PYSIDE-2404: Replacing the arguments in cpythonTypeNameExt by a function. +LIBSHIBOKEN_API PyTypeObject *get(TypeInitStruct &typeStruct); + +/// PYSIDE-2404: Make sure that mentioned classes really exist. +LIBSHIBOKEN_API void loadLazyClassesWithName(const char *name); + +/// PYSIDE-2404: incarnate all classes for star imports. +LIBSHIBOKEN_API void resolveLazyClasses(PyObject *module); /** * Imports and returns the module named \p moduleName, or a NULL pointer in case of failure. @@ -30,19 +44,43 @@ LIBSHIBOKEN_API PyObject *import(const char *moduleName); */ LIBSHIBOKEN_API PyObject *create(const char *moduleName, void *moduleData); +using TypeCreationFunction = PyTypeObject *(*)(PyObject *module); + +/// Adds a type creation function to the module. +LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module, + const char *name, + TypeCreationFunction func); + +LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module, + const char *name, + TypeCreationFunction func, + const char *containerName); + +LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module, + const char *name, + TypeCreationFunction func, + const char *outerContainerName, + const char *innerContainerName); + +LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module, + const char *name, + TypeCreationFunction func, + const char *containerName3, + const char *containerName2, + const char *containerName); /** * Registers the list of types created by \p module. * \param module Module where the types were created. * \param types Array of PyTypeObject *objects representing the types created on \p module. */ -LIBSHIBOKEN_API void registerTypes(PyObject *module, PyTypeObject **types); +LIBSHIBOKEN_API void registerTypes(PyObject *module, TypeInitStruct *types); /** * Retrieves the array of types. * \param module Module where the types were created. * \returns A pointer to the PyTypeObject *array of types. */ -LIBSHIBOKEN_API PyTypeObject **getTypes(PyObject *module); +LIBSHIBOKEN_API TypeInitStruct *getTypes(PyObject *module); /** * Registers the list of converters created by \p module for non-wrapper types. @@ -58,6 +96,6 @@ LIBSHIBOKEN_API void registerTypeConverters(PyObject *module, SbkConverter **con */ LIBSHIBOKEN_API SbkConverter **getTypeConverters(PyObject *module); -} } // namespace Shiboken::Module +} // namespace Shiboken::Module #endif // SBK_MODULE_H diff --git a/sources/shiboken6/libshiboken/sbknumpy.cpp b/sources/shiboken6/libshiboken/sbknumpy.cpp index 46b3af702..2e1c64d73 100644 --- a/sources/shiboken6/libshiboken/sbknumpy.cpp +++ b/sources/shiboken6/libshiboken/sbknumpy.cpp @@ -7,17 +7,42 @@ # include <numpy/arrayobject.h> #endif +#include "helper.h" #include "sbknumpycheck.h" +#include "sbkcpptonumpy.h" #include "sbknumpyview.h" +#include <algorithm> + namespace Shiboken::Numpy { +#ifdef HAVE_NUMPY +static void initNumPy() +{ + // PYSIDE-2404: Delay-initialize numpy from check() as it causes a + // significant startup delay (~770 allocations in memray) + static bool initialized = false; + if (initialized) + return; + initialized = true; + // Expanded from macro "import_array" in __multiarray_api.h + // Make sure to read about the magic defines PY_ARRAY_UNIQUE_SYMBOL etc., + // when changing this or spreading the code over several source files. + if (_import_array() < 0) { + PyErr_Print(); + PyErr_Clear(); + } +} +#endif // HAVE_NUMPY + bool check(PyObject *pyIn) { #ifdef HAVE_NUMPY + initNumPy(); return PyArray_Check(pyIn); #else + SBK_UNUSED(pyIn); return false; #endif } @@ -28,6 +53,7 @@ bool check(PyObject *pyIn) // translation unit (see comment at initNumPyArrayConverters()). #include "sbknumpyview.cpp" +#include "sbkcpptonumpy.cpp" #ifdef HAVE_NUMPY # include "sbknumpyarrayconverter.cpp" #endif diff --git a/sources/shiboken6/libshiboken/sbknumpyarrayconverter.cpp b/sources/shiboken6/libshiboken/sbknumpyarrayconverter.cpp index c8541adf5..835a97524 100644 --- a/sources/shiboken6/libshiboken/sbknumpyarrayconverter.cpp +++ b/sources/shiboken6/libshiboken/sbknumpyarrayconverter.cpp @@ -94,8 +94,7 @@ std::ostream &operator<<(std::ostream &str, PyArrayObject *o) return str; } -namespace Shiboken { -namespace Conversions { +namespace Shiboken::Conversions { // Internals from sbkarrayconverter.cpp SbkArrayConverter *createArrayConverter(IsArrayConvertibleToCppFunc toCppCheckFunc); @@ -105,6 +104,7 @@ SbkArrayConverter *unimplementedArrayConverter(); template <int dimension> static bool isPrimitiveArray(PyObject *pyIn, int expectedNpType) { + Shiboken::Numpy::initNumPy(); if (!PyArray_Check(pyIn)) return false; auto *pya = reinterpret_cast<PyArrayObject *>(pyIn); @@ -210,6 +210,9 @@ static PythonToCppFunc checkArray2(PyObject *pyIn, int dim1, int dim2) template <class T> static void setOrExtendArrayConverter(int dimension, IsArrayConvertibleToCppFunc toCppCheckFunc) { + // PYSIDE-2404/FIXME: When adding a C++ -> Python conversion, be sure + // to delay-initialize numpy in the converter (similar to the + // initialization in check() for the Python -> C++ conversion). SbkArrayConverter *arrayConverter = ArrayTypeConverter<T>(dimension); if (arrayConverter == unimplementedArrayConverter()) { arrayConverter = createArrayConverter(toCppCheckFunc); @@ -235,15 +238,6 @@ static inline void extendArrayConverter2() void initNumPyArrayConverters() { - // Expanded from macro "import_array" in __multiarray_api.h - // Make sure to read about the magic defines PY_ARRAY_UNIQUE_SYMBOL etc., - // when changing this or spreading the code over several source files. - if (_import_array() < 0) { - if (debugNumPy) - PyErr_Print(); - PyErr_Clear(); - return; - } // Extend the converters for primitive types by NumPy ones. extendArrayConverter1<short, NPY_SHORT>(); extendArrayConverter2<short, NPY_SHORT>(); @@ -273,5 +267,4 @@ void initNumPyArrayConverters() extendArrayConverter2<double, NPY_DOUBLE>(); } -} // namespace Conversions -} // namespace Shiboken +} // namespace Shiboken::Conversions diff --git a/sources/shiboken6/libshiboken/sbknumpyview.cpp b/sources/shiboken6/libshiboken/sbknumpyview.cpp index 44a4b5587..bafbf8038 100644 --- a/sources/shiboken6/libshiboken/sbknumpyview.cpp +++ b/sources/shiboken6/libshiboken/sbknumpyview.cpp @@ -6,12 +6,54 @@ #include "helper.h" #include <iostream> #include <iomanip> +#include <optional> #ifdef HAVE_NUMPY namespace Shiboken { namespace Numpy { +static std::optional<View::Type> viewTypeFromNumPy(int npt) +{ + switch (npt) { + case NPY_SHORT: + return View::Int16; + case NPY_USHORT: + return View::Unsigned16; + case NPY_INT: + return View::Int; + case NPY_UINT: + return View::Unsigned; + case NPY_LONG: + if constexpr (sizeof(long) == sizeof(int)) + return View::Int; + if constexpr (sizeof(long) == sizeof(int64_t)) + return View::Int64; + break; + case NPY_ULONG: + if constexpr (sizeof(long) == sizeof(int)) + return View::Unsigned; + if constexpr (sizeof(long) == sizeof(int64_t)) + return View::Unsigned64; + break; + case NPY_LONGLONG: + if constexpr (sizeof(long long) == 8) + return View::Int64; + break; + case NPY_ULONGLONG: + if constexpr (sizeof(long long) == 8) + return View::Unsigned64; + break; + case NPY_FLOAT: + return View::Float; + case NPY_DOUBLE: + return View::Double; + default: + break; + } + return {}; +} + View View::fromPyObject(PyObject *pyIn) { if (pyIn == nullptr || PyArray_Check(pyIn) == 0) @@ -23,27 +65,13 @@ View View::fromPyObject(PyObject *pyIn) if (ndim > 2) return {}; - View::Type type; - switch (PyArray_TYPE(ar)) { - case NPY_INT: - type = View::Int; - break; - case NPY_UINT: - type = View::Unsigned; - break; - case NPY_FLOAT: - type = View::Float; - break; - case NPY_DOUBLE: - type = View::Double; - break; - default: + const auto typeO = viewTypeFromNumPy(PyArray_TYPE(ar)); + if (!typeO.has_value()) return {}; - } View result; result.ndim = ndim; - result.type = type; + result.type = typeO.value(); result.data = PyArray_DATA(ar); result.dimensions[0] = PyArray_DIMS(ar)[0]; result.stride[0] = PyArray_STRIDES(ar)[0]; @@ -91,11 +119,29 @@ std::ostream &operator<<(std::ostream &str, const debugPyArrayObject &a) } str << "], type="; switch (type) { + case NPY_SHORT: + str << "short"; + break; + case NPY_USHORT: + str << "ushort"; + break; case NPY_INT: - str << "int"; + str << "int32"; break; case NPY_UINT: - str << "uint"; + str << "uint32"; + break; + case NPY_LONG: + str << "long"; + break; + case NPY_ULONG: + str << "ulong"; + break; + case NPY_LONGLONG: + str << "long long"; + break; + case NPY_ULONGLONG: + str << "ulong long"; break; case NPY_FLOAT: str << "float"; @@ -122,12 +168,30 @@ std::ostream &operator<<(std::ostream &str, const debugPyArrayObject &a) if (const int dim0 = PyArray_DIMS(ar)[0]) { auto *data = PyArray_DATA(ar); switch (type) { + case NPY_SHORT: + debugArray(str, reinterpret_cast<const short *>(data), dim0); + break; + case NPY_USHORT: + debugArray(str, reinterpret_cast<const unsigned short *>(data), dim0); + break; case NPY_INT: debugArray(str, reinterpret_cast<const int *>(data), dim0); break; case NPY_UINT: debugArray(str, reinterpret_cast<const unsigned *>(data), dim0); break; + case NPY_LONG: + debugArray(str, reinterpret_cast<const long *>(data), dim0); + break; + case NPY_ULONG: + debugArray(str, reinterpret_cast<const unsigned long*>(data), dim0); + break; + case NPY_LONGLONG: + debugArray(str, reinterpret_cast<const long long *>(data), dim0); + break; + case NPY_ULONGLONG: + debugArray(str, reinterpret_cast<const unsigned long long *>(data), dim0); + break; case NPY_FLOAT: debugArray(str, reinterpret_cast<const float *>(data), dim0); break; diff --git a/sources/shiboken6/libshiboken/sbknumpyview.h b/sources/shiboken6/libshiboken/sbknumpyview.h index d41e2c716..918913b78 100644 --- a/sources/shiboken6/libshiboken/sbknumpyview.h +++ b/sources/shiboken6/libshiboken/sbknumpyview.h @@ -22,7 +22,7 @@ LIBSHIBOKEN_API bool check(PyObject *pyIn); /// numpy headers. struct LIBSHIBOKEN_API View { - enum Type { Int, Unsigned, Float, Double}; + enum Type { Int, Unsigned, Float, Double, Int16, Unsigned16, Int64, Unsigned64 }; static View fromPyObject(PyObject *pyIn); diff --git a/sources/shiboken6/libshiboken/sbkpython.h b/sources/shiboken6/libshiboken/sbkpython.h index e8fa29cbb..e62fa13ae 100644 --- a/sources/shiboken6/libshiboken/sbkpython.h +++ b/sources/shiboken6/libshiboken/sbkpython.h @@ -5,7 +5,6 @@ #define SBKPYTHON_H #include "sbkversion.h" -#define PyEnumMeta_Check(x) (strcmp(Py_TYPE(x)->tp_name, "EnumMeta") == 0) // Qt's "slots" macro collides with the "slots" member variables // used in some Python structs. For compilers that support push_macro, diff --git a/sources/shiboken6/libshiboken/sbksmartpointer.cpp b/sources/shiboken6/libshiboken/sbksmartpointer.cpp new file mode 100644 index 000000000..ee28f7db8 --- /dev/null +++ b/sources/shiboken6/libshiboken/sbksmartpointer.cpp @@ -0,0 +1,58 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "sbksmartpointer.h" +#include "sbkstring.h" +#include "autodecref.h" + +#include <unordered_set> + +namespace Shiboken::SmartPointer +{ + +PyObject *repr(PyObject *pointer, PyObject *pointee) +{ + Shiboken::AutoDecRef pointerRepr(Shiboken::String::repr(pointer)); + if (pointer == nullptr) + return pointerRepr.release(); + + Shiboken::AutoDecRef pointeeRepr(pointee != nullptr + ? PyObject_Repr(pointee) + : Shiboken::String::repr(pointee)); + + return PyUnicode_FromFormat("%U (%U)", pointerRepr.object(), pointeeRepr.object()); +} + +// __dir__ for a smart pointer. Add the __dir__ entries of the pointee to the list. +PyObject *dir(PyObject *pointer, PyObject *pointee) +{ + if (pointer == nullptr) + return PyList_New(0); + // Get the pointer's dir entries. Note: PyObject_Dir() cannot be called on + // self, will crash. Work around by using the type dict keys. + AutoDecRef tpDict(PepType_GetDict(Py_TYPE(pointer))); + auto *result = PyMapping_Keys(tpDict); + + if (pointee != nullptr && pointee != Py_None) { + // Add the entries of the pointee that do not exist in the pointer's list. + // Since Python internally caches strings; we can use a set of PyObject *. + std::unordered_set<PyObject *> knownStrings; + for (Py_ssize_t i = 0, size = PySequence_Size(result); i < size; ++i) { + Shiboken::AutoDecRef item(PySequence_GetItem(result, i)); + knownStrings.insert(item.object()); + } + const auto knownEnd = knownStrings.end(); + + Shiboken::AutoDecRef pointeeDir(PyObject_Dir(pointee)); + for (Py_ssize_t i = 0, size = PySequence_Size(pointeeDir.object()); i < size; ++i) { + Shiboken::AutoDecRef item(PySequence_GetItem(pointeeDir, i)); + if (knownStrings.find(item.object()) == knownEnd) + PyList_Append(result, item.object()); + } + } + + PyList_Sort(result); + return result; +} + +} // namespace Shiboken::SmartPointer diff --git a/sources/shiboken6/libshiboken/sbksmartpointer.h b/sources/shiboken6/libshiboken/sbksmartpointer.h new file mode 100644 index 000000000..5e2022722 --- /dev/null +++ b/sources/shiboken6/libshiboken/sbksmartpointer.h @@ -0,0 +1,18 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef SBK_SBKSMARTPOINTER_H +#define SBK_SBKSMARTPOINTER_H + +#include "sbkpython.h" +#include "shibokenmacros.h" + +namespace Shiboken::SmartPointer +{ + +LIBSHIBOKEN_API PyObject *repr(PyObject *pointer, PyObject *pointee); +LIBSHIBOKEN_API PyObject *dir(PyObject *pointer, PyObject *pointee); + +} // namespace Shiboken::SmartPointer + +#endif // SBK_SBKSMARTPOINTER_H diff --git a/sources/shiboken6/libshiboken/sbkstaticstrings.cpp b/sources/shiboken6/libshiboken/sbkstaticstrings.cpp index 3f2484f31..023de0ea4 100644 --- a/sources/shiboken6/libshiboken/sbkstaticstrings.cpp +++ b/sources/shiboken6/libshiboken/sbkstaticstrings.cpp @@ -24,6 +24,7 @@ STATIC_STRING_IMPL(im_self, "im_self") STATIC_STRING_IMPL(loads, "loads") STATIC_STRING_IMPL(multi, "multi") STATIC_STRING_IMPL(name, "name") +STATIC_STRING_IMPL(orig_dict, "orig_dict") STATIC_STRING_IMPL(qApp, "qApp") STATIC_STRING_IMPL(result, "result") STATIC_STRING_IMPL(select_id, "select_id") @@ -75,8 +76,13 @@ STATIC_STRING_IMPL(iter, "__iter__") STATIC_STRING_IMPL(mro, "__mro__") STATIC_STRING_IMPL(new_, "__new__") STATIC_STRING_IMPL(objclass, "__objclass__") -STATIC_STRING_IMPL(signature, "__signature__") STATIC_STRING_IMPL(weakrefoffset, "__weakrefoffset__") STATIC_STRING_IMPL(opaque_container, "__opaque_container__") } // namespace PyMagicName + +namespace Messages +{ +STATIC_STRING_IMPL(unknownException, "An unknown exception was caught") +} + } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/sbkstaticstrings.h b/sources/shiboken6/libshiboken/sbkstaticstrings.h index c89fdc9cd..017790ee3 100644 --- a/sources/shiboken6/libshiboken/sbkstaticstrings.h +++ b/sources/shiboken6/libshiboken/sbkstaticstrings.h @@ -23,6 +23,7 @@ LIBSHIBOKEN_API PyObject *im_self(); LIBSHIBOKEN_API PyObject *loads(); LIBSHIBOKEN_API PyObject *multi(); LIBSHIBOKEN_API PyObject *name(); +LIBSHIBOKEN_API PyObject *orig_dict(); LIBSHIBOKEN_API PyObject *result(); LIBSHIBOKEN_API PyObject *select_id(); LIBSHIBOKEN_API PyObject *value(); @@ -50,6 +51,11 @@ LIBSHIBOKEN_API PyObject *code(); LIBSHIBOKEN_API PyObject *rlshift(); LIBSHIBOKEN_API PyObject *rrshift(); } // namespace PyMagicName + +namespace Messages +{ +LIBSHIBOKEN_API PyObject *unknownException(); +} // Messages } // namespace Shiboken #endif // SBKSTATICSTRINGS_H diff --git a/sources/shiboken6/libshiboken/sbkstring.cpp b/sources/shiboken6/libshiboken/sbkstring.cpp index 8f2dc6d52..1471cd7fe 100644 --- a/sources/shiboken6/libshiboken/sbkstring.cpp +++ b/sources/shiboken6/libshiboken/sbkstring.cpp @@ -233,4 +233,16 @@ PyObject *getSnakeCaseName(PyObject *name, bool lower) return name; } +// Return a generic representation of a PyObject as does PyObject_Repr(). +// Note: PyObject_Repr() may not be called on self from __repr__() as this +// causes a recursion. +PyObject *repr(PyObject *o) +{ + if (o == nullptr) + return PyUnicode_FromString("<NULL>"); + if (o == Py_None) + return PyUnicode_FromString("None"); + return PyUnicode_FromFormat("<%s object at %p>", Py_TYPE(o)->tp_name, o); +} + } // namespace Shiboken::String diff --git a/sources/shiboken6/libshiboken/sbkstring.h b/sources/shiboken6/libshiboken/sbkstring.h index a24c01def..f91847c11 100644 --- a/sources/shiboken6/libshiboken/sbkstring.h +++ b/sources/shiboken6/libshiboken/sbkstring.h @@ -29,6 +29,7 @@ namespace String LIBSHIBOKEN_API PyObject *createStaticString(const char *str); LIBSHIBOKEN_API PyObject *getSnakeCaseName(const char *name, bool lower); LIBSHIBOKEN_API PyObject *getSnakeCaseName(PyObject *name, bool lower); + LIBSHIBOKEN_API PyObject *repr(PyObject *o); } // namespace String } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/sbktypefactory.cpp b/sources/shiboken6/libshiboken/sbktypefactory.cpp index 0da1a8e23..079548eed 100644 --- a/sources/shiboken6/libshiboken/sbktypefactory.cpp +++ b/sources/shiboken6/libshiboken/sbktypefactory.cpp @@ -7,6 +7,8 @@ extern "C" { +using Shiboken::AutoDecRef; + PyTypeObject *SbkType_FromSpec(PyType_Spec *spec) { return SbkType_FromSpec_BMDWB(spec, nullptr, nullptr, 0, 0, nullptr); @@ -37,6 +39,60 @@ static PyObject *_PyType_FromSpecWithBases(PyType_Spec *, PyObject *); #endif // PYPY_VERSION +// PYSIDE-2230: Not so temporary fix for Python 3.12. +// A tp_new is no longer allowed in a meta class. +// Hopefully, the Python devs will supply the missing support. +// It turned out that they will not fix that, as expected. +// Note: Python 3.12 is the first version that grabs the metaclass from base classes. +static PyObject *_PyType_FromSpecWithBasesHack(PyType_Spec *spec, + PyObject *bases, + PyTypeObject *meta) +{ + PyTypeObject *keepMeta{}; + newfunc keepNew{}; + AutoDecRef basesPatch{}; + + if (bases) { + if (bases == Py_None) { + // PYSIDE-2230: This is the SbkObject entry which has no base to provide + // the metaclass. We patch it in by modifying `object`s class. + assert(meta); + auto *base = reinterpret_cast<PyObject *>(&PyBaseObject_Type); + base->ob_type = meta; + basesPatch.reset(Py_BuildValue("(O)", &PyBaseObject_Type)); + bases = basesPatch.object(); + } + + Py_ssize_t n = PyTuple_GET_SIZE(bases); + for (auto idx = 0; idx < n; ++idx) { + PyTypeObject *base = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(bases, idx)); + PyTypeObject *meta = Py_TYPE(base); + if (meta->tp_new != PyType_Type.tp_new) { + // make sure there is no second meta class + assert(keepMeta == nullptr); + keepMeta = meta; + keepNew = meta->tp_new; + meta->tp_new = PyType_Type.tp_new; + } + } + } + +#if !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030C0000 + auto *ret = PyType_FromMetaclass(meta, nullptr /*module*/, spec, bases); +#else + auto *ret = _PyType_FromSpecWithBases(spec, bases); +#endif + + if (keepMeta) + keepMeta->tp_new = keepNew; + if (basesPatch.object()) { + // undo the metaclass patch. + auto *base = PyTuple_GET_ITEM(basesPatch.object(), 0); + base->ob_type = &PyType_Type; + } + return ret; +} + PyTypeObject *SbkType_FromSpec_BMDWB(PyType_Spec *spec, PyObject *bases, PyTypeObject *meta, @@ -61,7 +117,7 @@ PyTypeObject *SbkType_FromSpec_BMDWB(PyType_Spec *spec, int package_level = atoi(spec->name); const char *mod = new_spec.name = colon + 1; - PyObject *obType = _PyType_FromSpecWithBases(&new_spec, bases); + PyObject *obType = _PyType_FromSpecWithBasesHack(&new_spec, bases, meta); if (obType == nullptr) return nullptr; @@ -73,8 +129,8 @@ PyTypeObject *SbkType_FromSpec_BMDWB(PyType_Spec *spec, qual = dot + 1; } int mlen = qual - mod - 1; - Shiboken::AutoDecRef module(Shiboken::String::fromCString(mod, mlen)); - Shiboken::AutoDecRef qualname(Shiboken::String::fromCString(qual)); + AutoDecRef module(Shiboken::String::fromCString(mod, mlen)); + AutoDecRef qualname(Shiboken::String::fromCString(qual)); auto *type = reinterpret_cast<PyTypeObject *>(obType); @@ -98,9 +154,10 @@ PyTypeObject *SbkType_FromSpec_BMDWB(PyType_Spec *spec, // PyType_Ready too early. (at least in PyPy, which caused pretty long debugging.) auto *ht = reinterpret_cast<PyHeapTypeObject *>(type); ht->ht_qualname = qualname; - if (PyDict_SetItem(type->tp_dict, Shiboken::PyMagicName::qualname(), qualname)) + AutoDecRef tpDict(PepType_GetDict(type)); + if (PyDict_SetItem(tpDict.object(), Shiboken::PyMagicName::qualname(), qualname)) return nullptr; - if (PyDict_SetItem(type->tp_dict, Shiboken::PyMagicName::module(), module)) + if (PyDict_SetItem(tpDict.object(), Shiboken::PyMagicName::module(), module)) return nullptr; PyType_Ready(type); #else @@ -329,7 +386,7 @@ _PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) /// Here is the only change needed: Do not finalize type creation. // if (PyType_Ready(type) < 0) // goto fail; - type->tp_dict = PyDict_New(); + PepType_SetDict(type, PyDict_New()); /// This is not found in PyPy: // if (type->tp_dictoffset) { // res->ht_cached_keys = _PyDict_NewKeysForClass(); diff --git a/sources/shiboken6/libshiboken/sbkversion.h.in b/sources/shiboken6/libshiboken/sbkversion.h.in index 7f99abc3e..5c0b38fdb 100644 --- a/sources/shiboken6/libshiboken/sbkversion.h.in +++ b/sources/shiboken6/libshiboken/sbkversion.h.in @@ -10,8 +10,8 @@ #define SHIBOKEN_MICRO_VERSION @shiboken_MICRO_VERSION@ #define SHIBOKEN_RELEASE_LEVEL "final" #define SHIBOKEN_SERIAL 0 -#define PYTHON_VERSION_MAJOR @PYTHON_VERSION_MAJOR@ -#define PYTHON_VERSION_MINOR @PYTHON_VERSION_MINOR@ -#define PYTHON_VERSION_PATCH @PYTHON_VERSION_PATCH@ +#define PYTHON_VERSION_MAJOR @Python_VERSION_MAJOR@ +#define PYTHON_VERSION_MINOR @Python_VERSION_MINOR@ +#define PYTHON_VERSION_PATCH @Python_VERSION_PATCH@ #endif diff --git a/sources/shiboken6/libshiboken/sbkwindows.h b/sources/shiboken6/libshiboken/sbkwindows.h new file mode 100644 index 000000000..9e753fa5e --- /dev/null +++ b/sources/shiboken6/libshiboken/sbkwindows.h @@ -0,0 +1,17 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef SBKWINDOWS_H +#define SBKWINDOWS_H + +#ifdef _WIN32 +# ifndef NOMINMAX +# define NOMINMAX +# endif +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include <windows.h> +#endif + +#endif // SBKWINDOWS_H diff --git a/sources/shiboken6/libshiboken/shiboken.h b/sources/shiboken6/libshiboken/shiboken.h index 13a15e1f4..fcf777ae0 100644 --- a/sources/shiboken6/libshiboken/shiboken.h +++ b/sources/shiboken6/libshiboken/shiboken.h @@ -11,10 +11,10 @@ #include "gilstate.h" #include "threadstatesaver.h" #include "helper.h" +#include "pyobjectholder.h" #include "sbkarrayconverter.h" #include "sbkconverter.h" #include "sbkenum.h" -#include "sbkenum_p.h" // PYSIDE-1735: This is during the migration, only. #include "sbkerrors.h" #include "sbkmodule.h" #include "sbkstring.h" diff --git a/sources/shiboken6/libshiboken/signature.h b/sources/shiboken6/libshiboken/signature.h index 94f234112..e0130b5a6 100644 --- a/sources/shiboken6/libshiboken/signature.h +++ b/sources/shiboken6/libshiboken/signature.h @@ -4,13 +4,15 @@ #ifndef SIGNATURE_H #define SIGNATURE_H +#include "shibokenmacros.h" +#include "sbkpython.h" + extern "C" { LIBSHIBOKEN_API int InitSignatureStrings(PyTypeObject *, const char *[]); LIBSHIBOKEN_API void FinishSignatureInitialization(PyObject *, const char *[]); LIBSHIBOKEN_API void SetError_Argument(PyObject *, const char *, PyObject *); -LIBSHIBOKEN_API PyObject *Sbk_TypeGet___signature__(PyObject *, PyObject *); LIBSHIBOKEN_API PyObject *Sbk_TypeGet___doc__(PyObject *); LIBSHIBOKEN_API PyObject *GetFeatureDict(); diff --git a/sources/shiboken6/libshiboken/signature/signature.cpp b/sources/shiboken6/libshiboken/signature/signature.cpp index e70d79581..3255cb56d 100644 --- a/sources/shiboken6/libshiboken/signature/signature.cpp +++ b/sources/shiboken6/libshiboken/signature/signature.cpp @@ -13,13 +13,16 @@ // General documentation can be found in `signature_doc.rst`. // +#include "signature.h" +#include "signature_p.h" + #include "basewrapper.h" #include "autodecref.h" #include "sbkstring.h" #include "sbkstaticstrings.h" #include "sbkstaticstrings_p.h" #include "sbkfeature_base.h" -#include "signature_p.h" + #include <structmember.h> using namespace Shiboken; @@ -75,27 +78,24 @@ PyObject *GetTypeKey(PyObject *ob) * * PYSIDE-1286: We use correct __module__ and __qualname__, now. */ - // XXX we obtain also the current selection. - // from the current module name. AutoDecRef module_name(PyObject_GetAttr(ob, PyMagicName::module())); if (module_name.isNull()) { // We have no module_name because this is a module ;-) PyErr_Clear(); module_name.reset(PyObject_GetAttr(ob, PyMagicName::name())); - return Py_BuildValue("O"/*i"*/, module_name.object()/*, getFeatureSelectId()*/); + return Py_BuildValue("O", module_name.object()); } AutoDecRef class_name(PyObject_GetAttr(ob, PyMagicName::qualname())); if (class_name.isNull()) { Py_FatalError("Signature: missing class name in GetTypeKey"); return nullptr; } - return Py_BuildValue("(O"/*i*/"O)", module_name.object(), /*getFeatureSelectId(),*/ - class_name.object()); + return Py_BuildValue("(OO)", module_name.object(), class_name.object()); } static PyObject *empty_dict = nullptr; -PyObject *TypeKey_to_PropsDict(PyObject *type_key, PyObject *obtype) +PyObject *TypeKey_to_PropsDict(PyObject *type_key) { PyObject *dict = PyDict_GetItem(pyside_globals->arg_dict, type_key); if (dict == nullptr) { @@ -143,7 +143,7 @@ PyObject *GetSignature_Method(PyObject *obfunc, PyObject *modifier) AutoDecRef type_key(GetTypeKey(obtype_mod)); if (type_key.isNull()) Py_RETURN_NONE; - PyObject *dict = TypeKey_to_PropsDict(type_key, obtype_mod); + PyObject *dict = TypeKey_to_PropsDict(type_key); if (dict == nullptr) return nullptr; AutoDecRef func_name(PyObject_GetAttr(obfunc, PyMagicName::name())); @@ -163,7 +163,7 @@ PyObject *GetSignature_Function(PyObject *obfunc, PyObject *modifier) AutoDecRef type_key(GetTypeKey(obtype_mod)); if (type_key.isNull()) Py_RETURN_NONE; - PyObject *dict = TypeKey_to_PropsDict(type_key, obtype_mod); + PyObject *dict = TypeKey_to_PropsDict(type_key); if (dict == nullptr) return nullptr; AutoDecRef func_name(PyObject_GetAttr(obfunc, PyMagicName::name())); @@ -191,13 +191,13 @@ PyObject *GetSignature_Wrapper(PyObject *ob, PyObject *modifier) AutoDecRef class_key(GetTypeKey(objclass)); if (func_name.isNull() || objclass.isNull() || class_key.isNull()) return nullptr; - PyObject *dict = TypeKey_to_PropsDict(class_key, objclass); + PyObject *dict = TypeKey_to_PropsDict(class_key); if (dict == nullptr) return nullptr; PyObject *props = PyDict_GetItem(dict, func_name); if (props == nullptr) { // handle `__init__` like the class itself - if (strcmp(String::toCString(func_name), "__init__") == 0) + if (PyUnicode_CompareWithASCIIString(func_name, "__init__") == 0) return GetSignature_TypeMod(objclass, modifier); Py_RETURN_NONE; } @@ -209,7 +209,7 @@ PyObject *GetSignature_TypeMod(PyObject *ob, PyObject *modifier) AutoDecRef ob_name(PyObject_GetAttr(ob, PyMagicName::name())); AutoDecRef ob_key(GetTypeKey(ob)); - PyObject *dict = TypeKey_to_PropsDict(ob_key, ob); + PyObject *dict = TypeKey_to_PropsDict(ob_key); if (dict == nullptr) return nullptr; PyObject *props = PyDict_GetItem(dict, ob_name); @@ -227,6 +227,8 @@ PyObject *GetSignature_TypeMod(PyObject *ob, PyObject *modifier) // The `modifier` argument is a string that is passed in from `loader.py`. // Configuration what the modifiers mean is completely in Python. // +// PYSIDE-2101: The __signature__ attribute is gone due to rlcompleter. +// PyObject *get_signature_intern(PyObject *ob, PyObject *modifier) { @@ -246,6 +248,9 @@ PyObject *get_signature_intern(PyObject *ob, PyObject *modifier) return pyside_tp_get___signature__(ob, modifier); if (Py_TYPE(ob) == &PyWrapperDescr_Type) return pyside_wd_get___signature__(ob, modifier); + // For classmethods we use the simple wrapper description implementation. + if (Py_TYPE(ob) == &PyClassMethodDescr_Type) + return pyside_wd_get___signature__(ob, modifier); return nullptr; } @@ -254,8 +259,6 @@ static PyObject *get_signature(PyObject * /* self */, PyObject *args) PyObject *ob; PyObject *modifier = nullptr; - init_module_1(); - if (!PyArg_ParseTuple(args, "O|O", &ob, &modifier)) return nullptr; if (Py_TYPE(ob) == PepFunction_TypePtr) @@ -287,14 +290,25 @@ static PyObject *feature_import(PyObject * /* self */, PyObject *args, PyObject if (import_func == nullptr) { Py_FatalError("builtins has no \"__orig_import__\" function"); } - return PyObject_Call(import_func, args, kwds); + ret = PyObject_Call(import_func, args, kwds); + if (ret) { + // PYSIDE-2029: Intercept after the import to search for PySide usage. + PyObject *post = PyObject_CallFunctionObjArgs(pyside_globals->feature_imported_func, + ret, nullptr); + Py_XDECREF(post); + if (post == nullptr) { + Py_DECREF(ret); + return nullptr; + } + } + return ret; } PyMethodDef signature_methods[] = { - {"__feature_import__", (PyCFunction)feature_import, METH_VARARGS | METH_KEYWORDS}, + {"__feature_import__", (PyCFunction)feature_import, METH_VARARGS | METH_KEYWORDS, nullptr}, {"get_signature", (PyCFunction)get_signature, METH_VARARGS, - "get the __signature__, but pass an optional string parameter"}, - {nullptr, nullptr} + "get the signature, passing an optional string parameter"}, + {nullptr, nullptr, 0, nullptr} }; //////////////////////////////////////////////////////////////////////////// @@ -317,7 +331,6 @@ PyMethodDef signature_methods[] = { static int PySide_BuildSignatureArgs(PyObject *obtype_mod, const char *signatures[]) { - init_module_1(); AutoDecRef type_key(GetTypeKey(obtype_mod)); /* * PYSIDE-996: Avoid string overflow in MSVC, which has a limit of @@ -345,7 +358,6 @@ PyObject *PySide_BuildSignatureProps(PyObject *type_key) * We simply pick up the arguments that we stored here and replace * them by the function result. */ - init_module_2(); if (type_key == nullptr) return nullptr; PyObject *numkey = PyDict_GetItem(pyside_globals->arg_dict, type_key); @@ -378,9 +390,7 @@ PyObject *PySide_BuildSignatureProps(PyObject *type_key) #ifdef PYPY_VERSION static bool get_lldebug_flag() { - PyObject *sysmodule = PyImport_AddModule("sys"); - auto *dic = PyModule_GetDict(sysmodule); - dic = PyDict_GetItemString(dic, "pypy_translation_info"); + auto *dic = PySys_GetObject("pypy_translation_info"); int lldebug = PyObject_IsTrue(PyDict_GetItemString(dic, "translation.lldebug")); int lldebug0 = PyObject_IsTrue(PyDict_GetItemString(dic, "translation.lldebug0")); return lldebug || lldebug0; @@ -421,8 +431,6 @@ static int PySide_FinishSignatures(PyObject *module, const char *signatures[]) if (PyCFunction_Check(func)) if (PyDict_SetItem(pyside_globals->map_dict, func, module) < 0) return -1; - if (_finish_nested_classes(obdict) < 0) - return -1; // The finish_import function will not work the first time since phase 2 // was not yet run. But that is ok, because the first import is always for // the shiboken module (or a test module). @@ -444,9 +452,12 @@ static int PySide_FinishSignatures(PyObject *module, const char *signatures[]) int InitSignatureStrings(PyTypeObject *type, const char *signatures[]) { + // PYSIDE-2404: This function now also builds the mapping for static methods. + // It was one missing spot to let Lazy import work. + init_shibokensupport_module(); auto *ob_type = reinterpret_cast<PyObject *>(type); int ret = PySide_BuildSignatureArgs(ob_type, signatures); - if (ret < 0) { + if (ret < 0 || _build_func_to_type(ob_type) < 0) { PyErr_Print(); PyErr_SetNone(PyExc_ImportError); } @@ -463,6 +474,8 @@ void FinishSignatureInitialization(PyObject *module, const char *signatures[]) * Still, it is not possible to call init phase 2 from here, * because the import is still running. Do it from Python! */ + init_shibokensupport_module(); + #ifndef PYPY_VERSION static const bool patch_types = true; #else @@ -526,8 +539,8 @@ static PyObject *adjustFuncName(const char *func_name) // Find the feature flags auto type = reinterpret_cast<PyTypeObject *>(obtype.object()); - auto dict = type->tp_dict; - int id = SbkObjectType_GetReserved(type); + AutoDecRef dict(PepType_GetDict(type)); + int id = currentSelectId(type); id = id < 0 ? 0 : id; // if undefined, set to zero auto lower = id & 0x01; auto is_prop = id & 0x02; @@ -550,33 +563,34 @@ static PyObject *adjustFuncName(const char *func_name) if (prop_name) { auto _prop_name = String::toCString(prop_name); if (is_class_prop) - sprintf(_buf, "%s.__dict__['%s'].fset", _path, _prop_name); + snprintf(_buf, sizeof(_buf), "%s.__dict__['%s'].fset", _path, _prop_name); else - sprintf(_buf, "%s.%s.fset", _path, _prop_name); + snprintf(_buf, sizeof(_buf), "%s.%s.fset", _path, _prop_name); } else { auto _name = String::toCString(name); - sprintf(_buf, "%s.%s", _path, _name); + snprintf(_buf, sizeof(_buf), "%s.%s", _path, _name); } return String::fromCString(_buf); } void SetError_Argument(PyObject *args, const char *func_name, PyObject *info) { + init_shibokensupport_module(); /* * This function replaces the type error construction with extra * overloads parameter in favor of using the signature module. * Error messages are rare, so we do it completely in Python. */ - init_module_1(); - init_module_2(); // PYSIDE-1305: Handle errors set by fillQtProperties. if (PyErr_Occurred()) { PyObject *e, *v, *t; // Note: These references are all borrowed. PyErr_Fetch(&e, &v, &t); + Py_DECREF(e); info = v; + Py_XDECREF(t); } // PYSIDE-1019: Modify the function name expression according to feature. AutoDecRef new_func_name(adjustFuncName(func_name)); @@ -607,21 +621,19 @@ void SetError_Argument(PyObject *args, const char *func_name, PyObject *info) * But the __doc__ attribute existed already by inheritance, and calling * PyType_Modified() is not supported. So we added the getsets explicitly * to the metatype. + * + * PYSIDE-2101: The __signature__ attribute is gone due to rlcompleter. */ -PyObject *Sbk_TypeGet___signature__(PyObject *ob, PyObject *modifier) -{ - return pyside_tp_get___signature__(ob, modifier); -} - PyObject *Sbk_TypeGet___doc__(PyObject *ob) { + init_shibokensupport_module(); return pyside_tp_get___doc__(ob); } PyObject *GetFeatureDict() { - init_module_1(); + init_shibokensupport_module(); return pyside_globals->feature_dict; } diff --git a/sources/shiboken6/libshiboken/signature/signature_doc.rst b/sources/shiboken6/libshiboken/signature/signature_doc.rst deleted file mode 100644 index 0fb26ae52..000000000 --- a/sources/shiboken6/libshiboken/signature/signature_doc.rst +++ /dev/null @@ -1,376 +0,0 @@ -************************* -The signature C extension -************************* - -This module is a C extension for CPython 3.5 and up, and CPython 2.7. -Its purpose is to provide support for the ``__signature__`` attribute -of builtin PyCFunction objects. - - -Short Introduction to the Topic -=============================== - -Beginning with CPython 3.5, Python functions began to grow a ``__signature__`` -attribute for normal Python functions. This is totally optional and just -a nice-to-have feature in Python. - -PySide, on the other hand, could use ``__signature__`` very much, because the -typing info for the 15000+ PySide functions is really missing, and it -would be nice to have this info directly available. - - -The Idea to Support Signatures -============================== - -We want to have an additional ``__signature__`` attribute in all PySide -methods, without changing lots of generated code. -Therefore, we did not change any of the existing data structures, -but supported the new attribute by a global dictionary. - -When the ``__signature__`` property is requested, a method is called that -does a lookup in the global dict. This is a flexible approach with little impact -to the rest of the project. It has very limited overhead compared to direct -attribute access, but for the need of a signature access from time to time, -this is an adequate compromise. - - -How this Code Works -------------------- - -Signatures are supported for regular Python functions, only. Creating signatures -for ``PyCFunction`` objects would require quite some extra effort in Python. - -Fortunately, we found this special *stealth* technique, that saves us most of the -needed effort: - -The basic idea is to create a dummy Python function with **varnames**, **defaults** -and **annotations** properties, and then to use the inspect -module to create a signature object. This object is returned as the computed -result of the ``__signature__`` attribute of the real ``PyCFunction`` object. - -There is one thing that really changes Python a bit: - -* We added the ``__signature__`` attribute to every function. - -That is a little change to Python that does not harm, but it saves us -tons of code, that was needed in the early versions of the module. - -The internal work is done in two steps: - -* All functions of a class get the *signature text* when the module is imported. - This is only a very small overhead added to the startup time. It is a single - string for each whole class. -* The actual signature object is created later, when the attribute is really - requested. Signatures are cached and only created on first access. - -Example: - -The ``PyCFunction`` ``QtWidgets.QApplication.palette`` is interrogated for its -signature. That means ``pyside_sm_get___signature__()`` is called. -It calls ``GetSignature_Function`` which returns the signature if it is found. - - -Why this Code is Fast ---------------------- - -It costs a little time (maybe 6 seconds) to run through every single signature -object, since these are more than 25000 Python objects. But all the signature -objects will be rarely accessed but in special applications. -The normal case are only a few accesses, and these are working pretty fast. - -The key to make this signature module fast is to avoid computation as much as -possible. When no signature objects are used, then almost no time is lost in -initialization. Only the above mentioned strings and some support modules are -additionally loaded on ``import PySide6``. -When it comes to signature usage, then late initialization is used and cached. -This technique is also known as *full laziness* in haskell. - -There are actually two locations where late initialization occurs: - -* ``dict`` can be no dict but a tuple. That is the initial argument tuple that - was saved by ``PySide_BuildSignatureArgs`` at module load time. - If so, then ``pyside_type_init`` in parser.py will be called, - which parses the string and creates the dict. -* ``props`` can be empty. Then ``create_signature`` in loader.py - is called, which uses a dummy function to produce a signature instance - with the inspect module. - -The initialization that is always done is just two dictionary writes -per class, and we have about 1000 classes. -To measure the additional overhead, we have simulated what happens -when ``from PySide6 import *`` is performed. -It turned out that the overhead is below 0.5 ms. - - -The Signature Package Structure -------------------------------- - -The C++ code involved with the signature module is completely in the file -shiboken6/libshiboken/signature.cpp . All other functionality is implemented in -the ``signature`` Python package. It has the following structure:: - - shiboken6/files.dir/shibokensupport/ - backport_inspect.py - - signature/ - loader.py - parser.py - mapping.py - errorhandler.py - layout.py - - lib/ - enum_sig.py - tool.py - - - -Really important are the **parser**, **mapping**, **errorhandler**, **enum_sig**, -**layout** and **loader** modules. The rest is needed to create Python 2 compatibility -or be compatible with embedding and installers. - - -loader.py -~~~~~~~~~ - -This module assembles and imports the ``inspect`` module, and then exports the -``create_signature`` function. This function takes a fake function and some -attributes and builds a ``__signature__`` object with the inspect module. - - -parser.py -~~~~~~~~~ - -This module takes a class signatures string from C++ and parses it into the -needed properties for the ``create_signature`` function. Its entry point is the -``pyside_type_init`` function, which is called from the C module via ``loader.py``. - - -mapping.py -~~~~~~~~~~ - -The purpose of the mapping module is maintaining a list of replacement strings -that map from the *signature text* in C to the property strings that Python -needs. A lot of mappings are resolved by rather complex expressions in ``parser.py``, -but a few hundred cases are better to spell explicitly, here. - - -errorhandler.py -~~~~~~~~~~~~~~~ - -Since ``Qt For Python 5.12``, we no longer use the builtin type error messages from C++. -Instead, we get much better results with the signature module. At the same time, -this enforced supporting shiboken as well, and the signature module was no longer -optional. - - -enum_sig.py -~~~~~~~~~~~ - -The diverse applications of the signature module all needed to iterate over modules, -classes and functions. In order to centralize this enumeration, the process has -been factored out as a context manager. The user has only to supply functions -that do the actual formatting. - -See for example the .pyi generator ``pyside6/PySide6/support/generate_pyi.py``. - - -layout.py -~~~~~~~~~ - -As more applications used the signature module, different formatting of signatures -was needed. To support that, we created the function ``create_signature``, which -has a parameter to choose from some predefined layouts. - - -*typing27.py* -~~~~~~~~~~~~~ - -Python 2 has no typing module at all. This is a backport of the minimum that is needed. - - -*backport_inspect.py* -~~~~~~~~~~~~~~~~~~~~~ - -Python 2 has an inspect module, but lacks the signature functions, completely. -This module adds the missing functionality, which is merged at runtime into -the inspect module. - - -Multiple Arities ----------------- - -One aspect that was ignored so far was *multiple arities*: How to handle it when -a function has more than one signature? - -I did not find any note on how multiple signatures should be treated in Python, -but this simple rules seem to work well: - -* If there is a list, then it is a multi-signature. -* Otherwise, it is a simple signature. - - -Impacts of The Signature Module -=============================== - -The signature module has a number of impacts to other PySide modules, which were -created as a consequence of its existence, and there will be a few more in the -future: - - -existence_test.py ------------------ - -The file ``pyside6/tests/registry/existence_test.py`` was written using the -signatures from the signatures module. The idea is that there are some 15000 -functions with a certain signature. - -These functions should not get lost by some bad check-in. Therefore, a list -of all existing signatures is kept as a module that assembles a -dictionary. The function existence is checked, and also the exact arity. - -This module exists for every PySide release and every platform. The initial -module is generated once and saved as ``exists_{plat}_{version}.py``. - -An error is normally only reported as a warning, but: - - -Interaction With The Coin Module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When this test program is run in COIN, then the warnings are turned into -errors. The reason is that only in COIN, we have a stable configuration -of PySide modules that can reliably be compared. - -These modules have the name ``exists_{platf}_{version}_ci.py``, and as a big -exception for generated code, these files are *intentionally* checked in. - - -What Happens When a List is Missing? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When a new version of PySide gets created, then the existence test files -initially do not exist. - -When a COIN test is run, then it will complain about the error and create -the missing module on standard output. -But since COIN tests are run multiple times, the output that was generated -by the first test will still exist at the subsequent runs. -(If COIN was properly implemented, we could not take that advantage and -would need to implement that as an extra exception.) - -As a result, a missing module will be reported as a test which partially -succeeded (called "FLAKY"). To avoid further flaky tests and to activate as a real test, -we can now capture the error output of COIN and check the generated module -in. - - -Explicitly Enforcing Recreation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The former way to regenerate the registry files was to remove the files -and check that in. This has the desired effect, but creates huge deltas. -As a more efficient way, we have prepared a comment in the first line -that contains the word "recreate". -By uncommenting this line, a NameError is triggered, which has the same -effect. - - -init_platform.py -~~~~~~~~~~~~~~~~ - -For generating the ``exists_{platf}_{version}`` modules, the module -``pyside6/tests/registry/init_platform.py`` was written. It can be used -standalone from the commandline, to check the compatibility of some -changes, directly. - - -scrape_testresults.py ---------------------- - -To simplify and automate the process of extracting the ``exists_{platf}_{version}_ci.py`` -files, the script ``pyside6/tests/registry/scrape_testresults.py`` has been written. - -This script scans the whole testresults website for PySide, that is:: - - https://testresults.qt.io/coin/api/results/pyside/pyside-setup/ - -On the first scan, the script runs less than 30 minutes. After that, a cache -is generated and the scan works *much* faster. The test results are placed -into the folder ``pyside6/tests/registry/testresults/embedded/`` with a -unique name that allows for easy sorting. Example:: - - testresults/embedded/2018_09_10_10_40_34-test_1536891759-exists_linux_5_11_2_ci.py - -These files are created only once. If they already exist, they are not touched, again. -The file `pyside6/tests/registry/known_urls.json`` holds all scanned URLs after -a successful scan. The ``testresults/embedded`` folder can be kept for reference -or can be removed. Important is only the json file. - -The result of a scan is then directly placed into the ``pyside6/tests/registry/`` -folder. It should be reviewed and then eventually checked in. - - -generate_pyi.py ---------------- - -``pyside6/PySide6/support/generate_pyi.py`` is still under development. -This module generates so-called hinting stubs for integration of PySide -with diverse *Python IDEs*. - -Although this module creates the stubs as an add-on, the -impact on the quality of the signature module is considerable: - -The module must create syntactically correct ``.pyi`` files which contain -not only signatures but also constants and enums of all PySide modules. -This serves as an extra challenge that has a very positive effect on -the completeness and correctness of signatures. - -The module has a ``--feature`` option to generate modified .pyi files. -A shortcut for this command is ``pyside6-genpyi``. - -A useful command to change all .pyi files to use all features is - -.. code-block:: python - - pyside6-genpyi all --feature snake_case true_property - - -pyi_generator.py ----------------- - -``shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py`` -has been extracted from ``generate_pyi.py``. It allows the generation of ``.pyi`` -files from arbitrary extension modules created with shiboken. - -A shortcut for this command is ``shiboken6-genpyi``. - - -Current Extensions ------------------- - -Before the signature module was written, there already existed the concept of -signatures, but in a more C++ - centric way. From that time, there existed -the error messages, which are created when a function gets wrong argument types. - -These error messages were replaced by text generated on demand by -the signature module, in order to be more consistent and correct. -This was implemented in ``Qt For Python 5.12.0``. - -Additionally, the ``__doc__`` attribute of PySide methods was not set. -It was easy to get a nice ``help()`` feature by creating signatures -as default content for docstrings. -This was implemented in ``Qt For Python 5.12.1``. - - -Literature -========== - - `PEP 362 – Function Signature Object <https://www.python.org/dev/peps/pep-0362/>`__ - - `PEP 484 – Type Hints <https://www.python.org/dev/peps/pep-0484/>`__ - - `PEP 3107 – Function Annotations <https://www.python.org/dev/peps/pep-3107/>`__ - - -*Personal Remark: This module is dedicated to our lovebird "Püppi", who died on 2017-09-15.* diff --git a/sources/shiboken6/libshiboken/signature/signature_extend.cpp b/sources/shiboken6/libshiboken/signature/signature_extend.cpp index 047b0e0e4..7292f8216 100644 --- a/sources/shiboken6/libshiboken/signature/signature_extend.cpp +++ b/sources/shiboken6/libshiboken/signature/signature_extend.cpp @@ -58,32 +58,26 @@ static PyObject *_get_written_signature(signaturefunc sf, PyObject *ob, PyObject #ifdef PYPY_VERSION PyObject *pyside_bm_get___signature__(PyObject *func, PyObject *modifier) { - init_module_2(); return _get_written_signature(GetSignature_Method, func, modifier); } #endif PyObject *pyside_cf_get___signature__(PyObject *func, PyObject *modifier) { - init_module_2(); return _get_written_signature(GetSignature_Function, func, modifier); } PyObject *pyside_sm_get___signature__(PyObject *sm, PyObject *modifier) { - init_module_2(); AutoDecRef func(PyObject_GetAttr(sm, PyMagicName::func())); - if (Py_TYPE(func) == PepFunction_TypePtr) - return PyObject_GetAttr(func, PyMagicName::signature()); return _get_written_signature(GetSignature_Function, func, modifier); } PyObject *pyside_md_get___signature__(PyObject *ob_md, PyObject *modifier) { - init_module_2(); AutoDecRef func(name_key_to_func(ob_md)); if (func.object() == Py_None) - return Py_None; + Py_RETURN_NONE; if (func.isNull()) Py_FatalError("missing mapping in MethodDescriptor"); return pyside_cf_get___signature__(func, modifier); @@ -91,13 +85,11 @@ PyObject *pyside_md_get___signature__(PyObject *ob_md, PyObject *modifier) PyObject *pyside_wd_get___signature__(PyObject *ob, PyObject *modifier) { - init_module_2(); return _get_written_signature(GetSignature_Wrapper, ob, modifier); } PyObject *pyside_tp_get___signature__(PyObject *obtype_mod, PyObject *modifier) { - init_module_2(); return _get_written_signature(GetSignature_TypeMod, obtype_mod, modifier); } @@ -127,17 +119,17 @@ static int handle_doc_in_progress = 0; static PyObject *handle_doc(PyObject *ob, PyObject *old_descr) { - init_module_1(); - init_module_2(); AutoDecRef ob_type_mod(GetClassOrModOf(ob)); const char *name; - if (PyModule_Check(ob_type_mod.object())) + bool isModule = PyModule_Check(ob_type_mod.object()); + if (isModule) name = PyModule_GetName(ob_type_mod.object()); else name = reinterpret_cast<PyTypeObject *>(ob_type_mod.object())->tp_name; PyObject *res{}; - if (handle_doc_in_progress || name == nullptr || strncmp(name, "PySide6.", 8) != 0) { + if (handle_doc_in_progress || name == nullptr + || (isModule && strncmp(name, "PySide6.", 8) != 0)) { res = PyObject_CallMethodObjArgs(old_descr, PyMagicName::get(), ob, nullptr); } else { handle_doc_in_progress++; @@ -177,59 +169,29 @@ static PyObject *pyside_wd_get___doc__(PyObject *wd) return handle_doc(wd, old_wd_doc_descr); } -// the default setter for all objects -static int pyside_set___signature__(PyObject *op, PyObject *value) -{ - // By this additional check, this function refuses write access. - // We consider both nullptr and Py_None as not been written. - AutoDecRef has_val(get_signature_intern(op, nullptr)); - if (!(has_val.isNull() || has_val == Py_None)) { - PyErr_Format(PyExc_AttributeError, - "Attribute '__signature__' of '%.50s' object is not writable", - Py_TYPE(op)->tp_name); - return -1; - } - int ret = value == nullptr ? PyDict_DelItem(pyside_globals->value_dict, op) - : PyDict_SetItem(pyside_globals->value_dict, op, value); - Py_XINCREF(value); - return ret; -} - // PYSIDE-535: We cannot patch types easily in PyPy. // Let's use the `get_signature` function, instead. static PyGetSetDef new_PyCFunction_getsets[] = { {const_cast<char *>("__doc__"), reinterpret_cast<getter>(pyside_cf_get___doc__), nullptr, nullptr, nullptr}, - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(pyside_cf_get___signature__), - reinterpret_cast<setter>(pyside_set___signature__), - nullptr, nullptr}, {nullptr, nullptr, nullptr, nullptr, nullptr} }; static PyGetSetDef new_PyStaticMethod_getsets[] = { {const_cast<char *>("__doc__"), reinterpret_cast<getter>(pyside_sm_get___doc__), nullptr, nullptr, nullptr}, - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(pyside_sm_get___signature__), - reinterpret_cast<setter>(pyside_set___signature__), - nullptr, nullptr}, {nullptr, nullptr, nullptr, nullptr, nullptr} }; static PyGetSetDef new_PyMethodDescr_getsets[] = { {const_cast<char *>("__doc__"), reinterpret_cast<getter>(pyside_md_get___doc__), nullptr, nullptr, nullptr}, - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(pyside_md_get___signature__), - reinterpret_cast<setter>(pyside_set___signature__), - nullptr, nullptr}, {nullptr, nullptr, nullptr, nullptr, nullptr} }; static PyGetSetDef new_PyWrapperDescr_getsets[] = { {const_cast<char *>("__doc__"), reinterpret_cast<getter>(pyside_wd_get___doc__), nullptr, nullptr, nullptr}, - {const_cast<char *>("__signature__"), reinterpret_cast<getter>(pyside_wd_get___signature__), - reinterpret_cast<setter>(pyside_set___signature__), - nullptr, nullptr}, {nullptr, nullptr, nullptr, nullptr, nullptr} }; diff --git a/sources/shiboken6/libshiboken/signature/signature_globals.cpp b/sources/shiboken6/libshiboken/signature/signature_globals.cpp index 71f3d36ae..3a79a12d5 100644 --- a/sources/shiboken6/libshiboken/signature/signature_globals.cpp +++ b/sources/shiboken6/libshiboken/signature/signature_globals.cpp @@ -28,26 +28,13 @@ static const unsigned char PySide_SignatureLoader[] = { #include "embed/signature_bootstrap_inc.h" }; -static PyObject *_init_pyside_extension(PyObject * /* self */, PyObject * /* args */) +static safe_globals_struc *init_phase_1() { - init_module_1(); - init_module_2(); - Py_RETURN_NONE; -} - -// This function will be inserted into __builtins__. -static PyMethodDef init_methods[] = { - {"_init_pyside_extension", (PyCFunction)_init_pyside_extension, METH_NOARGS}, - {nullptr, nullptr} -}; - -static safe_globals_struc *init_phase_1(PyMethodDef *init_meth) -{ - { + do { auto *p = reinterpret_cast<safe_globals_struc *> (malloc(sizeof(safe_globals_struc))); if (p == nullptr) - goto error; + break; /* * Initializing module signature_bootstrap. * Since we now have an embedding script, we can do this without any @@ -57,23 +44,21 @@ static safe_globals_struc *init_phase_1(PyMethodDef *init_meth) // We must work for multiple versions or we are cross-building for a different // Python version interpreter, so use source code. #else - AutoDecRef marshal_module(PyImport_Import(PyName::marshal())); - if (marshal_module.isNull()) - goto error; + AutoDecRef marshal_module(PyImport_Import(PyName::marshal())); // builtin AutoDecRef loads(PyObject_GetAttr(marshal_module, PyName::loads())); if (loads.isNull()) - goto error; + break; #endif char *bytes_cast = reinterpret_cast<char *>( const_cast<unsigned char *>(PySide_SignatureLoader)); AutoDecRef bytes(PyBytes_FromStringAndSize(bytes_cast, sizeof(PySide_SignatureLoader))); if (bytes.isNull()) - goto error; + break; #if defined(Py_LIMITED_API) || defined(SHIBOKEN_NO_EMBEDDING_PYC) PyObject *builtins = PyEval_GetBuiltins(); PyObject *compile = PyDict_GetItem(builtins, PyName::compile()); if (compile == nullptr) - goto error; + break; AutoDecRef code_obj(PyObject_CallFunction(compile, "Oss", bytes.object(), "signature_bootstrap.py", "exec")); #else @@ -81,69 +66,62 @@ static safe_globals_struc *init_phase_1(PyMethodDef *init_meth) loads, bytes.object(), nullptr)); #endif if (code_obj.isNull()) - goto error; + break; p->helper_module = PyImport_ExecCodeModule("signature_bootstrap", code_obj); if (p->helper_module == nullptr) - goto error; + break; // Initialize the module PyObject *mdict = PyModule_GetDict(p->helper_module); if (PyDict_SetItem(mdict, PyMagicName::builtins(), PyEval_GetBuiltins()) < 0) - goto error; - /* - * Unpack an embedded ZIP file with more signature modules. + break; + + /********************************************************************* + * + * Attention! + * ---------- + * + * We are unpacking an embedded ZIP file with more signature modules. * They will be loaded later with the zipimporter. - * Due to MSVC's limitation to 64k strings, we need to assemble pieces. + * The file `signature_bootstrap.py` does the unpacking and starts the + * loader. See `init_phase_2`. + * + * Due to MSVC's limitation to 64k strings, we needed to assemble pieces. */ auto **block_ptr = reinterpret_cast<const char **>(PySide_CompressedSignaturePackage); - int npieces = 0; - PyObject *piece, *zipped_string_sequence = PyList_New(0); - if (zipped_string_sequence == nullptr) - return nullptr; + PyObject *piece{}; + AutoDecRef zipped_string_sequence(PyList_New(0)); for (; **block_ptr != 0; ++block_ptr) { - npieces++; // we avoid the string/unicode dilemma by not using PyString_XXX: piece = Py_BuildValue("s", *block_ptr); if (piece == nullptr || PyList_Append(zipped_string_sequence, piece) < 0) - goto error; + break; } if (PyDict_SetItemString(mdict, "zipstring_sequence", zipped_string_sequence) < 0) - goto error; - Py_DECREF(zipped_string_sequence); + break; // build a dict for diverse mappings p->map_dict = PyDict_New(); - if (p->map_dict == nullptr) - goto error; // build a dict for the prepared arguments p->arg_dict = PyDict_New(); - if (p->arg_dict == nullptr - || PyObject_SetAttrString(p->helper_module, "pyside_arg_dict", p->arg_dict) < 0) - goto error; + if (PyObject_SetAttrString(p->helper_module, "pyside_arg_dict", p->arg_dict) < 0) + break; // build a dict for assigned signature values p->value_dict = PyDict_New(); - if (p->value_dict == nullptr) - goto error; // PYSIDE-1019: build a __feature__ dict p->feature_dict = PyDict_New(); - if (p->feature_dict == nullptr - || PyObject_SetAttrString(p->helper_module, "pyside_feature_dict", p->feature_dict) < 0) - goto error; + if (PyObject_SetAttrString(p->helper_module, "pyside_feature_dict", p->feature_dict) < 0) + break; // This function will be disabled until phase 2 is done. p->finish_import_func = nullptr; - // Initialize the explicit init function. - AutoDecRef init(PyCFunction_NewEx(init_meth, nullptr, nullptr)); - if (init.isNull() - || PyDict_SetItemString(PyEval_GetBuiltins(), init_meth->ml_name, init) != 0) - goto error; - return p; - } -error: + + } while (0); + PyErr_Print(); Py_FatalError("could not initialize part 1"); return nullptr; @@ -151,7 +129,7 @@ error: static int init_phase_2(safe_globals_struc *p, PyMethodDef *methods) { - { + do { PyMethodDef *ml; // The single function to be called, but maybe more to come. @@ -159,7 +137,7 @@ static int init_phase_2(safe_globals_struc *p, PyMethodDef *methods) PyObject *v = PyCFunction_NewEx(ml, nullptr, nullptr); if (v == nullptr || PyObject_SetAttrString(p->helper_module, ml->ml_name, v) != 0) - goto error; + break; Py_DECREF(v); } // The first entry is __feature_import__, add documentation. @@ -170,33 +148,59 @@ static int init_phase_2(safe_globals_struc *p, PyMethodDef *methods) PyObject *bootstrap_func = PyObject_GetAttrString(p->helper_module, "bootstrap"); if (bootstrap_func == nullptr) - goto error; - // The return value of the bootstrap function is the loader module. - PyObject *loader = PyObject_CallFunction(bootstrap_func, "()"); + break; + + /********************************************************************* + * + * Attention! + * ---------- + * + * This is the entry point where everything in folder + * `shibokensupport` becomes initialized. It starts with + * `signature_bootstrap.py` and continues from there to `loader.py`. + * + * The return value of the bootstrap function is the loader module. + */ + PyObject *loader = PyObject_CallFunctionObjArgs(bootstrap_func, nullptr); if (loader == nullptr) - goto error; + break; + // now the loader should be initialized p->pyside_type_init_func = PyObject_GetAttrString(loader, "pyside_type_init"); if (p->pyside_type_init_func == nullptr) - goto error; + break; p->create_signature_func = PyObject_GetAttrString(loader, "create_signature"); if (p->create_signature_func == nullptr) - goto error; + break; p->seterror_argument_func = PyObject_GetAttrString(loader, "seterror_argument"); if (p->seterror_argument_func == nullptr) - goto error; + break; p->make_helptext_func = PyObject_GetAttrString(loader, "make_helptext"); if (p->make_helptext_func == nullptr) - goto error; + break; p->finish_import_func = PyObject_GetAttrString(loader, "finish_import"); if (p->finish_import_func == nullptr) - goto error; + break; p->feature_import_func = PyObject_GetAttrString(loader, "feature_import"); if (p->feature_import_func == nullptr) - goto error; + break; + p->feature_imported_func = PyObject_GetAttrString(loader, "feature_imported"); + if (p->feature_imported_func == nullptr) + break; + + // We call stuff like the feature initialization late, + // after all the function pointers are in place. + PyObject *post_init_func = PyObject_GetAttrString(loader, "post_init"); + if (post_init_func == nullptr) + break; + PyObject *ret = PyObject_CallFunctionObjArgs(post_init_func, nullptr); + if (ret == nullptr) + break; + return 0; - } -error: + + } while (0); + PyErr_Print(); Py_FatalError("could not initialize part 2"); return -1; @@ -205,12 +209,12 @@ error: #ifndef _WIN32 //////////////////////////////////////////////////////////////////////////// // a stack trace for linux-like platforms -#include <stdio.h> +#include <cstdio> #if defined(__GLIBC__) # include <execinfo.h> #endif #include <signal.h> -#include <stdlib.h> +#include <cstdlib> #include <unistd.h> static void handler(int sig) { @@ -223,7 +227,7 @@ static void handler(int sig) { // print out all the frames to stderr #endif - fprintf(stderr, "Error: signal %d:\n", sig); + std::fprintf(stderr, "Error: signal %d:\n", sig); #if defined(__GLIBC__) backtrace_symbols_fd(array, size, STDERR_FILENO); #endif @@ -233,14 +237,14 @@ static void handler(int sig) { //////////////////////////////////////////////////////////////////////////// #endif // _WIN32 -safe_globals pyside_globals = nullptr; +safe_globals_struc *pyside_globals = nullptr; -void init_module_1(void) +void init_shibokensupport_module(void) { static int init_done = 0; if (!init_done) { - pyside_globals = init_phase_1(init_methods); + pyside_globals = init_phase_1(); if (pyside_globals != nullptr) init_done = 1; @@ -251,17 +255,6 @@ void init_module_1(void) signal(SIGSEGV, handler); // install our handler #endif // _WIN32 - } -} - -void init_module_2(void) -{ - static int init_done = 0; - - if (!init_done) { - // Phase 2 will call __init__.py which touches a signature, itself. - // Therefore we set init_done prior to init_phase_2(). - init_done = 1; init_phase_2(pyside_globals, signature_methods); // Enum must be initialized when signatures exist, not earlier. init_enum(); diff --git a/sources/shiboken6/libshiboken/signature/signature_helper.cpp b/sources/shiboken6/libshiboken/signature/signature_helper.cpp index ef0c021d5..cf84cfa13 100644 --- a/sources/shiboken6/libshiboken/signature/signature_helper.cpp +++ b/sources/shiboken6/libshiboken/signature/signature_helper.cpp @@ -51,10 +51,13 @@ int add_more_getsets(PyTypeObject *type, PyGetSetDef *gsp, PyObject **doc_descr) /* * This function is used to assign a new `__signature__` attribute, * and also to override a `__doc__` or `__name__` attribute. + * + * PYSIDE-2101: The __signature__ attribute is gone due to rlcompleter. */ assert(PyType_Check(type)); PyType_Ready(type); - PyObject *dict = type->tp_dict; + AutoDecRef tpDict(PepType_GetDict(type)); + auto *dict = tpDict.object(); for (; gsp->name != nullptr; gsp++) { PyObject *have_descr = PyDict_GetItemString(dict, gsp->name); if (have_descr != nullptr) { @@ -291,7 +294,7 @@ PyObject *_address_to_stringlist(PyObject *numkey) return res_list; } -static int _build_func_to_type(PyObject *obtype) +int _build_func_to_type(PyObject *obtype) { /* * There is no general way to directly get the type of a static method. @@ -307,7 +310,17 @@ static int _build_func_to_type(PyObject *obtype) * We also check for hidden methods, see below. */ auto *type = reinterpret_cast<PyTypeObject *>(obtype); - PyObject *dict = type->tp_dict; + AutoDecRef tpDict(PepType_GetDict(type)); + auto *dict = tpDict.object(); + + // PYSIDE-2404: Get the original dict for late initialization. + // The dict might have been switched before signature init. + static const auto *pyTypeType_tp_dict = PepType_GetDict(&PyType_Type); + if (Py_TYPE(dict) != Py_TYPE(pyTypeType_tp_dict)) { + tpDict.reset(PyObject_GetAttr(dict, PyName::orig_dict())); + dict = tpDict.object(); + } + PyMethodDef *meth = type->tp_methods; if (meth == nullptr) @@ -373,26 +386,4 @@ static int _build_func_to_type(PyObject *obtype) return 0; } -int _finish_nested_classes(PyObject *obdict) -{ - PyObject *key, *value, *obtype; - PyTypeObject *subtype; - Py_ssize_t pos = 0; - - if (obdict == nullptr) - return -1; - while (PyDict_Next(obdict, &pos, &key, &value)) { - if (PyType_Check(value)) { - obtype = value; - if (_build_func_to_type(obtype) < 0) - return -1; - // now continue with nested cases - subtype = reinterpret_cast<PyTypeObject *>(obtype); - if (_finish_nested_classes(subtype->tp_dict) < 0) - return -1; - } - } - return 0; -} - } // extern "C" diff --git a/sources/shiboken6/libshiboken/signature_p.h b/sources/shiboken6/libshiboken/signature_p.h index 99e339ce2..d0c4ee537 100644 --- a/sources/shiboken6/libshiboken/signature_p.h +++ b/sources/shiboken6/libshiboken/signature_p.h @@ -10,7 +10,7 @@ extern "C" { // signature_globals.cpp -typedef struct safe_globals_struc { +struct safe_globals_struc { // init part 1: get arg_dict PyObject *helper_module; PyObject *arg_dict; @@ -24,13 +24,13 @@ typedef struct safe_globals_struc { PyObject *make_helptext_func; PyObject *finish_import_func; PyObject *feature_import_func; -} safe_globals_struc, *safe_globals; + PyObject *feature_imported_func; +}; -extern safe_globals pyside_globals; +extern safe_globals_struc *pyside_globals; extern PyMethodDef signature_methods[]; -void init_module_1(void); -void init_module_2(void); +void init_shibokensupport_module(void); // signature.cpp @@ -63,6 +63,7 @@ PyObject *_get_class_of_cf(PyObject *ob_cf); PyObject *_get_class_of_sm(PyObject *ob_sm); PyObject *_get_class_of_descr(PyObject *ob); PyObject *_address_to_stringlist(PyObject *numkey); +int _build_func_to_type(PyObject *obtype); int _finish_nested_classes(PyObject *dict); #ifdef PYPY_VERSION diff --git a/sources/shiboken6/libshiboken/voidptr.cpp b/sources/shiboken6/libshiboken/voidptr.cpp index c047a8b7c..7045b08b1 100644 --- a/sources/shiboken6/libshiboken/voidptr.cpp +++ b/sources/shiboken6/libshiboken/voidptr.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "voidptr.h" +#include "pep384ext.h" #include "sbkconverter.h" #include "basewrapper.h" #include "basewrapper_p.h" @@ -10,22 +11,21 @@ extern "C" { // Void pointer object definition. -typedef struct { +struct SbkVoidPtrObject { PyObject_HEAD void *cptr; Py_ssize_t size; bool isWritable; -} SbkVoidPtrObject; +}; -PyObject *SbkVoidPtrObject_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +PyObject *SbkVoidPtrObject_new(PyTypeObject *type, PyObject * /* args */, PyObject * /* kwds */) { // PYSIDE-560: It is much safer to first call a function and then do a // type cast than to do everything in one line. The bad construct looked // like this, actual call forgotten: // SbkVoidPtrObject *self = // reinterpret_cast<SbkVoidPtrObject *>(type->tp_alloc); - PyObject *ob = type->tp_alloc(type, 0); - auto *self = reinterpret_cast<SbkVoidPtrObject *>(ob); + auto *self = PepExt_TypeCallAlloc<SbkVoidPtrObject>(type, 0); if (self != nullptr) { self->cptr = nullptr; @@ -153,7 +153,7 @@ PyObject *SbkVoidPtrObject_int(PyObject *v) return PyLong_FromVoidPtr(sbkObject->cptr); } -PyObject *toBytes(PyObject *self, PyObject *args) +PyObject *toBytes(PyObject *self, PyObject * /* args */) { auto *sbkObject = reinterpret_cast<SbkVoidPtrObject *>(self); if (sbkObject->size < 0) { @@ -167,8 +167,8 @@ PyObject *toBytes(PyObject *self, PyObject *args) } static struct PyMethodDef SbkVoidPtrObject_methods[] = { - {"toBytes", toBytes, METH_NOARGS}, - {nullptr} + {"toBytes", toBytes, METH_NOARGS, nullptr}, + {nullptr, nullptr, 0, nullptr} }; static Py_ssize_t SbkVoidPtrObject_length(PyObject *v) @@ -256,38 +256,42 @@ static PyBufferProcs SbkVoidPtrObjectBufferProc = { (releasebufferproc)nullptr // bf_releasebuffer }; -// Void pointer type definition. -static PyType_Slot SbkVoidPtrType_slots[] = { - {Py_tp_repr, reinterpret_cast<void *>(SbkVoidPtrObject_repr)}, - {Py_nb_int, reinterpret_cast<void *>(SbkVoidPtrObject_int)}, - {Py_sq_length, reinterpret_cast<void *>(SbkVoidPtrObject_length)}, - {Py_tp_str, reinterpret_cast<void *>(SbkVoidPtrObject_str)}, - {Py_tp_richcompare, reinterpret_cast<void *>(SbkVoidPtrObject_richcmp)}, - {Py_tp_init, reinterpret_cast<void *>(SbkVoidPtrObject_init)}, - {Py_tp_new, reinterpret_cast<void *>(SbkVoidPtrObject_new)}, - {Py_tp_dealloc, reinterpret_cast<void *>(Sbk_object_dealloc)}, - {Py_tp_methods, reinterpret_cast<void *>(SbkVoidPtrObject_methods)}, - {0, nullptr} -}; -static PyType_Spec SbkVoidPtrType_spec = { - "2:shiboken6.Shiboken.VoidPtr", - sizeof(SbkVoidPtrObject), - 0, - Py_TPFLAGS_DEFAULT, - SbkVoidPtrType_slots, -}; - - +static PyTypeObject *createVoidPtrType() +{ + PyType_Slot SbkVoidPtrType_slots[] = { + {Py_tp_repr, reinterpret_cast<void *>(SbkVoidPtrObject_repr)}, + {Py_nb_int, reinterpret_cast<void *>(SbkVoidPtrObject_int)}, + {Py_sq_length, reinterpret_cast<void *>(SbkVoidPtrObject_length)}, + {Py_tp_str, reinterpret_cast<void *>(SbkVoidPtrObject_str)}, + {Py_tp_richcompare, reinterpret_cast<void *>(SbkVoidPtrObject_richcmp)}, + {Py_tp_init, reinterpret_cast<void *>(SbkVoidPtrObject_init)}, + {Py_tp_new, reinterpret_cast<void *>(SbkVoidPtrObject_new)}, + {Py_tp_dealloc, reinterpret_cast<void *>(Sbk_object_dealloc)}, + {Py_tp_methods, reinterpret_cast<void *>(SbkVoidPtrObject_methods)}, + {0, nullptr} + }; + + PyType_Spec SbkVoidPtrType_spec = { + "2:shiboken6.Shiboken.VoidPtr", + sizeof(SbkVoidPtrObject), + 0, + Py_TPFLAGS_DEFAULT, + SbkVoidPtrType_slots, + }; + + return SbkType_FromSpec_BMDWB(&SbkVoidPtrType_spec, + nullptr, nullptr, 0, 0, + &SbkVoidPtrObjectBufferProc); } PyTypeObject *SbkVoidPtr_TypeF(void) { - static PyTypeObject *type = SbkType_FromSpec_BMDWB(&SbkVoidPtrType_spec, - nullptr, nullptr, 0, 0, - &SbkVoidPtrObjectBufferProc); + static auto *type = createVoidPtrType(); return type; } +} // extern "C" + namespace VoidPtr { static int voidPointerInitialized = false; |