PEP 697: Use the new type extension provision

By Python 3.12, there is now an official way to extend
heap types by custom extra data.

When we supported PyPy, the old type extension of PySide
did no longer work, and we introduced shadow dicts.
With the interface found in Python 3.12, we can use direct
extended data, again.

The supporting structures are not Limited API compatible.
We implemented a patch that enables this anyway, but it
is valid for this version only without a new review.

NOTE: The documentation lists `PyObject_GetTypeData` as
Limited API since Version 3.12, but in fact we had to
write a cheating patch.

[ChangeLog][PySide6] Hidden Type Extensions according to
PEP 697 are now used instead of shadow dictionaries.

Change-Id: I4b724ba7bcc72470b13f55ea5ebf94666061420d
Task-number: PYSIDE-2230
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

5 files changed, 126 insertions, 19 deletions

diff --git a/sources/pyside6/libpyside/pyside.cpp b/sources/pyside6/libpyside/pyside.cpp
index 891279473..074c289b8 100644
--- a/sources/pyside6/libpyside/pyside.cpp
+++ b/sources/pyside6/libpyside/pyside.cpp
@@ -446,6 +446,8 @@ void initDynamicMetaObject(PyTypeObject *type, const QMetaObject *base, std::siz
 
 TypeUserData *retrieveTypeUserData(PyTypeObject *pyTypeObj)
 {
+    if (!SbkObjectType_Check(pyTypeObj))
+        return nullptr;
     return reinterpret_cast<TypeUserData *>(Shiboken::ObjectType::getTypeUserData(pyTypeObj));
 }
 
diff --git a/sources/shiboken6/libshiboken/basewrapper.cpp b/sources/shiboken6/libshiboken/basewrapper.cpp
index 828c54a89..6244cc395 100644
--- a/sources/shiboken6/libshiboken/basewrapper.cpp
+++ b/sources/shiboken6/libshiboken/basewrapper.cpp
@@ -152,11 +152,21 @@ static PyTypeObject *createObjectTypeType()
         "1:Shiboken.ObjectType",
         static_cast<int>(PyType_Type.tp_basicsize) + 1,           // see above
         0, // sizeof(PyMemberDef), not for PyPy without a __len__ defined
-        Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+        Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_TYPE_SUBCLASS,
         SbkObjectType_Type_slots,
     };
 
-    return SbkType_FromSpec(&SbkObjectType_Type_spec);
+    PyType_Spec SbkObjectType_Type_spec_312 = {
+        "1:Shiboken.ObjectType",
+        -long(sizeof(SbkObjectTypePrivate)),
+        0, // sizeof(PyMemberDef), not for PyPy without a __len__ defined
+        Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_TYPE_SUBCLASS,
+        SbkObjectType_Type_slots,
+    };
+
+    return SbkType_FromSpec(_PepRuntimeVersion() >= 0x030C00 ?
+                            &SbkObjectType_Type_spec_312 :
+                            &SbkObjectType_Type_spec);
 }
 
 PyTypeObject *SbkObjectType_TypeF(void)
@@ -249,12 +259,15 @@ static PyTypeObject *createObjectType()
     //              But before 3.12 is the minimum version, we cannot use the new
     //              function, although we would need this for 3.12 :-D
     //              We do a special patching here that is triggered through Py_None.
-    return SbkType_FromSpec_BMDWB(&SbkObject_Type_spec,
-                                  Py_None,     // bases, special flag!
-                                  SbkObjectType_TypeF(),
-                                  offsetof(SbkObject, ob_dict),
-                                  offsetof(SbkObject, weakreflist),
-                                  nullptr);    // bufferprocs
+    auto *type = SbkType_FromSpec_BMDWB(&SbkObject_Type_spec,
+                                        Py_None,     // bases, spectial flag!
+                                        SbkObjectType_TypeF(),
+                                        offsetof(SbkObject, ob_dict),
+                                        offsetof(SbkObject, weakreflist),
+                                        nullptr);    // bufferprocs
+    // Initialize the hidden data area.
+    _PepPostInit_SbkObject_Type(type);
+    return type;
 }
 
 PyTypeObject *SbkObject_TypeF(void)
@@ -661,10 +674,8 @@ PyObject *FallbackRichCompare(PyObject *self, PyObject *other, int op)
 
 bool SbkObjectType_Check(PyTypeObject *type)
 {
-    static auto *obMeta = reinterpret_cast<PyObject *>(SbkObjectType_TypeF());
-    auto *obType = reinterpret_cast<PyObject *>(type);
-    return obMeta == reinterpret_cast<PyObject *>(Py_TYPE(obType))
-           || PyObject_IsInstance(obType, obMeta);
+    static auto *meta = SbkObjectType_TypeF();
+    return Py_TYPE(type) == meta || PyType_IsSubtype(Py_TYPE(type), meta);
 }
 
 } //extern "C"
@@ -1006,16 +1017,19 @@ introduceWrapperType(PyObject *enclosingObject,
 
 void setSubTypeInitHook(PyTypeObject *type, SubTypeInitHook func)
 {
+    assert(SbkObjectType_Check(type));
     PepType_SOTP(type)->subtype_init = func;
 }
 
 void *getTypeUserData(PyTypeObject *type)
 {
+    assert(SbkObjectType_Check(type));
     return PepType_SOTP(type)->user_data;
 }
 
 void setTypeUserData(PyTypeObject *type, void *userData, DeleteUserDataFunc d_func)
 {
+    assert(SbkObjectType_Check(type));
     auto *sotp = PepType_SOTP(type);
     sotp->user_data = userData;
     sotp->d_func = d_func;
diff --git a/sources/shiboken6/libshiboken/pep384impl.cpp b/sources/shiboken6/libshiboken/pep384impl.cpp
index 4942595ce..171259803 100644
--- a/sources/shiboken6/libshiboken/pep384impl.cpp
+++ b/sources/shiboken6/libshiboken/pep384impl.cpp
@@ -991,33 +991,116 @@ long _PepRuntimeVersion()
  *
  */
 
+///////////////////////////////////////////////////////////////////////
+//
+// PEP 697: Support for embedded type structures.
+//
+// According to `https://docs.python.org/3/c-api/object.html?highlight=pyobject_gettypedata#c.PyObject_GetTypeData`
+// the function `PyObject_GetTypeData` should belong to the Stable API
+// since version 3.12.0, but it does not. We use instead some copies
+// from Python source code.
+
+#if !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030C0000
+
+# define PepObject_GetTypeData PyObject_GetTypeData
+
+SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *type)
+{
+    assert(SbkObjectType_Check(type));
+    auto *obType = reinterpret_cast<PyObject *>(type);
+    void *data = PyObject_GetTypeData(obType, Py_TYPE(obType));
+    return reinterpret_cast<SbkObjectTypePrivate *>(data);
+}
+
+void PepType_SOTP_delete(PyTypeObject * /*type*/)
+{
+}
+
+#else
+
+// The following comments are directly copied from Python 3.12
+//
+
+// Make sure we have maximum alignment, even if the current compiler
+// does not support max_align_t. Note that:
+// - Autoconf reports alignment of unknown types to 0.
+// - 'long double' has maximum alignment on *most* platforms,
+//   looks like the best we can do for pre-C11 compilers.
+// - The value is tested, see test_alignof_max_align_t
+#  if !defined(ALIGNOF_MAX_ALIGN_T) || ALIGNOF_MAX_ALIGN_T == 0
+#    undef ALIGNOF_MAX_ALIGN_T
+#    define ALIGNOF_MAX_ALIGN_T alignof(long double)
+#  endif
+
+/* Align up to the nearest multiple of alignof(max_align_t)
+ * (like _Py_ALIGN_UP, but for a size rather than pointer)
+ */
+static Py_ssize_t _align_up(Py_ssize_t size)
+{
+    return (size + ALIGNOF_MAX_ALIGN_T - 1) & ~(ALIGNOF_MAX_ALIGN_T - 1);
+}
+
+static void *PepObject_GetTypeData(PyObject *obj, PyTypeObject *cls)
+{
+    assert(PyObject_TypeCheck(obj, cls));
+    return reinterpret_cast<char *>(obj) + _align_up(cls->tp_base->tp_basicsize);
+}
+//
+///////////////////////////////////////////////////////////////////////
+
 /*
  * PyTypeObject extender
  */
+
 static std::unordered_map<PyTypeObject *, SbkObjectTypePrivate > SOTP_extender{};
 static thread_local PyTypeObject *SOTP_key{};
 static thread_local SbkObjectTypePrivate *SOTP_value{};
 
-SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *sbkType)
+SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *type)
 {
-    if (sbkType == SOTP_key)
+    static bool use_312 = _PepRuntimeVersion() >= 0x030C00;
+    assert(SbkObjectType_Check(type));
+    if (use_312) {
+        auto *obType = reinterpret_cast<PyObject *>(type);
+        void *data = PepObject_GetTypeData(obType, Py_TYPE(obType));
+        return reinterpret_cast<SbkObjectTypePrivate *>(data);
+    }
+    if (type == SOTP_key)
         return SOTP_value;
-    auto it = SOTP_extender.find(sbkType);
+    auto it = SOTP_extender.find(type);
     if (it == SOTP_extender.end()) {
-        it = SOTP_extender.insert({sbkType, {}}).first;
+        it = SOTP_extender.insert({type, {}}).first;
         memset(&it->second, 0, sizeof(SbkObjectTypePrivate));
     }
-    SOTP_key = sbkType;
+    SOTP_key = type;
     SOTP_value = &it->second;
     return SOTP_value;
 }
 
-void PepType_SOTP_delete(PyTypeObject *sbkType)
+void PepType_SOTP_delete(PyTypeObject *type)
 {
-    SOTP_extender.erase(sbkType);
+    static bool use_312 = _PepRuntimeVersion() >= 0x030C00;
+    assert(SbkObjectType_Check(type));
+    if (use_312)
+        return;
+    SOTP_extender.erase(type);
     SOTP_key = nullptr;
 }
 
+#endif // !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030C0000
+
+void _PepPostInit_SbkObject_Type(PyTypeObject *type)
+{
+    // Special init for SbkObject_Type.
+    // A normal initialization would recurse PepType_SOTP.
+    if (_PepRuntimeVersion() >= 0x030C00) {
+        auto *obType = reinterpret_cast<PyObject *>(type);
+        void *data = PepObject_GetTypeData(obType, Py_TYPE(obType));
+        auto *sbkExt = reinterpret_cast<SbkObjectTypePrivate *>(data);
+        std::fill_n(reinterpret_cast<char *>(data), sizeof(*sbkExt), 0);
+    }
+}
+
 /*
  * SbkEnumType extender
  */
@@ -1027,6 +1110,7 @@ static thread_local SbkEnumTypePrivate *SETP_value{};
 
 SbkEnumTypePrivate *PepType_SETP(SbkEnumType *enumType)
 {
+    // PYSIDE-2230: This makes no sense at all for Enum types.
     if (enumType == SETP_key)
         return SETP_value;
     auto it = SETP_extender.find(enumType);
diff --git a/sources/shiboken6/libshiboken/pep384impl.h b/sources/shiboken6/libshiboken/pep384impl.h
index 0d4082585..dbf3b0c80 100644
--- a/sources/shiboken6/libshiboken/pep384impl.h
+++ b/sources/shiboken6/libshiboken/pep384impl.h
@@ -128,6 +128,9 @@ struct SbkObjectTypePrivate;
 LIBSHIBOKEN_API SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *type);
 LIBSHIBOKEN_API void PepType_SOTP_delete(PyTypeObject *type);
 
+// PYSIDE-2230: SbkObjectType needs a special init
+LIBSHIBOKEN_API void _PepPostInit_SbkObject_Type(PyTypeObject *type);
+
 struct SbkEnumType;
 struct SbkEnumTypePrivate;
 
diff --git a/sources/shiboken6/libshiboken/sbktypefactory.cpp b/sources/shiboken6/libshiboken/sbktypefactory.cpp
index 4699f7c68..079548eed 100644
--- a/sources/shiboken6/libshiboken/sbktypefactory.cpp
+++ b/sources/shiboken6/libshiboken/sbktypefactory.cpp
@@ -77,7 +77,11 @@ static PyObject *_PyType_FromSpecWithBasesHack(PyType_Spec *spec,
         }
     }
 
+#if !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030C0000
+    auto *ret = PyType_FromMetaclass(meta, nullptr /*module*/, spec, bases);
+#else
     auto *ret = _PyType_FromSpecWithBases(spec, bases);
+#endif
 
     if (keepMeta)
         keepMeta->tp_new = keepNew;