diff options
Diffstat (limited to 'sources/shiboken6')
53 files changed, 1393 insertions, 685 deletions
diff --git a/sources/shiboken6/ApiExtractor/abstractmetabuilder.cpp b/sources/shiboken6/ApiExtractor/abstractmetabuilder.cpp index 1acfe1375..29566a272 100644 --- a/sources/shiboken6/ApiExtractor/abstractmetabuilder.cpp +++ b/sources/shiboken6/ApiExtractor/abstractmetabuilder.cpp @@ -1451,12 +1451,12 @@ void AbstractMetaBuilderPrivate::traverseFunctions(const ScopeModelItem& scopeIt } } - if (metaFunction->functionType() == AbstractMetaFunction::ConstructorFunction) { - if (metaFunction->isPrivate()) - metaClass->setHasPrivateConstructor(true); - else - metaClass->setHasNonPrivateConstructor(true); + if (metaFunction->functionType() == AbstractMetaFunction::ConstructorFunction + && metaFunction->isPrivate()) { + metaClass->setHasPrivateConstructor(true); } + if (metaFunction->isConstructor() && !metaFunction->isPrivate()) // Including Copy CT + metaClass->setHasNonPrivateConstructor(true); if (!metaFunction->isDestructor() && !(metaFunction->isPrivate() && metaFunction->functionType() == AbstractMetaFunction::ConstructorFunction)) { @@ -1574,7 +1574,8 @@ bool AbstractMetaBuilderPrivate::setupInheritance(const AbstractMetaClassPtr &me &info, &baseContainerType); if (templ) { setupInheritance(templ); - inheritTemplate(metaClass, templ, info); + if (!inheritTemplate(metaClass, templ, info)) + return false; metaClass->typeEntry()->setBaseContainerType(templ->typeEntry()); return true; } @@ -3143,54 +3144,81 @@ AbstractMetaClassPtr return result; } + +static std::optional<AbstractMetaType> + inheritTemplateParameter(const AbstractMetaClassPtr &subclass, + const AbstractMetaClassCPtr &templateClass, + const TypeInfo &info, QString *errorMessage) +{ + QString typeName = info.qualifiedName().join("::"_L1); + TypeDatabase *typeDb = TypeDatabase::instance(); + TypeEntryPtr t; + // Check for a non-type template integer parameter, that is, for a base + // "template <int R, int C> Matrix<R, C>" and subclass + // "typedef Matrix<2,3> Matrix2x3;". If so, create dummy entries of + // EnumValueTypeEntry for the integer values encountered on the fly. + if (isNumber(typeName)) { + t = typeDb->findType(typeName); + if (!t) { + auto parent = typeSystemTypeEntry(subclass->typeEntry()); + t = TypeDatabase::instance()->addConstantValueTypeEntry(typeName, parent); + } + } else { + QStringList possibleNames; + possibleNames << subclass->qualifiedCppName() + "::"_L1 + typeName; + possibleNames << templateClass->qualifiedCppName() + "::"_L1 + typeName; + if (subclass->enclosingClass()) + possibleNames << subclass->enclosingClass()->qualifiedCppName() + "::"_L1 + typeName; + possibleNames << typeName; + + for (const QString &possibleName : std::as_const(possibleNames)) { + t = typeDb->findType(possibleName); + if (t) + break; + } + } + + if (!t) { + *errorMessage = msgIgnoringTemplateParameter(typeName, + "The corresponding type was not found in the typesystem."); + return std::nullopt; + } + + if (t->isContainer()) { + *errorMessage = msgIgnoringTemplateParameter(typeName, + "Template inheritance from nested containers is not supported"); + return std::nullopt; + } + AbstractMetaType result(t); + result.setConstant(info.isConstant()); + result.setReferenceType(info.referenceType()); + result.setIndirectionsV(info.indirectionsV()); + result.decideUsagePattern(); + return result; +} + bool AbstractMetaBuilderPrivate::inheritTemplate(const AbstractMetaClassPtr &subclass, const AbstractMetaClassCPtr &templateClass, const TypeInfo &info) { AbstractMetaTypeList templateTypes; + QString errorMessage; for (const TypeInfo &i : info.instantiations()) { - QString typeName = i.qualifiedName().join(u"::"_s); - TypeDatabase *typeDb = TypeDatabase::instance(); - TypeEntryPtr t; - // Check for a non-type template integer parameter, that is, for a base - // "template <int R, int C> Matrix<R, C>" and subclass - // "typedef Matrix<2,3> Matrix2x3;". If so, create dummy entries of - // EnumValueTypeEntry for the integer values encountered on the fly. - if (isNumber(typeName)) { - t = typeDb->findType(typeName); - if (!t) { - auto parent = typeSystemTypeEntry(subclass->typeEntry()); - t = TypeDatabase::instance()->addConstantValueTypeEntry(typeName, parent); - } + const auto typeO = inheritTemplateParameter(subclass, templateClass, i, &errorMessage); + if (typeO.has_value()) { + templateTypes.append(typeO.value()); } else { - QStringList possibleNames; - possibleNames << subclass->qualifiedCppName() + u"::"_s + typeName; - possibleNames << templateClass->qualifiedCppName() + u"::"_s + typeName; - if (subclass->enclosingClass()) - possibleNames << subclass->enclosingClass()->qualifiedCppName() + u"::"_s + typeName; - possibleNames << typeName; - - for (const QString &possibleName : std::as_const(possibleNames)) { - t = typeDb->findType(possibleName); - if (t) - break; - } - } - - if (t) { - AbstractMetaType temporaryType(t); - temporaryType.setConstant(i.isConstant()); - temporaryType.setReferenceType(i.referenceType()); - temporaryType.setIndirectionsV(i.indirectionsV()); - temporaryType.decideUsagePattern(); - templateTypes << temporaryType; - } else { - qCWarning(lcShiboken).noquote().nospace() - << "Ignoring template parameter " << typeName << " from " - << info.toString() << ". The corresponding type was not found in the typesystem."; + errorMessage = msgInheritTemplateIssue(subclass, info, errorMessage); + qCWarning(lcShiboken, "%s", qPrintable(errorMessage)); } } + if (templateTypes.isEmpty()) { + errorMessage = msgInheritTemplateIssue(subclass, info, + "No template parameters could be inherited"_L1); + qCWarning(lcShiboken, "%s", qPrintable(errorMessage)); + return false; + } return inheritTemplate(subclass, templateClass, templateTypes); } @@ -3540,16 +3568,22 @@ static QList<std::shared_ptr<MetaClass> > if (!result.isValid() && graph.nodeCount()) { QTemporaryFile tempFile(QDir::tempPath() + u"/cyclic_depXXXXXX.dot"_s); tempFile.setAutoRemove(false); - tempFile.open(); - graph.dumpDot(tempFile.fileName(), - [] (const AbstractMetaClassCPtr &c) { return c->name(); }); + const bool ok = tempFile.open(); + if (ok) { + graph.dumpDot(tempFile.fileName(), + [] (const AbstractMetaClassCPtr &c) { return c->name(); }); + } QString message; QTextStream str(&message); str << "Cyclic dependency of classes found:"; for (const auto &c : result.cyclic) str << ' ' << c->name(); - str << ". Graph can be found at \"" << QDir::toNativeSeparators(tempFile.fileName()) << '"'; + str << '.'; + if (ok) { + str << " Graph can be found at \"" + << QDir::toNativeSeparators(tempFile.fileName()) << '"'; + } qCWarning(lcShiboken, "%s", qPrintable(message)); } diff --git a/sources/shiboken6/ApiExtractor/abstractmetafunction.cpp b/sources/shiboken6/ApiExtractor/abstractmetafunction.cpp index f8faba103..11a02f154 100644 --- a/sources/shiboken6/ApiExtractor/abstractmetafunction.cpp +++ b/sources/shiboken6/ApiExtractor/abstractmetafunction.cpp @@ -592,6 +592,11 @@ bool AbstractMetaFunction::isUserAdded() const return d->m_addedFunction && !d->m_addedFunction->isDeclaration(); } +bool AbstractMetaFunction::isUserAddedPythonOverride() const +{ + return d->m_addedFunction && d->m_addedFunction->isPythonOverride(); +} + bool AbstractMetaFunction::isUserDeclared() const { return d->m_addedFunction && d->m_addedFunction->isDeclaration(); diff --git a/sources/shiboken6/ApiExtractor/abstractmetafunction.h b/sources/shiboken6/ApiExtractor/abstractmetafunction.h index bdf5127d1..e252e439d 100644 --- a/sources/shiboken6/ApiExtractor/abstractmetafunction.h +++ b/sources/shiboken6/ApiExtractor/abstractmetafunction.h @@ -300,6 +300,7 @@ public: /// Returns true if the AbstractMetaFunction was added by the user via the type system description. bool isUserAdded() const; + bool isUserAddedPythonOverride() const; /// Returns true if the AbstractMetaFunction was declared by the user via /// the type system description. bool isUserDeclared() const; diff --git a/sources/shiboken6/ApiExtractor/abstractmetalang.cpp b/sources/shiboken6/ApiExtractor/abstractmetalang.cpp index 7cc036cbc..fb49cc9d0 100644 --- a/sources/shiboken6/ApiExtractor/abstractmetalang.cpp +++ b/sources/shiboken6/ApiExtractor/abstractmetalang.cpp @@ -102,6 +102,7 @@ public: AbstractMetaClassCPtr m_templateBaseClass; AbstractMetaFunctionCList m_functions; + AbstractMetaFunctionCList m_userAddedPythonOverrides; AbstractMetaFieldList m_fields; AbstractMetaEnumList m_enums; QList<QPropertySpec> m_propertySpecs; @@ -323,6 +324,11 @@ const AbstractMetaFunctionCList &AbstractMetaClass::functions() const return d->m_functions; } +const AbstractMetaFunctionCList &AbstractMetaClass::userAddedPythonOverrides() const +{ + return d->m_userAddedPythonOverrides; +} + void AbstractMetaClassPrivate::sortFunctions() { std::sort(m_functions.begin(), m_functions.end(), function_sorter); @@ -390,7 +396,13 @@ void AbstractMetaClass::addFunction(const AbstractMetaClassPtr &klass, // to function properly. Such as function modifications nonConstF->setImplementingClass(klass); - klass->d->addFunction(function); + if (function->isUserAddedPythonOverride()) { + nonConstF->setConstant(false); + nonConstF->setCppAttribute(FunctionAttribute::Static); + klass->d->m_userAddedPythonOverrides.append(function); + } else { + klass->d->addFunction(function); + } } bool AbstractMetaClass::hasSignal(const AbstractMetaFunction *other) const @@ -1452,6 +1464,12 @@ void AbstractMetaClass::fixFunctions(const AbstractMetaClassPtr &klass) } for (const auto &superClassC : d->m_baseClasses) { + for (const auto &pof : superClassC->userAddedPythonOverrides()) { + auto *clonedPof = pof->copy(); + clonedPof->setOwnerClass(klass); + d->m_userAddedPythonOverrides.append(AbstractMetaFunctionCPtr{clonedPof}); + } + auto superClass = std::const_pointer_cast<AbstractMetaClass>(superClassC); AbstractMetaClass::fixFunctions(superClass); // Since we always traverse the complete hierarchy we are only diff --git a/sources/shiboken6/ApiExtractor/abstractmetalang.h b/sources/shiboken6/ApiExtractor/abstractmetalang.h index f7ae7b69f..3dc876690 100644 --- a/sources/shiboken6/ApiExtractor/abstractmetalang.h +++ b/sources/shiboken6/ApiExtractor/abstractmetalang.h @@ -66,6 +66,7 @@ public: ~AbstractMetaClass(); const AbstractMetaFunctionCList &functions() const; + const AbstractMetaFunctionCList &userAddedPythonOverrides() const; void setFunctions(const AbstractMetaFunctionCList &functions); static void addFunction(const AbstractMetaClassPtr &klass, const AbstractMetaFunctionCPtr &function); diff --git a/sources/shiboken6/ApiExtractor/abstractmetatype.cpp b/sources/shiboken6/ApiExtractor/abstractmetatype.cpp index dcfc74bbb..3ec07509d 100644 --- a/sources/shiboken6/ApiExtractor/abstractmetatype.cpp +++ b/sources/shiboken6/ApiExtractor/abstractmetatype.cpp @@ -543,6 +543,7 @@ void AbstractMetaType::decideUsagePattern() pattern = ObjectPattern; } setTypeUsagePattern(pattern); + Q_ASSERT(pattern != ContainerPattern || !d->m_instantiations.isEmpty()); } bool AbstractMetaTypeData::hasTemplateChildren() const diff --git a/sources/shiboken6/ApiExtractor/addedfunction.h b/sources/shiboken6/ApiExtractor/addedfunction.h index 06986a47b..b8d189b7a 100644 --- a/sources/shiboken6/ApiExtractor/addedfunction.h +++ b/sources/shiboken6/ApiExtractor/addedfunction.h @@ -79,6 +79,9 @@ struct AddedFunction bool isDeclaration() const { return m_isDeclaration; } // <declare-function> void setDeclaration(bool value) { m_isDeclaration = value; } + bool isPythonOverride() const { return m_isPythonOverride; } + void setPythonOverride(bool o) { m_isPythonOverride = o; } + const FunctionModificationList &modifications() const { return m_modifications; } FunctionModificationList &modifications() { return m_modifications; } @@ -101,6 +104,7 @@ private: bool m_isClassMethod = false; bool m_isStatic = false; bool m_isDeclaration = false; + bool m_isPythonOverride = false; }; QDebug operator<<(QDebug d, const AddedFunction::Argument &a); diff --git a/sources/shiboken6/ApiExtractor/apiextractor.cpp b/sources/shiboken6/ApiExtractor/apiextractor.cpp index ea45f22ba..83ee4437e 100644 --- a/sources/shiboken6/ApiExtractor/apiextractor.cpp +++ b/sources/shiboken6/ApiExtractor/apiextractor.cpp @@ -669,6 +669,8 @@ ApiExtractorPrivate::collectInstantiatedContainersAndSmartPointers(Instantiation return; for (const auto &func : metaClass->functions()) collectInstantiatedContainersAndSmartPointers(context, func); + for (const auto &func : metaClass->userAddedPythonOverrides()) + collectInstantiatedContainersAndSmartPointers(context, func); for (const AbstractMetaField &field : metaClass->fields()) addInstantiatedContainersAndSmartPointers(context, field.type(), field.name()); diff --git a/sources/shiboken6/ApiExtractor/messages.cpp b/sources/shiboken6/ApiExtractor/messages.cpp index f9f46f520..b1f0b240e 100644 --- a/sources/shiboken6/ApiExtractor/messages.cpp +++ b/sources/shiboken6/ApiExtractor/messages.cpp @@ -481,6 +481,21 @@ QString msgCannotFindTypeEntryForSmartPointer(const QString &t, const QString &s + u"\" for instantiation of \""_s +smartPointerType + u"\"."_s; } +QString msgInheritTemplateIssue(const AbstractMetaClassPtr &subclass, + const TypeInfo &info, + const QString &what) +{ + return "While inheriting template "_L1 + subclass->name() + + " from "_L1 + info.toString() + ": "_L1 + what; +} + +QString msgIgnoringTemplateParameter(const QString &typeName, + const char *why) +{ + return "Ignoring template parameter "_L1 + typeName + + ": "_L1 + QLatin1StringView(why); +} + QString msgInvalidSmartPointerType(const TypeInfo &i) { return u"Invalid smart pointer type \""_s +i.toString() + u"\"."_s; diff --git a/sources/shiboken6/ApiExtractor/messages.h b/sources/shiboken6/ApiExtractor/messages.h index 2899cbdfa..e3f582b49 100644 --- a/sources/shiboken6/ApiExtractor/messages.h +++ b/sources/shiboken6/ApiExtractor/messages.h @@ -126,6 +126,10 @@ QString msgUnableToTranslateType(const TypeInfo &typeInfo, QString msgCannotFindTypeEntry(const QString &t); QString msgCannotFindTypeEntryForSmartPointer(const QString &t, const QString &smartPointerType); +QString msgInheritTemplateIssue(const AbstractMetaClassPtr &subclass, + const TypeInfo &info, const QString &what); +QString msgIgnoringTemplateParameter(const QString &typeName, + const char *why); QString msgInvalidSmartPointerType(const TypeInfo &i); QString msgCannotFindSmartPointerInstantion(const TypeInfo &i); diff --git a/sources/shiboken6/ApiExtractor/tests/testconversionruletag.cpp b/sources/shiboken6/ApiExtractor/tests/testconversionruletag.cpp index 731ce161c..b5efd92a6 100644 --- a/sources/shiboken6/ApiExtractor/tests/testconversionruletag.cpp +++ b/sources/shiboken6/ApiExtractor/tests/testconversionruletag.cpp @@ -23,7 +23,7 @@ void TestConversionRuleTag::testConversionRuleTagWithFile() // temp file used later constexpr auto conversionData = "Hi! I'm a conversion rule."_L1; QTemporaryFile file; - file.open(); + QVERIFY(file.open()); QCOMPARE(file.write(conversionData.constData()), conversionData.size()); file.close(); diff --git a/sources/shiboken6/ApiExtractor/typesystem_enums.h b/sources/shiboken6/ApiExtractor/typesystem_enums.h index 81304e6c2..9ecbb08a1 100644 --- a/sources/shiboken6/ApiExtractor/typesystem_enums.h +++ b/sources/shiboken6/ApiExtractor/typesystem_enums.h @@ -35,6 +35,7 @@ enum CodeSnipPosition { CodeSnipPositionBeginning, CodeSnipPositionEnd, CodeSnipPositionDeclaration, + CodeSnipPositionPyOverride, CodeSnipPositionAny }; diff --git a/sources/shiboken6/ApiExtractor/typesystem_typedefs.h b/sources/shiboken6/ApiExtractor/typesystem_typedefs.h index 59bb452a3..5a4e12ff2 100644 --- a/sources/shiboken6/ApiExtractor/typesystem_typedefs.h +++ b/sources/shiboken6/ApiExtractor/typesystem_typedefs.h @@ -68,6 +68,7 @@ using TypedefEntryCPtr = std::shared_ptr<const TypedefEntry>; using TypeSystemTypeEntryCPtr = std::shared_ptr<const TypeSystemTypeEntry>; using ValueTypeEntryCPtr = std::shared_ptr<const ValueTypeEntry>; +using ComplexTypeEntryCList = QList<ComplexTypeEntryCPtr>; using ContainerTypeEntryCList = QList<ContainerTypeEntryCPtr>; using NamespaceTypeEntryList = QList<NamespaceTypeEntryPtr>; using PrimitiveTypeEntryCList = QList<PrimitiveTypeEntryCPtr>; diff --git a/sources/shiboken6/ApiExtractor/typesystemparser.cpp b/sources/shiboken6/ApiExtractor/typesystemparser.cpp index 4e666151d..2b686e997 100644 --- a/sources/shiboken6/ApiExtractor/typesystemparser.cpp +++ b/sources/shiboken6/ApiExtractor/typesystemparser.cpp @@ -93,6 +93,7 @@ constexpr auto positionAttribute = "position"_L1; constexpr auto preferredConversionAttribute = "preferred-conversion"_L1; constexpr auto preferredTargetLangTypeAttribute = "preferred-target-lang-type"_L1; constexpr auto pythonEnumTypeAttribute = "python-type"_L1; +constexpr auto pythonOverrideAttribute = "python-override"_L1; constexpr auto cppEnumTypeAttribute = "cpp-type"_L1; constexpr auto qtMetaObjectFunctionsAttribute = "qt-metaobject"_L1; constexpr auto qtMetaTypeAttribute = "qt-register-metatype"_L1; @@ -331,7 +332,8 @@ ENUM_LOOKUP_BEGIN(TypeSystem::CodeSnipPosition, Qt::CaseInsensitive, { {u"beginning", TypeSystem::CodeSnipPositionBeginning}, {u"end", TypeSystem::CodeSnipPositionEnd}, - {u"declaration", TypeSystem::CodeSnipPositionDeclaration} + {u"declaration", TypeSystem::CodeSnipPositionDeclaration}, + {u"override", TypeSystem::CodeSnipPositionPyOverride} }; ENUM_LOOKUP_LINEAR_SEARCH @@ -2600,6 +2602,7 @@ bool TypeSystemParser::parseAddFunction(const ConditionalStreamReader &, QString returnType; bool staticFunction = false; bool classMethod = false; + bool pythonOverride = false; QString access; for (auto i = attributes->size() - 1; i >= 0; --i) { const auto name = attributes->at(i).qualifiedName(); @@ -2615,6 +2618,9 @@ bool TypeSystemParser::parseAddFunction(const ConditionalStreamReader &, classmethodAttribute, false); } else if (name == accessAttribute) { access = attributes->takeAt(i).value().toString(); + } else if (name == pythonOverrideAttribute) { + pythonOverride = convertBoolean(attributes->takeAt(i).value(), + pythonOverrideAttribute, false); } } @@ -2638,6 +2644,7 @@ bool TypeSystemParser::parseAddFunction(const ConditionalStreamReader &, func->setStatic(staticFunction); func->setClassMethod(classMethod); + func->setPythonOverride(pythonOverride); func->setTargetLangPackage(m_defaultPackage); // Create signature for matching modifications diff --git a/sources/shiboken6/doc/shibokenmodule.rst b/sources/shiboken6/doc/shibokenmodule.rst index b31efbcd1..3bc4fa6ba 100644 --- a/sources/shiboken6/doc/shibokenmodule.rst +++ b/sources/shiboken6/doc/shibokenmodule.rst @@ -116,6 +116,20 @@ To import the module: This method should be used **only** for debug purposes by developers. + .. function:: dumpTypeGraph(file_name) + + Dumps the inheritance graph of the types existing in libshiboken + to ``.dot`` file for use with `Graphviz <https://graphviz.org/>`_. + +.. function:: dumpWrapperMap() + + Dumps the map of wrappers existing in libshiboken to standard error. + +.. function:: dumpConverters() + + Dumps the map of named converters existing in libshiboken to standard + error. + .. py:class:: VoidPtr(address, size = -1, writeable = 0) :param address: (PyBuffer, SbkObject, int, VoidPtr) diff --git a/sources/shiboken6/doc/typediscovery.rst b/sources/shiboken6/doc/typediscovery.rst new file mode 100644 index 000000000..76d3adf7b --- /dev/null +++ b/sources/shiboken6/doc/typediscovery.rst @@ -0,0 +1,145 @@ +.. _typediscovery: + +************** +Type Discovery +************** + +When converting objects which are part of a class hierarchy from a pointer to a +base class, it is expected to get the Python type of the actual, most derived +type, as opposed to C++ which requires a cast for this: + +.. code-block:: python + + def event(self, event): + if event.type() == QEvent.Type.MousePress: + self.do_things(event.position()) + ... + + +.. code-block:: c++ + + bool event(QEvent *event) override + { + if (event->type() == QEvent::MousePress) { + auto *mouseEvent = static_cast<QMouseEvent *>(event); + doThings(mouseEvent->position()); + ... + } + +The process of determining the type of the event is called `type discovery`. + +Shiboken generates code to automatically detect the type. First, it tries to +find a converter for the name obtained by ``typeid(*pointer).name()``. This +should normally work as this name is registered by the binding. If that fails, +it starts walking a type inheritance graph built up in libshiboken to find the +most derived class by using a cast function (``dynamic_cast<>`` by default) to +check. + +For normal class hierarchies with virtual destructors, no special handling +is required since ``typeid()`` usually detects the proper class name. + +Multiple inheritance +==================== + +In case of multiple inheritance in C++, the conversion to the derived class is +not done in case it is not a single-line direct inheritance. For example, in +Qt, the class ``QWidget`` inherits both ``QObject`` (base of the ``QObject`` +hierarchy) and ``QPaintDevice``. + +When calling a function returning a ``QPaintDevice *``, for example +``QPainter.device()``, a Python type representing ``QPaintDevice`` is returned +instead of the underlying widget type. This restriction exists because the +underlying pointer in C++ is a pointer to a ``QPaintDevice *`` and differs from +the pointer to the ``QWidget``. + +Hierarchies of classes with non-virtual destructors +=================================================== + +There are some hierarchies of value-ish C++ classes that do not have virtual +destructors. This makes type discovery based on ``typeid()`` and +``dynamic_cast<>`` impossible. + +Examples in Qt are the ``QStyleOption``-derived or the ``QGradient`` +-derived classes. + +For such classes, some attributes need to be specified on the type entries: + +Primarily, a :ref:`polymorphic-id-expression` attribute +must be specified to be used as a check replacing ``dynamic_cast<>``. + +In addition, a :ref:`polymorphic-name-function` attribute can be specified. +This replaces the type name guess obtained by ``typeid()`` and is mainly a hint +to speed things up by skipping the checks for each type in the inheritance +graph. + +A :ref:`polymorphic-base` attribute identifies the base class of a hierarchy. +It should be given in case the base class inherits from another class to +prevent the logic from going below the base class. + +Using type discovery attributes for class hierarchies with virtual destructors +============================================================================== + +It is possible to use :ref:`polymorphic-id-expression` and +:ref:`polymorphic-name-function` for normal class hierarchies with virtual +destructors as well since they basically replace ``typeid()`` and +``dynamic_cast<>``. This makes sense if expressions can be specified that are +faster than the checks on virtual tables. + +Specifying :ref:`polymorphic-base` can also make sense for generating special +cast functions in case of multiple inheritance. For example, in Qt, +``QWindow``, ``QLayout``, ``QWidget`` are base classes of hierarchies. Since +they all inherit from ``QObject``, indicating the base classes prevents +the logic from using ``QObject`` as a base class. + +.. _typediscovery-attributes: + +Type discovery attributes reference +=================================== + +The following attributes related to type discovery may be be specified on the +:ref:`object-type` or :ref:`value-type` elements: + +.. _polymorphic-id-expression: + +polymorphic-id-expression ++++++++++++++++++++++++++ + +The **polymorphic-id-expression** attribute specifies an expression checking +whether a base class pointer is of the matching type. For example, in a +``virtual eventHandler(BaseEvent *e)`` function, this is used to construct a +Python wrapper matching the derived class (for example, a ``MouseEvent`` or +similar). The attribute value may contain placeholders: + +%1 + Fully qualified class name + +%B + Fully qualified name of the base class (found by base class + search or as indicated by **polymorphic-base**). + +To check for a class inheriting ``BaseEvent``, specify: + +.. code-block:: xml + + <object-type name="MouseEvent" + polymorphic-id-expression="%B->type() == BaseEvent::MouseEvent"/> + +.. _polymorphic-name-function: + +polymorphic-name-function ++++++++++++++++++++++++++ + +The **polymorphic-name-function** attribute specifies the name of a function +returning the type name of a derived class on the base class type entry. +Normally, ``typeid(ptr).name()`` is used for this. + +The function is expected to return ``const char *``. + +.. _polymorphic-base: + +polymorphic-base +++++++++++++++++ + +The boolean **polymorphic-base** attribute indicates whether the class is the +base class of a class hierarchy. It is used for the *%B* placeholder in +**polymorphic-id-expression** and for cast operations in multiple inheritance. diff --git a/sources/shiboken6/doc/typesystem.rst b/sources/shiboken6/doc/typesystem.rst index e1e4fdda2..26f929801 100644 --- a/sources/shiboken6/doc/typesystem.rst +++ b/sources/shiboken6/doc/typesystem.rst @@ -65,3 +65,4 @@ Extra options and Python caveats typesystem_solving_compilation.rst typesystem_specialfunctions.rst + typediscovery.rst diff --git a/sources/shiboken6/doc/typesystem_codeinjection.rst b/sources/shiboken6/doc/typesystem_codeinjection.rst index d0a5a0390..03d5f4b16 100644 --- a/sources/shiboken6/doc/typesystem_codeinjection.rst +++ b/sources/shiboken6/doc/typesystem_codeinjection.rst @@ -74,6 +74,9 @@ function. | |shell |declaration|Used only for virtual functions. This code is injected at the | | | | |top. | | | +-----------+--------------------------------------------------------------+ +| | |override |Used only for virtual functions. The code is injected before | +| | | |the code calling the Python override. | +| | +-----------+--------------------------------------------------------------+ | | |beginning |Used only for virtual functions. The code is injected when the| | | | |function does not has a Python implementation, then the code | | | | |is inserted before c++ call | diff --git a/sources/shiboken6/doc/typesystem_converters.rst b/sources/shiboken6/doc/typesystem_converters.rst index 7bdabc49c..ab6fba930 100644 --- a/sources/shiboken6/doc/typesystem_converters.rst +++ b/sources/shiboken6/doc/typesystem_converters.rst @@ -233,61 +233,3 @@ Variables & Functions **%CHECKTYPE[CPPTYPE]** Replaced by a |project| type checking function for a Python variable. The C++ type is indicated by ``CPPTYPE``. - - -.. _oldconverters: - -Converting The Old Converters -============================= - -If you use |project| for your bindings, and has defined some type conversions -using the ``Shiboken::Converter`` template, then you must update your converters -to the new scheme. - -Previously your conversion rules were declared in one line, like this: - - -.. code-block:: xml - - <primitive-type name="Complex" target-lang-api-name="PyComplex"> - <include file-name="complex.h" location="global"/> - <conversion-rule file="complex_conversions.h"/> - </primitive-type> - - -And implemented in a separate C++ file, like this: - - -.. code-block:: c++ - - namespace Shiboken { - template<> struct Converter<Complex> - { - static inline bool checkType(PyObject* pyObj) { - return PyComplex_Check(pyObj); - } - static inline bool isConvertible(PyObject* pyObj) { - return PyComplex_Check(pyObj); - } - static inline PyObject* toPython(void* cppobj) { - return toPython(*reinterpret_cast<Complex*>(cppobj)); - } - static inline PyObject* toPython(const Complex& cpx) { - return PyComplex_FromDoubles(cpx.real(), cpx.imag()); - } - static inline Complex toCpp(PyObject* pyobj) { - double real = PyComplex_RealAsDouble(pyobj); - double imag = PyComplex_ImagAsDouble(pyobj); - return Complex(real, imag); - } - }; - } - - -In this case, the parts of the implementation that will be used in the new -conversion-rule are the ones in the two last method -``static inline PyObject* toPython(const Complex& cpx)`` and -``static inline Complex toCpp(PyObject* pyobj)``. The ``isConvertible`` method -is gone, and the ``checkType`` is now an attribute of the :ref:`add-conversion <add-conversion>` -tag. Refer back to the first example in this page and you will be able to -correlate the above template with the new scheme of conversion rule definition. diff --git a/sources/shiboken6/doc/typesystem_manipulating_objects.rst b/sources/shiboken6/doc/typesystem_manipulating_objects.rst index 89e3879b4..e024cdf00 100644 --- a/sources/shiboken6/doc/typesystem_manipulating_objects.rst +++ b/sources/shiboken6/doc/typesystem_manipulating_objects.rst @@ -282,6 +282,7 @@ logic. This can be done using the :ref:`inject-code` node. access="public | protected" overload-number="number" static="yes | no" classmethod="yes | no" + python-override ="yes | no" since="..."/> </object-type> @@ -320,6 +321,10 @@ within the `signature` field See :ref:`sequence-protocol` for adding the respective functions. +The *optional* attribute ``python-override`` indicates a special type +of added function, a python-override that will be generated into +the native wrapper (see :ref:`modifying-virtual-functions`). + .. _declare-function: declare-function @@ -500,3 +505,52 @@ configuration (see also option :ref:`drop-type-entries`) intended for building several configurations from one generated source tree, but still requires listing the correct source files in the ``CMakeLists.txt`` file. + +.. _modifying-virtual-functions: + +Modifying virtual functions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Some C++ virtual functions are unsuitable for Python bindings: + +.. code-block:: c + + virtual void getInt(int *result) const; + +In that case, you would modify it to return the integer instead (or a tuple +in case of several out-parameters): + +.. code-block:: c + + virtual int getInt() const; + +For the binding itself, use the common argument modifications (removing +arguments, modifying return types with injected code snippets) to modify the +signature. + +To make it possible to reimplement the function in Python with the modified +signature, add a ``python-override`` function with that signature, using an +arbitrary name for disambiguation: + +.. code-block:: xml + + <add-function signature="getIntPyOverride()" + return-type="int" python-override="true"/> + +This causes a static function performing the call into Python for the override +to be generated into the native wrapper. + +In the existing virtual function, inject a code snippet at the ``shell`` / +``override`` position which calls the newly added function. The first 2 +arguments are the `Global interpreter lock handle` (``Shiboken::GilState``) and +the Python method determined by the override check (``PyObject *``). The +snippet then converts the arguments and return values and returns after that: + +.. code-block:: xml + + <modify-function signature="getInt(int*)const"> + <inject-code class="shell" position="override"> + *result = getIntPyOverride(gil, pyOverride.object()); + return; + </inject-code> + </modify-function> diff --git a/sources/shiboken6/doc/typesystem_specifying_types.rst b/sources/shiboken6/doc/typesystem_specifying_types.rst index 66e68ae2b..f65b79bb4 100644 --- a/sources/shiboken6/doc/typesystem_specifying_types.rst +++ b/sources/shiboken6/doc/typesystem_specifying_types.rst @@ -536,37 +536,8 @@ type system has this attribute set, the heuristics will be applied to all classes. In shiboken 7, it will be mandatory to set the attribute. -The *optional* **polymorphic-id-expression** attribute specifies an -expression checking whether a base class pointer is of the matching -type. For example, in a ``virtual eventHandler(BaseEvent *e)`` -function, this is used to construct a Python wrapper matching -the derived class (for example, a ``MouseEvent`` or similar). -The attribute value may contain placeholders: - -%1 - Fully qualified class name - -%B - Fully qualified name of the base class (found by base class - search or as indicated by **polymorphic-base**). - -To check for a class inheriting ``BaseEvent``, specify: - -.. code-block:: xml - - <object-type name="MouseEvent" - polymorphic-id-expression="%B->type() == BaseEvent::MouseEvent"/> - -The *optional* **polymorphic-name-function** specifies the name of a -function returning the type name of a derived class on the base class -type entry. Normally, ``typeid(ptr).name()`` is used for this. -However, this fails if the type hierarchy does not have virtual functions. -In this case, a function is required which typically decides depending -on some type enumeration. - -The *optional* **polymorphic-base** attribute indicates -whether the class is the base class of a class hierarchy -(used for the *%B* placeholder in **polymorphic-id-expression**). +For the *optional* **polymorphic-id-expression**, **polymorphic-name-function** +and **polymorphic-base** attributes, see :ref:`typediscovery-attributes`. interface-type ^^^^^^^^^^^^^^ diff --git a/sources/shiboken6/generator/qtdoc/qtdocgenerator.cpp b/sources/shiboken6/generator/qtdoc/qtdocgenerator.cpp index 1103cef29..2797ff254 100644 --- a/sources/shiboken6/generator/qtdoc/qtdocgenerator.cpp +++ b/sources/shiboken6/generator/qtdoc/qtdocgenerator.cpp @@ -147,8 +147,13 @@ static bool shouldSkip(const AbstractMetaFunctionCPtr &func) static bool functionSort(const AbstractMetaFunctionCPtr &func1, const AbstractMetaFunctionCPtr &func2) { const bool ctor1 = func1->isConstructor(); - const bool ctor2 = func2->isConstructor(); - return ctor1 != ctor2 ? ctor1 : func1->name() < func2->name(); + if (ctor1 != func2->isConstructor()) + return ctor1; + const QString &name1 = func1->name(); + const QString &name2 = func2->name(); + if (name1 != name2) + return name1 < name2; + return func1->arguments().size() < func2->arguments().size(); } static inline QVersionNumber versionOf(const TypeEntryCPtr &te) @@ -1421,7 +1426,7 @@ GeneratorDocumentation std::remove_copy_if(allFunctions.cbegin(), allFunctions.cend(), std::back_inserter(result.allFunctions), shouldSkip); - std::sort(result.allFunctions.begin(), result.allFunctions.end(), functionSort); + std::stable_sort(result.allFunctions.begin(), result.allFunctions.end(), functionSort); for (const auto &func : std::as_const(result.allFunctions)) { if (func->isStatic()) diff --git a/sources/shiboken6/generator/shiboken/cppgenerator.cpp b/sources/shiboken6/generator/shiboken/cppgenerator.cpp index 05afde69f..48e7f4fe5 100644 --- a/sources/shiboken6/generator/shiboken/cppgenerator.cpp +++ b/sources/shiboken6/generator/shiboken/cppgenerator.cpp @@ -60,6 +60,9 @@ static const char shibokenErrorsOccurred[] = "Shiboken::Errors::occurred() != nu static constexpr auto virtualMethodStaticReturnVar = "result"_L1; +static constexpr auto sbkObjectTypeF = "SbkObject_TypeF()"_L1; +static const char initInheritanceFunction[] = "initInheritance"; + static QString mangleName(QString name) { if (name == u"None" || name == u"False" || name == u"True" || name == u"from") @@ -624,6 +627,9 @@ void CppGenerator::generateClass(TextStream &s, const GeneratorContext &classCon writeDestructorNative(s, classContext); } + for (const auto &f : metaClass->userAddedPythonOverrides()) + writeUserAddedPythonOverride(s, f); + StringStream smd(TextStream::Language::Cpp); StringStream md(TextStream::Language::Cpp); StringStream signatureStream(TextStream::Language::Cpp); @@ -1129,6 +1135,29 @@ static inline void writeVirtualMethodStaticReturnVar(TextStream &s, const Abstra << virtualMethodStaticReturnVar << ";\n"; } +static void writeFuncNameVar(TextStream &s, const AbstractMetaFunctionCPtr &func, + const QString &funcName) +{ + // PYSIDE-1019: Add info about properties + int propFlag = 0; + if (func->isPropertyReader()) + propFlag |= 1; + if (func->isPropertyWriter()) + propFlag |= 2; + if (propFlag && func->isStatic()) + propFlag |= 4; + QString propStr; + if (propFlag != 90) + propStr = QString::number(propFlag) + u':'; + + if (propFlag != 0) + s << "// This method belongs to a property.\n"; + s << "static const char *funcName = \""; + if (propFlag != 0) + s << propFlag << ':'; + s << funcName << "\";\n"; +} + void CppGenerator::writeVirtualMethodNative(TextStream &s, const AbstractMetaFunctionCPtr &func, int cacheIndex) const @@ -1191,23 +1220,9 @@ void CppGenerator::writeVirtualMethodNative(TextStream &s, s << "if (" << shibokenErrorsOccurred << ")\n" << indent << returnStatement.statement << '\n' << outdent; - // PYSIDE-1019: Add info about properties - int propFlag = 0; - if (func->isPropertyReader()) - propFlag |= 1; - if (func->isPropertyWriter()) - propFlag |= 2; - if (propFlag && func->isStatic()) - propFlag |= 4; - QString propStr; - if (propFlag) - propStr = QString::number(propFlag) + u':'; - s << "static PyObject *nameCache[2] = {};\n"; - if (propFlag) - s << "// This method belongs to a property.\n"; - s << "static const char *funcName = \"" << propStr << funcName << "\";\n" - << "Shiboken::AutoDecRef " << PYTHON_OVERRIDE_VAR + writeFuncNameVar(s, func, funcName); + s << "Shiboken::AutoDecRef " << PYTHON_OVERRIDE_VAR << "(Shiboken::BindingManager::instance().getOverride(this, nameCache, funcName));\n" << "if (" << PYTHON_OVERRIDE_VAR << ".isNull()) {\n" << indent; if (useOverrideCaching(func->ownerClass())) @@ -1216,6 +1231,11 @@ void CppGenerator::writeVirtualMethodNative(TextStream &s, returnStatement.statement, true); s << outdent << "}\n\n"; //WS + if (!snips.isEmpty()) { + writeCodeSnips(s, snips, TypeSystem::CodeSnipPositionPyOverride, + TypeSystem::ShellCode, func, false, lastArg); + } + writeVirtualMethodPythonOverride(s, func, snips, returnStatement); } @@ -1425,6 +1445,28 @@ void CppGenerator::writeVirtualMethodPythonOverride(TextStream &s, s << outdent << "}\n\n"; } +void CppGenerator::writeUserAddedPythonOverride(TextStream &s, + const AbstractMetaFunctionCPtr &func) const +{ + TypeEntryCPtr retType = func->type().typeEntry(); + const QString funcName = func->isOperatorOverload() + ? pythonOperatorFunctionName(func) : func->definitionNames().constFirst(); + + const CodeSnipList snips = func->hasInjectedCode() + ? func->injectedCodeSnips() : CodeSnipList(); + + QString prefix = wrapperName(func->ownerClass()) + u"::"_s; + s << '\n' << functionSignature(func, prefix, QString(), Generator::SkipDefaultValues | + Generator::OriginalTypeDescription) + << "\n{\n" << indent << sbkUnusedVariableCast("gil"); + + writeFuncNameVar(s, func, funcName); + + const auto returnStatement = virtualMethodReturn(api(), func, + func->modifications()); + writeVirtualMethodPythonOverride(s, func, snips, returnStatement); +} + void CppGenerator::writeMetaObjectMethod(TextStream &s, const GeneratorContext &classContext) const { @@ -1548,6 +1590,33 @@ void CppGenerator::writeEnumConverterFunctions(TextStream &s, const AbstractMeta s << '\n'; } +static void writePointerToPythonConverter(TextStream &c, + const AbstractMetaClassCPtr &metaClass, + const QString &typeName, + const QString &cpythonType) +{ + c << "auto *pyOut = reinterpret_cast<PyObject *>(Shiboken::BindingManager::instance().retrieveWrapper(cppIn));\n" + << "if (pyOut) {\n" << indent + << "Py_INCREF(pyOut);\nreturn pyOut;\n" << outdent + << "}\n"; + + const QString nameFunc = metaClass->typeEntry()->polymorphicNameFunction(); + if (nameFunc.isEmpty() && !metaClass->hasVirtualDestructor()) { + c << "return Shiboken::Object::newObjectWithHeuristics(" + << cpythonType << ", const_cast<void *>(cppIn), false);\n"; + return; + } + + c << "auto *tCppIn = reinterpret_cast<const " << typeName << R"( *>(cppIn); +const char *typeName = )"; + if (nameFunc.isEmpty()) + c << "typeid(*tCppIn).name();\n"; + else + c << nameFunc << "(tCppIn);\n"; + c << "return Shiboken::Object::newObjectForPointer(" + << cpythonType << ", const_cast<void *>(cppIn), false, typeName);\n"; +} + void CppGenerator::writeConverterFunctions(TextStream &s, const AbstractMetaClassCPtr &metaClass, const GeneratorContext &classContext) const { @@ -1593,30 +1662,7 @@ void CppGenerator::writeConverterFunctions(TextStream &s, const AbstractMetaClas c << "return PySide::getWrapperForQObject(reinterpret_cast<" << typeName << " *>(const_cast<void *>(cppIn)), " << cpythonType << ");\n"; } else { - c << "auto *pyOut = reinterpret_cast<PyObject *>(Shiboken::BindingManager::instance().retrieveWrapper(cppIn));\n" - << "if (pyOut) {\n" << indent - << "Py_INCREF(pyOut);\nreturn pyOut;\n" << outdent - << "}\n" - << "bool changedTypeName = false;\n" - << "auto *tCppIn = reinterpret_cast<const " << typeName << R"( *>(cppIn); -const char *typeName = )"; - - const QString nameFunc = metaClass->typeEntry()->polymorphicNameFunction(); - if (nameFunc.isEmpty()) - c << "typeid(*tCppIn).name();\n"; - else - c << nameFunc << "(tCppIn);\n"; - c << R"(auto *sbkType = Shiboken::ObjectType::typeForTypeName(typeName); -if (sbkType != nullptr && Shiboken::ObjectType::hasSpecialCastFunction(sbkType)) { - typeName = Shiboken::typeNameOf(typeid(*tCppIn).name()); - changedTypeName = true; -} -)" - << "PyObject *result = Shiboken::Object::newObject(" << cpythonType - << R"(, const_cast<void *>(cppIn), false, /* exactType */ changedTypeName, typeName); -if (changedTypeName) - delete [] typeName; -return result;)"; + writePointerToPythonConverter(c, metaClass, typeName, cpythonType); } std::swap(targetTypeName, sourceTypeName); writeCppToPythonFunction(s, c.toString(), sourceTypeName, targetTypeName); @@ -4219,6 +4265,12 @@ QString CppGenerator::writeContainerConverterInitialization(TextStream &s, return converter; } +QString CppGenerator::typeInitStruct(const TypeEntryCPtr &te) +{ + return cppApiVariableName(te->targetLangPackage()) + u'[' + + getTypeIndexVariableName(te) + u']'; +} + void CppGenerator::writeExtendedConverterInitialization(TextStream &s, const TypeEntryCPtr &externalType, const AbstractMetaClassCList &conversions) @@ -4226,15 +4278,13 @@ void CppGenerator::writeExtendedConverterInitialization(TextStream &s, s << "// Extended implicit conversions for " << externalType->qualifiedTargetLangName() << ".\n"; for (const auto &sourceClass : conversions) { - const QString converterVar = cppApiVariableName(externalType->targetLangPackage()) + u'[' - + getTypeIndexVariableName(externalType) + u']'; QString sourceTypeName = fixedCppTypeName(sourceClass->typeEntry()); QString targetTypeName = fixedCppTypeName(externalType); QString toCpp = pythonToCppFunctionName(sourceTypeName, targetTypeName); QString isConv = convertibleToCppFunctionName(sourceTypeName, targetTypeName); if (!externalType->isPrimitive()) s << cpythonTypeNameExt(externalType) << ";\n"; - writeAddPythonToCppConversion(s, converterVar, toCpp, isConv); + writeAddPythonToCppConversion(s, typeInitStruct(externalType), toCpp, isConv); } } @@ -4317,7 +4367,6 @@ void CppGenerator::writeClassDefinition(TextStream &s, QString tp_hash; QString tp_call; const QString className = chopType(cpythonTypeName(metaClass)); - QString baseClassName; AbstractMetaFunctionCList ctors; const auto &allCtors = metaClass->queryFunctions(FunctionQueryOption::AnyConstructor); for (const auto &f : allCtors) { @@ -4327,9 +4376,6 @@ void CppGenerator::writeClassDefinition(TextStream &s, } } - if (!metaClass->baseClass()) - baseClassName = u"SbkObject_TypeF()"_s; - bool onlyPrivCtor = !metaClass->hasNonPrivateConstructor(); const bool isQApp = usePySideExtensions() @@ -4663,7 +4709,7 @@ void CppGenerator::writeTpTraverseFunction(TextStream &s, const AbstractMetaClas s << "static int " << baseName << "_traverse(PyObject *self, visitproc visit, void *arg)\n{\n" << indent << "auto traverseProc = " - << pyTypeGetSlot("traverseproc", "SbkObject_TypeF()", "Py_tp_traverse") << ";\n" + << pyTypeGetSlot("traverseproc", sbkObjectTypeF, "Py_tp_traverse") << ";\n" << "return traverseProc(self, visit, arg);\n" << outdent << "}\n"; } @@ -4673,7 +4719,7 @@ void CppGenerator::writeTpClearFunction(TextStream &s, const AbstractMetaClassCP QString baseName = cpythonBaseName(metaClass); s << "static int " << baseName << "_clear(PyObject *self)\n{\n" << indent << "auto clearProc = " - << pyTypeGetSlot("inquiry", "SbkObject_TypeF()", "Py_tp_clear") << ";\n" + << pyTypeGetSlot("inquiry", sbkObjectTypeF, "Py_tp_clear") << ";\n" << "return clearProc(self);\n" << outdent << "}\n"; } @@ -5076,10 +5122,15 @@ QString CppGenerator::signatureParameter(const AbstractMetaArgument &arg) const const AbstractMetaFunctionCList conversions = api().implicitConversions(metaType); for (const auto &f : conversions) { - if (f->isConstructor() && !f->arguments().isEmpty()) - signatures << f->arguments().constFirst().type().pythonSignature(); - else if (f->isConversionOperator()) + if (f->isConstructor() && !f->arguments().isEmpty()) { + // PYSIDE-2712: modified types from converting constructors are not always correct + // candidates if they are modified by the type system reference + if (!f->arguments().constFirst().isTypeModified()) { + signatures << f->arguments().constFirst().type().pythonSignature(); + } + } else if (f->isConversionOperator()) { signatures << f->ownerClass()->fullName(); + } } const qsizetype size = signatures.size(); @@ -5402,6 +5453,61 @@ QString CppGenerator::destructorClassName(const AbstractMetaClassCPtr &metaClass return metaClass->qualifiedCppName(); } +// Return the base type entries for introduceWrapperType() +static ComplexTypeEntryCList pyBaseTypeEntries(const AbstractMetaClassCPtr &metaClass) +{ + ComplexTypeEntryCList result; + if (metaClass->isNamespace()) { + if (auto extended = metaClass->extendedNamespace()) + result.append(extended->typeEntry()); + return result; + } + + const auto &baseClasses = metaClass->typeSystemBaseClasses(); + for (auto base : baseClasses) { + for (; base != nullptr; base = base->baseClass()) { // Find a type that is not disabled. + const auto ct = base->typeEntry()->codeGeneration(); + if (ct == TypeEntry::GenerateCode || ct == TypeEntry::GenerateForSubclass) + break; + } + result.append(base->typeEntry()); + } + return result; +} + +// Return the base type strings for introduceWrapperType() +QStringList CppGenerator::pyBaseTypes(const AbstractMetaClassCPtr &metaClass) +{ + const auto &baseEntries = pyBaseTypeEntries(metaClass); + QStringList result; + for (const auto &baseEntry : baseEntries) + result.append("reinterpret_cast<PyObject *>("_L1 + cpythonTypeNameExt(baseEntry) + u')'); + if (result.isEmpty()) // no base classes -> SbkObjectType. + result.append(sbkObjectTypeF); + return result; +} + +void CppGenerator::writeInitInheritance(TextStream &s) const +{ + s << "static void " << initInheritanceFunction << "()\n{\n" << indent + << "auto &bm = Shiboken::BindingManager::instance();\n" + << sbkUnusedVariableCast("bm"); + for (const auto &cls : api().classes()){ + auto te = cls->typeEntry(); + if (shouldGenerate(te)) { + const auto &baseEntries = pyBaseTypeEntries(cls); + if (!baseEntries.isEmpty()) { + const QString childTypeInitStruct = typeInitStruct(cls->typeEntry()); + for (const auto &baseEntry : baseEntries) { + s << "bm.addClassInheritance(&" << typeInitStruct(baseEntry) << ",\n" + << Pad(' ', 23) << '&' << childTypeInitStruct << ");\n"; + } + } + } + } + s << outdent << "}\n\n"; +} + void CppGenerator::writeClassRegister(TextStream &s, const AbstractMetaClassCPtr &metaClass, const GeneratorContext &classContext, @@ -5428,18 +5534,15 @@ void CppGenerator::writeClassRegister(TextStream &s, // Multiple inheritance QString pyTypeBasesVariable = chopType(pyTypeName) + u"_Type_bases"_s; - const auto &baseClasses = metaClass->typeSystemBaseClasses(); - if (metaClass->baseClassNames().size() > 1) { - s << "PyObject *" << pyTypeBasesVariable - << " = PyTuple_Pack(" << baseClasses.size() << ',' << '\n' << indent; - for (qsizetype i = 0, size = baseClasses.size(); i < size; ++i) { - if (i) - s << ",\n"; - s << "reinterpret_cast<PyObject *>(" - << cpythonTypeNameExt(baseClasses.at(i)->typeEntry()) << ')'; - } - s << ");\n\n" << outdent; + const QStringList pyBases = pyBaseTypes(metaClass); + s << "Shiboken::AutoDecRef " << pyTypeBasesVariable << "(PyTuple_Pack(" + << pyBases.size() << ",\n" << indent; + for (qsizetype i = 0, size = pyBases.size(); i < size; ++i) { + if (i) + s << ",\n"; + s << pyBases.at(i); } + s << "));\n\n" << outdent; // Create type and insert it in the module or enclosing class. const QString typePtr = u"_"_s + chopType(pyTypeName) @@ -5474,27 +5577,8 @@ void CppGenerator::writeClassRegister(TextStream &s, s << "&Shiboken::callCppDestructor< " << globalScopePrefix(classContext) << dtorClassName << " >,\n"; - // 6:baseType: Find a type that is not disabled. - auto base = metaClass->isNamespace() - ? metaClass->extendedNamespace() : metaClass->baseClass(); - if (!metaClass->isNamespace()) { - for (; base != nullptr; base = base->baseClass()) { - const auto ct = base->typeEntry()->codeGeneration(); - if (ct == TypeEntry::GenerateCode || ct == TypeEntry::GenerateForSubclass) - break; - } - } - if (base) { - s << cpythonTypeNameExt(base->typeEntry()) << ",\n"; - } else { - s << "nullptr,\n"; - } - // 7:baseTypes - if (metaClass->baseClassNames().size() > 1) - s << pyTypeBasesVariable << ',' << '\n'; - else - s << "0,\n"; + s << pyTypeBasesVariable << ".object()," << '\n'; // 8:wrapperflags QByteArrayList wrapperFlags; @@ -5750,12 +5834,12 @@ void CppGenerator::writeTypeDiscoveryFunction(TextStream &s, if (!polymorphicExpr.isEmpty()) { replacePolymorphicIdPlaceHolders(metaClass, &polymorphicExpr); - s << " if (" << polymorphicExpr << ")\n" << indent + s << "if (" << polymorphicExpr << ")\n" << indent << "return cptr;\n" << outdent; } else if (metaClass->isPolymorphic()) { const auto &ancestors = metaClass->allTypeSystemAncestors(); for (const auto &ancestor : ancestors) { - if (ancestor->baseClass()) + if (ancestor->baseClass() && !ancestor->typeEntry()->isPolymorphicBase()) continue; if (ancestor->isPolymorphic()) { s << "if (instanceType == Shiboken::SbkType< " << m_gsp @@ -5975,18 +6059,14 @@ void CppGenerator::writeInitFunc(TextStream &declStr, TextStream &callStr, } else if (hasParent) { const QString &enclosingName = enclosingEntry->name(); const auto parts = QStringView{enclosingName}.split(u"::", Qt::SkipEmptyParts); + const QString namePathPrefix = enclosingEntry->name().replace("::"_L1, "."_L1); callStr << "Shiboken::Module::AddTypeCreationFunction(" - << "module, \"" << pythonName << "\", " << functionName << ", \""; - for (qsizetype i = 0; i < parts.size(); ++i) { - if (i > 0) - callStr << "\", \""; - callStr << parts.at(i); - } - callStr << "\");\n"; + << "module, \"" << parts[0] << "\", " + << functionName << ", \"" << namePathPrefix << '.' << pythonName << "\");\n"; } else { callStr << "Shiboken::Module::AddTypeCreationFunction(" << "module, \"" << pythonName << "\", " - << "init_" << initFunctionName << ");\n"; + << functionName << ");\n"; } } @@ -6288,6 +6368,8 @@ bool CppGenerator::finishGeneration() // PYSIDE-510: Create a signatures string for the introspection feature. writeSignatureStrings(s, signatureStream.toString(), moduleName(), "global functions"); + writeInitInheritance(s); + // Write module init function const QString globalModuleVar = pythonModuleObjectName(); s << "extern \"C\" LIBSHIBOKEN_EXPORT PyObject *PyInit_" @@ -6425,7 +6507,8 @@ bool CppGenerator::finishGeneration() } } - s << "\nif (" << shibokenErrorsOccurred << ") {\n" << indent + s << '\n' << initInheritanceFunction << "();\n" + << "\nif (" << shibokenErrorsOccurred << ") {\n" << indent << "PyErr_Print();\n" << "Py_FatalError(\"can't initialize module " << moduleName() << "\");\n" << outdent << "}\n"; diff --git a/sources/shiboken6/generator/shiboken/cppgenerator.h b/sources/shiboken6/generator/shiboken/cppgenerator.h index 7e87fd9f3..a31c2ca14 100644 --- a/sources/shiboken6/generator/shiboken/cppgenerator.h +++ b/sources/shiboken6/generator/shiboken/cppgenerator.h @@ -87,6 +87,8 @@ private: const AbstractMetaFunctionCPtr &func, const CodeSnipList &snips, const VirtualMethodReturn &returnStatement) const; + void writeUserAddedPythonOverride(TextStream &s, + const AbstractMetaFunctionCPtr &func) const; void writeVirtualMethodCppCall(TextStream &s, const AbstractMetaFunctionCPtr &func, const QString &funcName, const QList<CodeSnip> &snips, const AbstractMetaArgument *lastArg, const TypeEntryCPtr &retType, @@ -393,10 +395,12 @@ private: static void writeSignatureStrings(TextStream &s, const QString &signatures, const QString &arrayName, const char *comment); + void writeInitInheritance(TextStream &s) const; void writeClassRegister(TextStream &s, const AbstractMetaClassCPtr &metaClass, const GeneratorContext &classContext, const QString &signatures) const; + static QStringList pyBaseTypes(const AbstractMetaClassCPtr &metaClass); static QString destructorClassName(const AbstractMetaClassCPtr &metaClass, const GeneratorContext &classContext); static void writeStaticFieldInitialization(TextStream &s, @@ -477,6 +481,8 @@ private: const AbstractMetaType &type, const ApiExtractorResult &api); void writeSmartPointerConverterInitialization(TextStream &s, const AbstractMetaType &ype) const; + + static QString typeInitStruct(const TypeEntryCPtr &te); static void writeExtendedConverterInitialization(TextStream &s, const TypeEntryCPtr &externalType, const AbstractMetaClassCList &conversions); diff --git a/sources/shiboken6/generator/shiboken/headergenerator.cpp b/sources/shiboken6/generator/shiboken/headergenerator.cpp index c0aa2e129..1f574b47c 100644 --- a/sources/shiboken6/generator/shiboken/headergenerator.cpp +++ b/sources/shiboken6/generator/shiboken/headergenerator.cpp @@ -274,8 +274,15 @@ void *qt_metacast(const char *_clname) override; s << "static void pysideInitQtMetaTypes();\n"; s << "void resetPyMethodCache();\n" - << outdent << "private:\n" << indent - << "mutable bool m_PyMethodCache[" << maxOverrides << "];\n" + << outdent << "private:\n" << indent; + + if (!metaClass->userAddedPythonOverrides().isEmpty()) { + for (const auto &f : metaClass->userAddedPythonOverrides()) + s << functionSignature(f, {}, {}, Generator::OriginalTypeDescription) << ";\n"; + s << '\n'; + } + + s << "mutable bool m_PyMethodCache[" << maxOverrides << "];\n" << outdent << "};\n\n"; } @@ -286,8 +293,6 @@ void HeaderGenerator::writeMemberFunctionWrapper(TextStream &s, { Q_ASSERT(!func->isConstructor() && !func->isOperatorOverload()); s << "inline "; - if (func->isStatic()) - s << "static "; s << functionSignature(func, {}, postfix, Generator::OriginalTypeDescription) << " { "; if (!func->isVoid()) diff --git a/sources/shiboken6/generator/shiboken/shibokengenerator.cpp b/sources/shiboken6/generator/shiboken/shibokengenerator.cpp index ac098f394..a1417e5d9 100644 --- a/sources/shiboken6/generator/shiboken/shibokengenerator.cpp +++ b/sources/shiboken6/generator/shiboken/shibokengenerator.cpp @@ -1147,6 +1147,10 @@ void ShibokenGenerator::writeFunctionArguments(TextStream &s, Options options) const { int argUsed = 0; + if (func->isUserAddedPythonOverride()) { + s << "Shiboken::GilState &gil, PyObject *" << PYTHON_OVERRIDE_VAR; + argUsed += 2; + } for (const auto &arg : func->arguments()) { if (options.testFlag(Generator::SkipRemovedArguments) && arg.isModifiedRemoved()) continue; @@ -1183,6 +1187,8 @@ QString ShibokenGenerator::functionSignature(const AbstractMetaFunctionCPtr &fun { StringStream s(TextStream::Language::Cpp); // The actual function + if (!options.testFlag(Option::SkipDefaultValues) && func->isStatic()) // Declaration + s << "static "; if (func->isEmptyFunction() || func->needsReturnType()) s << functionReturnType(func, options) << ' '; else diff --git a/sources/shiboken6/libshiboken/basewrapper.cpp b/sources/shiboken6/libshiboken/basewrapper.cpp index 35d46b1aa..a79a4cb69 100644 --- a/sources/shiboken6/libshiboken/basewrapper.cpp +++ b/sources/shiboken6/libshiboken/basewrapper.cpp @@ -27,7 +27,9 @@ #include "signature_p.h" #include "voidptr.h" +#include <string> #include <iostream> +#include <sstream> #if defined(__APPLE__) #include <dlfcn.h> @@ -37,7 +39,73 @@ namespace { void _destroyParentInfo(SbkObject *obj, bool keepReference); } -static void callDestructor(const Shiboken::DtorAccumulatorVisitor::DestructorEntries &dts) +namespace Shiboken +{ +// Walk through the first level of non-user-type Sbk base classes relevant for +// C++ object allocation. Return true from the predicate to terminate. +template <class Predicate> +bool walkThroughBases(PyTypeObject *currentType, Predicate predicate) +{ + PyObject *bases = currentType->tp_bases; + const Py_ssize_t numBases = PyTuple_Size(bases); + bool result = false; + for (Py_ssize_t i = 0; !result && i < numBases; ++i) { + auto type = reinterpret_cast<PyTypeObject *>(PyTuple_GetItem(bases, i)); + if (PyType_IsSubtype(type, SbkObject_TypeF()) != 0) { + result = PepType_SOTP(type)->is_user_type + ? walkThroughBases(type, predicate) : predicate(type); + } + } + return result; +} + +int getTypeIndexOnHierarchy(PyTypeObject *baseType, PyTypeObject *desiredType) +{ + int index = -1; + walkThroughBases(baseType, [&index, desiredType](PyTypeObject *node) { + ++index; + return PyType_IsSubtype(node, desiredType) != 0; + }); + return index; +} + +int getNumberOfCppBaseClasses(PyTypeObject *baseType) +{ + int count = 0; + walkThroughBases(baseType, [&count](PyTypeObject *) { + ++count; + return false; + }); + return count; +} + +std::vector<PyTypeObject *> getCppBaseClasses(PyTypeObject *baseType) +{ + std::vector<PyTypeObject *> cppBaseClasses; + walkThroughBases(baseType, [&cppBaseClasses](PyTypeObject *node) { + cppBaseClasses.push_back(node); + return false; + }); + return cppBaseClasses; +} + +using DestructorEntries = std::vector<DestructorEntry>; + +DestructorEntries getDestructorEntries(SbkObject *o) +{ + DestructorEntries result; + void **cptrs = o->d->cptr; + walkThroughBases(Py_TYPE(o), [&result, cptrs](PyTypeObject *node) { + auto *sotp = PepType_SOTP(node); + auto index = result.size(); + result.push_back(DestructorEntry{sotp->cpp_dtor, + cptrs[index]}); + return false; + }); + return result; +} + +static void callDestructor(const DestructorEntries &dts) { for (const auto &e : dts) { Shiboken::ThreadStateSaver threadSaver; @@ -46,6 +114,8 @@ static void callDestructor(const Shiboken::DtorAccumulatorVisitor::DestructorEnt } } +} // namespace Shiboken + extern "C" { @@ -265,8 +335,6 @@ static PyTypeObject *createObjectType() offsetof(SbkObject, ob_dict), offsetof(SbkObject, weakreflist), nullptr); // bufferprocs - // Initialize the hidden data area. - _PepPostInit_SbkObject_Type(type); return type; } @@ -342,9 +410,8 @@ static void SbkDeallocWrapperCommon(PyObject *pyObj, bool canDelete) if (sotp->delete_in_main_thread && Shiboken::currentThreadId() != Shiboken::mainThreadId()) { auto &bindingManager = Shiboken::BindingManager::instance(); if (sotp->is_multicpp) { - Shiboken::DtorAccumulatorVisitor visitor(sbkObj); - Shiboken::walkThroughClassHierarchy(Py_TYPE(pyObj), &visitor); - for (const auto &e : visitor.entries()) + const auto entries = Shiboken::getDestructorEntries(sbkObj); + for (const auto &e : entries) bindingManager.addToDeletionInMainThread(e); } else { Shiboken::DestructorEntry e{sotp->cpp_dtor, sbkObj->d->cptr[0]}; @@ -362,10 +429,9 @@ static void SbkDeallocWrapperCommon(PyObject *pyObj, bool canDelete) if (canDelete) { if (sotp->is_multicpp) { - Shiboken::DtorAccumulatorVisitor visitor(sbkObj); - Shiboken::walkThroughClassHierarchy(Py_TYPE(pyObj), &visitor); + const auto entries = Shiboken::getDestructorEntries(sbkObj); Shiboken::Object::deallocData(sbkObj, true); - callDestructor(visitor.entries()); + callDestructor(entries); } else { void *cptr = sbkObj->d->cptr[0]; Shiboken::Object::deallocData(sbkObj, true); @@ -702,54 +768,24 @@ void _destroyParentInfo(SbkObject *obj, bool keepReference) namespace Shiboken { -bool walkThroughClassHierarchy(PyTypeObject *currentType, HierarchyVisitor *visitor) -{ - PyObject *bases = currentType->tp_bases; - Py_ssize_t numBases = PyTuple_GET_SIZE(bases); - bool result = false; - for (int i = 0; !result && i < numBases; ++i) { - auto type = reinterpret_cast<PyTypeObject *>(PyTuple_GET_ITEM(bases, i)); - if (PyType_IsSubtype(type, reinterpret_cast<PyTypeObject *>(SbkObject_TypeF()))) { - result = PepType_SOTP(type)->is_user_type - ? walkThroughClassHierarchy(type, visitor) : visitor->visit(type); - } - } - return result; -} // Wrapper metatype and base type ---------------------------------------------------------- -HierarchyVisitor::HierarchyVisitor() = default; -HierarchyVisitor::~HierarchyVisitor() = default; - -bool BaseCountVisitor::visit(PyTypeObject *) -{ - m_count++; - return false; -} - -bool BaseAccumulatorVisitor::visit(PyTypeObject *node) -{ - m_bases.push_back(node); - return false; -} - -bool GetIndexVisitor::visit(PyTypeObject *node) -{ - m_index++; - return PyType_IsSubtype(node, m_desiredType); -} +void _initMainThreadId(); // helper.cpp -bool DtorAccumulatorVisitor::visit(PyTypeObject *node) +static std::string msgFailedToInitializeType(const char *description) { - auto *sotp = PepType_SOTP(node); - m_entries.push_back(DestructorEntry{sotp->cpp_dtor, - m_pyObject->d->cptr[m_entries.size()]}); - return false; + std::ostringstream stream; + stream << "[libshiboken] Failed to initialize " << description; + if (auto *error = PepErr_GetRaisedException()) { + if (auto *str = PyObject_Str(error)) + stream << ": " << Shiboken::String::toCString(str); + Py_DECREF(error); + } + stream << '.'; + return stream.str(); } -void _initMainThreadId(); // helper.cpp - namespace Conversions { void init(); } void init() @@ -765,11 +801,13 @@ void init() //Init private data Pep384_Init(); - if (PyType_Ready(SbkObjectType_TypeF()) < 0) - Py_FatalError("[libshiboken] Failed to initialize Shiboken.BaseWrapperType metatype."); + auto *type = SbkObjectType_TypeF(); + if (type == nullptr || PyType_Ready(type) < 0) + Py_FatalError(msgFailedToInitializeType("Shiboken.BaseWrapperType metatype").c_str()); - if (PyType_Ready(SbkObject_TypeF()) < 0) - Py_FatalError("[libshiboken] Failed to initialize Shiboken.BaseWrapper type."); + type = SbkObject_TypeF(); + if (type == nullptr || PyType_Ready(type) < 0) + Py_FatalError(msgFailedToInitializeType("Shiboken.BaseWrapper type").c_str()); VoidPtr::init(); @@ -843,20 +881,6 @@ PyObject *checkInvalidArgumentCount(Py_ssize_t numArgs, Py_ssize_t minArgs, Py_s return result; } -class FindBaseTypeVisitor : public HierarchyVisitor -{ -public: - explicit FindBaseTypeVisitor(PyTypeObject *typeToFind) : m_typeToFind(typeToFind) {} - - bool visit(PyTypeObject *node) override - { - return node == m_typeToFind; - } - -private: - PyTypeObject *m_typeToFind; -}; - std::vector<SbkObject *> splitPyObject(PyObject *pyObj) { std::vector<SbkObject *> result; @@ -897,8 +921,8 @@ bool isUserType(PyTypeObject *type) bool canCallConstructor(PyTypeObject *myType, PyTypeObject *ctorType) { - FindBaseTypeVisitor visitor(ctorType); - if (!walkThroughClassHierarchy(myType, &visitor)) { + auto findBasePred = [ctorType](PyTypeObject *type) { return type == ctorType; }; + if (!walkThroughBases(myType, findBasePred)) { PyErr_Format(PyExc_TypeError, "%s isn't a direct base class of %s", ctorType->tp_name, myType->tp_name); return false; } @@ -970,21 +994,15 @@ introduceWrapperType(PyObject *enclosingObject, const char *originalName, PyType_Spec *typeSpec, ObjectDestructor cppObjDtor, - PyTypeObject *baseType, - PyObject *baseTypes, + PyObject *bases, unsigned wrapperFlags) { - auto *base = baseType ? baseType : SbkObject_TypeF(); - typeSpec->slots[0].pfunc = reinterpret_cast<void *>(base); - auto *bases = baseTypes ? baseTypes : PyTuple_Pack(1, base); + const auto basesSize = PySequence_Fast_GET_SIZE(bases); + assert(basesSize > 0); + typeSpec->slots[0].pfunc = PySequence_Fast_GET_ITEM(bases, 0); auto *type = SbkType_FromSpecBasesMeta(typeSpec, bases, SbkObjectType_TypeF()); - for (int i = 0; i < PySequence_Fast_GET_SIZE(bases); ++i) { - auto *st = reinterpret_cast<PyTypeObject *>(PySequence_Fast_GET_ITEM(bases, i)); - BindingManager::instance().addClassInheritance(st, type); - } - auto sotp = PepType_SOTP(type); if (wrapperFlags & DeleteInMainThread) sotp->delete_in_main_thread = 1; @@ -1054,6 +1072,26 @@ bool hasSpecialCastFunction(PyTypeObject *sbkType) return d != nullptr && d->mi_specialcast != nullptr; } +// Find whether base is a direct single line base class of type +// (no multiple inheritance), that is, a C++ pointer cast can safely be done. +static bool isDirectAncestor(PyTypeObject *type, PyTypeObject *base) +{ + if (type == base) + return true; + if (PyTuple_Size(type->tp_bases) == 0) + return false; + auto *sbkObjectType = SbkObject_TypeF(); + auto *firstBase = reinterpret_cast<PyTypeObject *>(PyTuple_GetItem(type->tp_bases, 0)); + return firstBase != sbkObjectType + && PyType_IsSubtype(type, sbkObjectType) != 0 + && isDirectAncestor(firstBase, base); +} + +bool canDowncastTo(PyTypeObject *baseType, PyTypeObject *targetType) +{ + return isDirectAncestor(targetType, baseType); +} + } // namespace ObjectType @@ -1142,9 +1180,7 @@ void callCppDestructors(SbkObject *pyObj) PyTypeObject *type = Py_TYPE(pyObj); auto *sotp = PepType_SOTP(type); if (sotp->is_multicpp) { - Shiboken::DtorAccumulatorVisitor visitor(pyObj); - Shiboken::walkThroughClassHierarchy(type, &visitor); - callDestructor(visitor.entries()); + callDestructor(getDestructorEntries(pyObj)); } else { Shiboken::ThreadStateSaver threadSaver; threadSaver.save(); @@ -1425,20 +1461,67 @@ SbkObject *findColocatedChild(SbkObject *wrapper, return nullptr; } +// Legacy, for compatibility only. PyObject *newObject(PyTypeObject *instanceType, void *cptr, bool hasOwnership, bool isExactType, const char *typeName) { - // Try to find the exact type of cptr. - if (!isExactType) { - if (PyTypeObject *exactType = ObjectType::typeForTypeName(typeName)) - instanceType = exactType; - else - instanceType = BindingManager::instance().resolveType(&cptr, instanceType); + return isExactType + ? newObjectForType(instanceType, cptr, hasOwnership) + : newObjectWithHeuristics(instanceType, cptr, hasOwnership, typeName); +} + +static PyObject *newObjectWithHeuristicsHelper(PyTypeObject *instanceType, + PyTypeObject *exactType, + void *cptr, + bool hasOwnership) +{ + // Try to find the exact type of cptr. For hierarchies with + // non-virtual destructors, typeid() will return the base name. + // Try type discovery in these cases. + if (exactType == nullptr || exactType == instanceType) { + auto resolved = BindingManager::instance().findDerivedType(cptr, instanceType); + if (resolved.first != nullptr + && Shiboken::ObjectType::canDowncastTo(instanceType, resolved.first)) { + exactType = resolved.first; + cptr = resolved.second; + } } + return newObjectForType(exactType != nullptr ? exactType : instanceType, + cptr, hasOwnership); +} + +PyObject *newObjectForPointer(PyTypeObject *instanceType, + void *cptr, + bool hasOwnership, + const char *typeName) +{ + // Try to find the exact type of cptr. + PyTypeObject *exactType = ObjectType::typeForTypeName(typeName); + // PYSIDE-868: In case of multiple inheritance, (for example, + // a function returning a QPaintDevice * from a QWidget *), + // use instance type to avoid pointer offset errors. + return exactType != nullptr && !Shiboken::ObjectType::canDowncastTo(instanceType, exactType) + ? newObjectForType(instanceType, cptr, hasOwnership) + : newObjectWithHeuristicsHelper(instanceType, exactType, cptr, hasOwnership); +} + + +PyObject *newObjectWithHeuristics(PyTypeObject *instanceType, + void *cptr, + bool hasOwnership, + const char *typeName) +{ + return newObjectWithHeuristicsHelper(instanceType, + ObjectType::typeForTypeName(typeName), + cptr, hasOwnership); +} + +PyObject *newObjectForType(PyTypeObject *instanceType, void *cptr, bool hasOwnership) +{ bool shouldCreate = true; bool shouldRegister = true; SbkObject *self = nullptr; diff --git a/sources/shiboken6/libshiboken/basewrapper.h b/sources/shiboken6/libshiboken/basewrapper.h index f2189d824..ec5545aea 100644 --- a/sources/shiboken6/libshiboken/basewrapper.h +++ b/sources/shiboken6/libshiboken/basewrapper.h @@ -229,13 +229,12 @@ enum WrapperFlags * \returns true if the initialization went fine, false otherwise. */ LIBSHIBOKEN_API PyTypeObject *introduceWrapperType(PyObject *enclosingObject, - const char *typeName, - const char *originalName, - PyType_Spec *typeSpec, - ObjectDestructor cppObjDtor, - PyTypeObject *baseType, - PyObject *baseTypes, - unsigned wrapperFlags = 0); + const char *typeName, + const char *originalName, + PyType_Spec *typeSpec, + ObjectDestructor cppObjDtor, + PyObject *bases, + unsigned wrapperFlags = 0); /** * Set the subtype init hook for a type. @@ -266,6 +265,14 @@ LIBSHIBOKEN_API PyTypeObject *typeForTypeName(const char *typeName); * \since 5.12 */ LIBSHIBOKEN_API bool hasSpecialCastFunction(PyTypeObject *sbkType); + +/// Returns whether a C++ pointer of \p baseType can be safely downcast +/// to \p targetType (base is a direct, single line base class of targetType). +/// (is a direct, single-line inheritance) +/// \param baseType Python type of base class +/// \param targetType Python type of derived class +/// \since 6.8 +LIBSHIBOKEN_API bool canDowncastTo(PyTypeObject *baseType, PyTypeObject *targetType); } namespace Object { @@ -298,7 +305,8 @@ LIBSHIBOKEN_API SbkObject *findColocatedChild(SbkObject *wrapper, const PyTypeObject *instanceType); /** - * Bind a C++ object to Python. + * Bind a C++ object to Python. Forwards to + * newObjectWithHeuristics(), newObjectForType() depending on \p isExactType. * \param instanceType equivalent Python type for the C++ object. * \param hasOwnership if true, Python will try to delete the underlying C++ object when there's no more refs. * \param isExactType if false, Shiboken will use some heuristics to detect the correct Python type of this C++ @@ -312,6 +320,40 @@ LIBSHIBOKEN_API PyObject *newObject(PyTypeObject *instanceType, bool isExactType = false, const char *typeName = nullptr); +/// Bind a C++ object to Python for polymorphic pointers. Calls +/// newObjectWithHeuristics() with an additional check for multiple +/// inheritance, in which case it will fall back to instanceType. +/// \param instanceType Equivalent Python type for the C++ object. +/// \param hasOwnership if true, Python will try to delete the underlying C++ object +/// when there's no more refs. +/// \param typeName If non-null, this will be used as helper to find the correct +/// Python type for this object (obtained by typeid().name(). +LIBSHIBOKEN_API PyObject *newObjectForPointer(PyTypeObject *instanceType, + void *cptr, + bool hasOwnership = true, + const char *typeName = nullptr); + +/// Bind a C++ object to Python using some heuristics to detect the correct +/// Python type of this C++ object. In any case \p instanceType must be provided; +/// it'll be used as search starting point and as fallback. +/// \param instanceType Equivalent Python type for the C++ object. +/// \param hasOwnership if true, Python will try to delete the underlying C++ object +/// C++ object when there are no more references. +/// when there's no more refs. +/// \param typeName If non-null, this will be used as helper to find the correct +/// Python type for this object (obtained by typeid().name(). +LIBSHIBOKEN_API PyObject *newObjectWithHeuristics(PyTypeObject *instanceType, + void *cptr, + bool hasOwnership = true, + const char *typeName = nullptr); + +/// Bind a C++ object to Python using the given type. +/// \param instanceType Equivalent Python type for the C++ object. +/// \param hasOwnership if true, Python will try to delete the underlying +/// C++ object when there are no more references. +LIBSHIBOKEN_API PyObject *newObjectForType(PyTypeObject *instanceType, + void *cptr, bool hasOwnership = true); + /** * Changes the valid flag of a PyObject, invalid objects will raise an exception when someone tries to access it. */ diff --git a/sources/shiboken6/libshiboken/basewrapper_p.h b/sources/shiboken6/libshiboken/basewrapper_p.h index 526bf9fa3..fb9140793 100644 --- a/sources/shiboken6/libshiboken/basewrapper_p.h +++ b/sources/shiboken6/libshiboken/basewrapper_p.h @@ -114,7 +114,8 @@ struct SbkObjectTypePrivate /// True if this type holds two or more C++ instances, e.g.: a Python class which inherits from two C++ classes. unsigned int is_multicpp : 1; - /// True if this type was defined by the user. + /// True if this type was defined by the user (a class written in Python inheriting + /// a class provided by a Shiboken binding). unsigned int is_user_type : 1; /// Tells is the type is a value type or an object-type, see BEHAVIOUR_ *constants. unsigned int type_behaviour : 2; @@ -142,107 +143,7 @@ struct DestructorEntry **/ std::vector<SbkObject *> splitPyObject(PyObject *pyObj); -/** -* Visitor class used by walkOnClassHierarchy function. -*/ -class HierarchyVisitor -{ -public: - HierarchyVisitor(const HierarchyVisitor &) = delete; - HierarchyVisitor(HierarchyVisitor &&) = delete; - HierarchyVisitor &operator=(const HierarchyVisitor &) = delete; - HierarchyVisitor &operator=(HierarchyVisitor &&) = delete; - - HierarchyVisitor(); - virtual ~HierarchyVisitor(); - - virtual bool visit(PyTypeObject *node) = 0; // return true to terminate -}; - -class BaseCountVisitor : public HierarchyVisitor -{ -public: - bool visit(PyTypeObject *) override; - - int count() const { return m_count; } - -private: - int m_count = 0; -}; - -class BaseAccumulatorVisitor : public HierarchyVisitor -{ -public: - using Result = std::vector<PyTypeObject *>; - - bool visit(PyTypeObject *node) override; - - Result bases() const { return m_bases; } - -private: - Result m_bases; -}; - -class GetIndexVisitor : public HierarchyVisitor -{ -public: - explicit GetIndexVisitor(PyTypeObject *desiredType) : m_desiredType(desiredType) {} - - bool visit(PyTypeObject *node) override; - - int index() const { return m_index; } - -private: - int m_index = -1; - PyTypeObject *m_desiredType; -}; - -/// Collect destructors and C++ instances of each C++ object held by a Python -/// object -class DtorAccumulatorVisitor : public HierarchyVisitor -{ -public: - explicit DtorAccumulatorVisitor(SbkObject *pyObj) : m_pyObject(pyObj) {} - - bool visit(PyTypeObject *node) override; - - using DestructorEntries = std::vector<DestructorEntry>; - - const DestructorEntries &entries() const { return m_entries; } - -private: - DestructorEntries m_entries; - SbkObject *m_pyObject; -}; - -/// \internal Internal function used to walk on classes inheritance trees. -/** -* Walk on class hierarchy using a DFS algorithm. -* For each pure Shiboken type found, HierarchyVisitor::visit is called and the algorithm -* considers all children of this type as visited. -*/ -bool walkThroughClassHierarchy(PyTypeObject *currentType, HierarchyVisitor *visitor); - -inline int getTypeIndexOnHierarchy(PyTypeObject *baseType, PyTypeObject *desiredType) -{ - GetIndexVisitor visitor(desiredType); - walkThroughClassHierarchy(baseType, &visitor); - return visitor.index(); -} - -inline int getNumberOfCppBaseClasses(PyTypeObject *baseType) -{ - BaseCountVisitor visitor; - walkThroughClassHierarchy(baseType, &visitor); - return visitor.count(); -} - -inline std::vector<PyTypeObject *> getCppBaseClasses(PyTypeObject *baseType) -{ - BaseAccumulatorVisitor visitor; - walkThroughClassHierarchy(baseType, &visitor); - return visitor.bases(); -} +int getNumberOfCppBaseClasses(PyTypeObject *baseType); namespace Object { diff --git a/sources/shiboken6/libshiboken/bindingmanager.cpp b/sources/shiboken6/libshiboken/bindingmanager.cpp index a0acc4e4b..83c927ae5 100644 --- a/sources/shiboken6/libshiboken/bindingmanager.cpp +++ b/sources/shiboken6/libshiboken/bindingmanager.cpp @@ -7,6 +7,7 @@ #include "bindingmanager.h" #include "gilstate.h" #include "helper.h" +#include "sbkmodule.h" #include "sbkstring.h" #include "sbkstaticstrings.h" #include "sbkfeature_base.h" @@ -15,94 +16,152 @@ #include <cstddef> #include <cstring> #include <fstream> +#include <iostream> #include <mutex> +#include <string_view> #include <unordered_map> +#include <unordered_set> + +// GraphNode for the dependency graph. It keeps a pointer to +// the TypeInitStruct to be able to lazily create the type and hashes +// by the full type name. +struct GraphNode +{ + explicit GraphNode(Shiboken::Module::TypeInitStruct *i) : name(i->fullName), initStruct(i) {} + explicit GraphNode(const char *n) : name(n), initStruct(nullptr) {} // Only for searching + + std::string_view name; + Shiboken::Module::TypeInitStruct *initStruct; + + friend bool operator==(const GraphNode &n1, const GraphNode &n2) { return n1.name == n2.name; } + friend bool operator!=(const GraphNode &n1, const GraphNode &n2) { return n1.name != n2.name; } +}; + +template <> +struct std::hash<GraphNode> { + size_t operator()(const GraphNode &n) const noexcept + { + return std::hash<std::string_view>{}(n.name); + } +}; namespace Shiboken { using WrapperMap = std::unordered_map<const void *, SbkObject *>; -class Graph +template <class NodeType> +class BaseGraph { public: - using NodeList = std::vector<PyTypeObject *>; - using Edges = std::unordered_map<PyTypeObject *, NodeList>; + using NodeList = std::vector<NodeType>; + using NodeSet = std::unordered_set<NodeType>; + + using Edges = std::unordered_map<NodeType, NodeList>; Edges m_edges; - Graph() = default; + BaseGraph() = default; - void addEdge(PyTypeObject *from, PyTypeObject *to) + void addEdge(NodeType from, NodeType to) { m_edges[from].push_back(to); } -#ifndef NDEBUG - void dumpDotGraph() const + NodeSet nodeSet() const { - std::ofstream file("/tmp/shiboken_graph.dot"); - - file << "digraph D {\n"; - + NodeSet result; for (const auto &p : m_edges) { - auto *node1 = p.first; - const NodeList &nodeList = p.second; - for (const PyTypeObject *o : nodeList) { - auto *node2 = o; - file << '"' << node2->tp_name << "\" -> \"" - << node1->tp_name << "\"\n"; - } + result.insert(p.first); + for (const auto node2 : p.second) + result.insert(node2); } - file << "}\n"; + return result; } -#endif +}; - PyTypeObject *identifyType(void **cptr, PyTypeObject *type, PyTypeObject *baseType) const +class Graph : public BaseGraph<GraphNode> +{ +public: + using TypeCptrPair = BindingManager::TypeCptrPair; + + TypeCptrPair identifyType(void *cptr, PyTypeObject *type, PyTypeObject *baseType) const { - auto edgesIt = m_edges.find(type); - if (edgesIt != m_edges.end()) { - const NodeList &adjNodes = m_edges.find(type)->second; - for (PyTypeObject *node : adjNodes) { - PyTypeObject *newType = identifyType(cptr, node, baseType); - if (newType) - return newType; - } - } - void *typeFound = nullptr; - auto *sotp = PepType_SOTP(type); - if (sotp->type_discovery) - typeFound = sotp->type_discovery(*cptr, baseType); - if (typeFound) { - // This "typeFound != type" is needed for backwards compatibility with old modules using a newer version of - // libshiboken because old versions of type_discovery function used to return a PyTypeObject *instead of - // a possible variation of the C++ instance pointer (*cptr). - if (typeFound != type) - *cptr = typeFound; - return type; - } - return nullptr; + return identifyType(cptr, GraphNode(type->tp_name), type, baseType); } -}; + bool dumpTypeGraph(const char *fileName) const; -#ifndef NDEBUG -static void showWrapperMap(const WrapperMap &wrapperMap) +private: + TypeCptrPair identifyType(void *cptr, const GraphNode &typeNode, PyTypeObject *type, + PyTypeObject *baseType) const; +}; + +Graph::TypeCptrPair Graph::identifyType(void *cptr, + const GraphNode &typeNode, PyTypeObject *type, + PyTypeObject *baseType) const { - if (Shiboken::pyVerbose() > 0) { - fprintf(stderr, "-------------------------------\n"); - fprintf(stderr, "WrapperMap: %p (size: %d)\n", &wrapperMap, (int) wrapperMap.size()); - for (auto it = wrapperMap.begin(), end = wrapperMap.end(); it != end; ++it) { - const SbkObject *sbkObj = it->second; - fprintf(stderr, "key: %p, value: %p (%s, refcnt: %d)\n", it->first, - static_cast<const void *>(sbkObj), - (Py_TYPE(sbkObj))->tp_name, - int(Py_REFCNT(reinterpret_cast<const PyObject *>(sbkObj)))); + assert(typeNode.initStruct != nullptr || type != nullptr); + auto edgesIt = m_edges.find(typeNode); + if (edgesIt != m_edges.end()) { + const NodeList &adjNodes = edgesIt->second; + for (const auto &node : adjNodes) { + auto newType = identifyType(cptr, node, nullptr, baseType); + if (newType.first != nullptr) + return newType; } - fprintf(stderr, "-------------------------------\n"); } + + if (type == nullptr) { + if (typeNode.initStruct->type == nullptr) // Layzily create type + type = Shiboken::Module::get(*typeNode.initStruct); + else + type = typeNode.initStruct->type; + } + + auto *sotp = PepType_SOTP(type); + if (sotp->type_discovery != nullptr) { + if (void *derivedCPtr = sotp->type_discovery(cptr, baseType)) + return {type, derivedCPtr}; + } + return {nullptr, nullptr}; +} + +static void formatDotNode(std::string_view name, std::ostream &file) +{ + auto lastDot = name.rfind('.'); + file << " \"" << name << "\" [ label="; + if (lastDot != std::string::npos) { + file << '"' << name.substr(lastDot + 1) << "\" tooltip=\"" + << name.substr(0, lastDot) << '"'; + } else { + file << '"' << name << '"'; + } + file << " ]\n"; +} + +bool Graph::dumpTypeGraph(const char *fileName) const +{ + std::ofstream file(fileName); + if (!file.good()) + return false; + + file << "digraph D {\n"; + + // Define nodes with short names + for (const auto &node : nodeSet()) + formatDotNode(node.name, file); + + // Write edges + for (const auto &p : m_edges) { + const auto &node1 = p.first; + const NodeList &nodeList = p.second; + for (const auto &node2 : nodeList) + file << " \"" << node2.name << "\" -> \"" << node1.name << "\"\n"; + } + file << "}\n"; + return true; } -#endif struct BindingManager::BindingManagerPrivate { using DestructorEntries = std::vector<DestructorEntry>; @@ -115,9 +174,6 @@ struct BindingManager::BindingManagerPrivate { std::recursive_mutex wrapperMapLock; Graph classHierarchy; DestructorEntries deleteInMainThread; - bool destroying; - - BindingManagerPrivate() : destroying(false) {} bool releaseWrapper(void *cptr, SbkObject *wrapper, const int *bases = nullptr); bool releaseWrapperHelper(void *cptr, SbkObject *wrapper); @@ -189,7 +245,8 @@ BindingManager::~BindingManager() debugRemoveFreeHook(); #endif #ifndef NDEBUG - showWrapperMap(m_d->wrapperMapper); + if (Shiboken::pyVerbose() > 0) + dumpWrapperMap(); #endif /* Cleanup hanging references. We just invalidate them as when * the BindingManager is being destroyed the interpreter is alredy @@ -363,15 +420,24 @@ PyObject *BindingManager::getOverride(const void *cptr, return nullptr; } -void BindingManager::addClassInheritance(PyTypeObject *parent, PyTypeObject *child) +void BindingManager::addClassInheritance(Module::TypeInitStruct *parent, + Module::TypeInitStruct *child) +{ + m_d->classHierarchy.addEdge(GraphNode(parent), GraphNode(child)); +} + +BindingManager::TypeCptrPair BindingManager::findDerivedType(void *cptr, PyTypeObject *type) const { - m_d->classHierarchy.addEdge(parent, child); + return m_d->classHierarchy.identifyType(cptr, type, type); } +// FIXME PYSIDE7: remove, just for compatibility PyTypeObject *BindingManager::resolveType(void **cptr, PyTypeObject *type) { - PyTypeObject *identifiedType = m_d->classHierarchy.identifyType(cptr, type, type); - return identifiedType ? identifiedType : type; + auto result = findDerivedType(*cptr, type); + if (result.second != nullptr) + *cptr = result.second; + return result.first != nullptr ? result.first : type; } std::set<PyObject *> BindingManager::getAllPyObjects() @@ -395,6 +461,27 @@ void BindingManager::visitAllPyObjects(ObjectVisitor visitor, void *data) } } +bool BindingManager::dumpTypeGraph(const char *fileName) const +{ + return m_d->classHierarchy.dumpTypeGraph(fileName); +} + +void BindingManager::dumpWrapperMap() +{ + const auto &wrapperMap = m_d->wrapperMapper; + std::cerr << "-------------------------------\n" + << "WrapperMap size: " << wrapperMap.size() << " Types: " + << m_d->classHierarchy.nodeSet().size() << '\n'; + for (auto it = wrapperMap.begin(), end = wrapperMap.end(); it != end; ++it) { + const SbkObject *sbkObj = it->second; + std::cerr << "key: " << it->first << ", value: " + << static_cast<const void *>(sbkObj) << " (" + << (Py_TYPE(sbkObj))->tp_name << ", refcnt: " + << Py_REFCNT(reinterpret_cast<const PyObject *>(sbkObj)) << ")\n"; + } + std::cerr << "-------------------------------\n"; +} + static bool isPythonType(PyTypeObject *type) { // This is a type which should be called by multiple inheritance. diff --git a/sources/shiboken6/libshiboken/bindingmanager.h b/sources/shiboken6/libshiboken/bindingmanager.h index 4b21ae835..54c4e486a 100644 --- a/sources/shiboken6/libshiboken/bindingmanager.h +++ b/sources/shiboken6/libshiboken/bindingmanager.h @@ -5,14 +5,20 @@ #define BINDINGMANAGER_H #include "sbkpython.h" -#include <set> #include "shibokenmacros.h" +#include <set> +#include <utility> + struct SbkObject; namespace Shiboken { +namespace Module { +struct TypeInitStruct; +} + struct DestructorEntry; using ObjectVisitor = void (*)(SbkObject *, void *); @@ -38,7 +44,15 @@ public: SbkObject *retrieveWrapper(const void *cptr); PyObject *getOverride(const void *cptr, PyObject *nameCache[], const char *methodName); - void addClassInheritance(PyTypeObject *parent, PyTypeObject *child); + void addClassInheritance(Module::TypeInitStruct *parent, Module::TypeInitStruct *child); + /// Try to find the correct type of cptr via type discovery knowing that it's at least + /// of type \p type. If a derived class is found, it returns a cptr cast to the type + /// (which may be different in case of multiple inheritance. + /// \param cptr a pointer to the instance of type \p type + /// \param type type of cptr + using TypeCptrPair = std::pair<PyTypeObject *, void *>; + TypeCptrPair findDerivedType(void *cptr, PyTypeObject *type) const; + /** * Try to find the correct type of *cptr knowing that it's at least of type \p type. * In case of multiple inheritance this function may change the contents of cptr. @@ -46,7 +60,7 @@ public: * \param type type of *cptr * \warning This function is slow, use it only as last resort. */ - PyTypeObject *resolveType(void **cptr, PyTypeObject *type); + [[deprecated]] PyTypeObject *resolveType(void **cptr, PyTypeObject *type); std::set<PyObject *> getAllPyObjects(); @@ -59,6 +73,9 @@ public: */ void visitAllPyObjects(ObjectVisitor visitor, void *data); + bool dumpTypeGraph(const char *fileName) const; + void dumpWrapperMap(); + private: ~BindingManager(); BindingManager(); diff --git a/sources/shiboken6/libshiboken/helper.cpp b/sources/shiboken6/libshiboken/helper.cpp index 23663433e..46af68956 100644 --- a/sources/shiboken6/libshiboken/helper.cpp +++ b/sources/shiboken6/libshiboken/helper.cpp @@ -13,6 +13,7 @@ #include <iomanip> #include <iostream> +#include <climits> #include <cstring> #include <cstdarg> #include <cctype> @@ -43,14 +44,21 @@ static std::optional<int> getIntAttr(PyObject *obj, const char *what) return std::nullopt; } +static bool verbose = false; + static void formatTypeTuple(PyObject *t, const char *what, std::ostream &str); static void formatPyTypeObject(const PyTypeObject *obj, std::ostream &str, bool verbose) { - if (obj) { + if (obj == nullptr) { + str << '0'; + return; + } + + str << '"' << obj->tp_name << '"'; + if (verbose) { bool immutableType = false; - str << '"' << obj->tp_name << "\", 0x" << std::hex - << obj->tp_flags << std::dec; + str << ", 0x" << std::hex << obj->tp_flags << std::dec; if (obj->tp_flags & Py_TPFLAGS_HEAPTYPE) str << " [heaptype]"; if (obj->tp_flags & Py_TPFLAGS_BASETYPE) @@ -108,8 +116,6 @@ static void formatPyTypeObject(const PyTypeObject *obj, std::ostream &str, bool } } } - } else { - str << '0'; } } @@ -206,6 +212,8 @@ static void formatPyUnicode(PyObject *obj, std::ostream &str) { // Note: The below call create the PyCompactUnicodeObject.utf8 representation str << '"' << _PepUnicode_AsString(obj) << '"'; + if (!verbose) + return; str << " (" << PyUnicode_GetLength(obj) << ')'; const auto kind = _PepUnicode_KIND(obj); @@ -322,7 +330,11 @@ static void formatPyObjectHelper(PyObject *obj, std::ostream &str) str << "False"; return; } - str << "refs=" << Py_REFCNT(obj) << ", "; + const auto refs = Py_REFCNT(obj); + if (refs == UINT_MAX) // _Py_IMMORTAL_REFCNT + str << "immortal, "; + else + str << "refs=" << refs << ", "; if (PyType_Check(obj)) { str << "type: "; formatPyTypeObject(reinterpret_cast<PyTypeObject *>(obj), str, true); @@ -424,6 +436,18 @@ std::ostream &operator<<(std::ostream &str, const debugPyBuffer &b) return str; } +std::ios_base &debugVerbose(std::ios_base &s) +{ + verbose = true; + return s; +} + +std::ios_base &debugBrief(std::ios_base &s) +{ + verbose = false; + return s; +} + #ifdef _WIN32 // Converts a Unicode string to a string encoded in the Windows console's // code page via wchar_t for use with argv (PYSIDE-1425). diff --git a/sources/shiboken6/libshiboken/helper.h b/sources/shiboken6/libshiboken/helper.h index 4e14b8c4b..f226e8c24 100644 --- a/sources/shiboken6/libshiboken/helper.h +++ b/sources/shiboken6/libshiboken/helper.h @@ -112,7 +112,8 @@ LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugSbkObject LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugPyTypeObject &o); LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugPyBuffer &b); LIBSHIBOKEN_API std::ostream &operator<<(std::ostream &str, const debugPyArrayObject &b); - +LIBSHIBOKEN_API std::ios_base &debugVerbose(std::ios_base &s); +LIBSHIBOKEN_API std::ios_base &debugBrief(std::ios_base &s); } // namespace Shiboken diff --git a/sources/shiboken6/libshiboken/pep384impl.cpp b/sources/shiboken6/libshiboken/pep384impl.cpp index 4826fb379..2b04af857 100644 --- a/sources/shiboken6/libshiboken/pep384impl.cpp +++ b/sources/shiboken6/libshiboken/pep384impl.cpp @@ -105,13 +105,13 @@ static PyType_Spec typeprobe_spec = { static void check_PyTypeObject_valid() { - auto *obtype = reinterpret_cast<PyObject *>(&PyType_Type); - auto *probe_tp_base = reinterpret_cast<PyTypeObject *>( - PyObject_GetAttr(obtype, Shiboken::PyMagicName::base())); + auto *typetype = &PyType_Type; + auto *obtype = reinterpret_cast<PyObject *>(typetype); + auto *probe_tp_base_obj = PyObject_GetAttr(obtype, Shiboken::PyMagicName::base()); + auto *probe_tp_base = reinterpret_cast<PyTypeObject *>(probe_tp_base_obj); auto *probe_tp_bases = PyObject_GetAttr(obtype, Shiboken::PyMagicName::bases()); - auto *check = reinterpret_cast<PyTypeObject *>( - PyType_FromSpecWithBases(&typeprobe_spec, probe_tp_bases)); - auto *typetype = reinterpret_cast<PyTypeObject *>(obtype); + auto *checkObj = PyType_FromSpecWithBases(&typeprobe_spec, probe_tp_bases); + auto *check = reinterpret_cast<PyTypeObject *>(checkObj); PyObject *w = PyObject_GetAttr(obtype, Shiboken::PyMagicName::weakrefoffset()); long probe_tp_weakrefoffset = PyLong_AsLong(w); PyObject *d = PyObject_GetAttr(obtype, Shiboken::PyMagicName::dictoffset()); @@ -149,8 +149,8 @@ check_PyTypeObject_valid() || probe_tp_mro != typetype->tp_mro || Py_TPFLAGS_DEFAULT != (check->tp_flags & Py_TPFLAGS_DEFAULT)) Py_FatalError("The structure of type objects has changed!"); - Py_DECREF(check); - Py_DECREF(probe_tp_base); + Py_DECREF(checkObj); + Py_DECREF(probe_tp_base_obj); Py_DECREF(w); Py_DECREF(d); Py_DECREF(probe_tp_bases); @@ -482,6 +482,22 @@ Pep_GetVerboseFlag() } #endif // Py_LIMITED_API +// Support for pyerrors.h + +#if defined(Py_LIMITED_API) || PY_VERSION_HEX < 0x030C0000 +// Emulate PyErr_GetRaisedException() using the deprecated PyErr_Fetch()/PyErr_Store() +PyObject *PepErr_GetRaisedException() +{ + PyObject *type{}; + PyObject *value{}; + PyObject *traceback{}; + PyErr_Fetch(&type, &value, &traceback); + Py_XINCREF(value); + PyErr_Restore(type, value, traceback); + return value; +} +#endif // Limited or < 3.12 + /***************************************************************************** * * Support for code.h @@ -722,11 +738,8 @@ PyTypeObject *PepStaticMethod_TypePtr = nullptr; static PyTypeObject * getStaticMethodType(void) { - // this works for Python 3, only - // "StaticMethodType = type(str.__dict__['maketrans'])\n"; static const char prog[] = - "from xxsubtype import spamlist\n" - "result = type(spamlist.__dict__['staticmeth'])\n"; + "result = type(str.__dict__['maketrans'])\n"; return reinterpret_cast<PyTypeObject *>(PepRun_GetResult(prog)); } @@ -1009,9 +1022,12 @@ long _PepRuntimeVersion() SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *type) { + // PYSIDE-2676: Use the meta type explicitly. + // A derived type would fail the offset calculation. + static auto *meta = SbkObjectType_TypeF(); assert(SbkObjectType_Check(type)); auto *obType = reinterpret_cast<PyObject *>(type); - void *data = PyObject_GetTypeData(obType, Py_TYPE(obType)); + void *data = PyObject_GetTypeData(obType, meta); return reinterpret_cast<SbkObjectTypePrivate *>(data); } @@ -1061,11 +1077,12 @@ static thread_local SbkObjectTypePrivate *SOTP_value{}; SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *type) { + static auto *meta = SbkObjectType_TypeF(); static bool use_312 = _PepRuntimeVersion() >= 0x030C00; assert(SbkObjectType_Check(type)); if (use_312) { auto *obType = reinterpret_cast<PyObject *>(type); - void *data = PepObject_GetTypeData(obType, Py_TYPE(obType)); + void *data = PepObject_GetTypeData(obType, meta); return reinterpret_cast<SbkObjectTypePrivate *>(data); } if (type == SOTP_key) @@ -1092,18 +1109,6 @@ void PepType_SOTP_delete(PyTypeObject *type) #endif // !defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030C0000 -void _PepPostInit_SbkObject_Type(PyTypeObject *type) -{ - // Special init for SbkObject_Type. - // A normal initialization would recurse PepType_SOTP. - if (_PepRuntimeVersion() >= 0x030C00) { - auto *obType = reinterpret_cast<PyObject *>(type); - void *data = PepObject_GetTypeData(obType, Py_TYPE(obType)); - auto *sbkExt = reinterpret_cast<SbkObjectTypePrivate *>(data); - std::fill_n(reinterpret_cast<char *>(data), sizeof(*sbkExt), 0); - } -} - /* * SbkEnumType extender */ diff --git a/sources/shiboken6/libshiboken/pep384impl.h b/sources/shiboken6/libshiboken/pep384impl.h index 31fd65219..e1bf773b3 100644 --- a/sources/shiboken6/libshiboken/pep384impl.h +++ b/sources/shiboken6/libshiboken/pep384impl.h @@ -156,9 +156,6 @@ struct SbkObjectTypePrivate; LIBSHIBOKEN_API SbkObjectTypePrivate *PepType_SOTP(PyTypeObject *type); LIBSHIBOKEN_API void PepType_SOTP_delete(PyTypeObject *type); -// PYSIDE-2230: SbkObjectType needs a special init -LIBSHIBOKEN_API void _PepPostInit_SbkObject_Type(PyTypeObject *type); - struct SbkEnumType; struct SbkEnumTypePrivate; @@ -191,6 +188,13 @@ LIBSHIBOKEN_API int Pep_GetFlag(const char *name); LIBSHIBOKEN_API int Pep_GetVerboseFlag(void); #endif +// pyerrors.h +#if defined(Py_LIMITED_API) || PY_VERSION_HEX < 0x030C0000 +LIBSHIBOKEN_API PyObject *PepErr_GetRaisedException(); +#else +# define PepErr_GetRaisedException PyErr_GetRaisedException +#endif + /***************************************************************************** * * RESOLVED: unicodeobject.h diff --git a/sources/shiboken6/libshiboken/sbkcontainer.h b/sources/shiboken6/libshiboken/sbkcontainer.h index 240c772a9..8ad5aadc6 100644 --- a/sources/shiboken6/libshiboken/sbkcontainer.h +++ b/sources/shiboken6/libshiboken/sbkcontainer.h @@ -74,10 +74,9 @@ public: static PyObject *tpNewInvalid(PyTypeObject * /* subtype */, PyObject * /* args */, PyObject * /* kwds */) { - PyErr_Format(PyExc_NotImplementedError, + return PyErr_Format(PyExc_NotImplementedError, "Opaque containers of type '%s' cannot be instantiated.", typeid(SequenceContainer).name()); - return nullptr; } static int tpInit(PyObject * /* self */, PyObject * /* args */, PyObject * /* kwds */) @@ -105,10 +104,8 @@ public: static PyObject *sqGetItem(PyObject *self, Py_ssize_t i) { auto *d = get(self); - if (i < 0 || i >= Py_ssize_t(d->m_list->size())) { - PyErr_SetString(PyExc_IndexError, "index out of bounds"); - return nullptr; - } + if (i < 0 || i >= Py_ssize_t(d->m_list->size())) + return PyErr_Format(PyExc_IndexError, "index out of bounds"); auto it = std::cbegin(*d->m_list); std::advance(it, i); return ShibokenContainerValueConverter<value_type>::convertValueToPython(*it); @@ -133,14 +130,10 @@ public: static PyObject *push_back(PyObject *self, PyObject *pyArg) { auto *d = get(self); - if (!ShibokenContainerValueConverter<value_type>::checkValue(pyArg)) { - PyErr_SetString(PyExc_TypeError, "wrong type passed to append."); - return nullptr; - } - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (!ShibokenContainerValueConverter<value_type>::checkValue(pyArg)) + return PyErr_Format(PyExc_TypeError, "wrong type passed to append."); + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); OptionalValue value = ShibokenContainerValueConverter<value_type>::convertValueToCpp(pyArg); if (!value.has_value()) @@ -152,14 +145,10 @@ public: static PyObject *push_front(PyObject *self, PyObject *pyArg) { auto *d = get(self); - if (!ShibokenContainerValueConverter<value_type>::checkValue(pyArg)) { - PyErr_SetString(PyExc_TypeError, "wrong type passed to append."); - return nullptr; - } - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (!ShibokenContainerValueConverter<value_type>::checkValue(pyArg)) + return PyErr_Format(PyExc_TypeError, "wrong type passed to append."); + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); OptionalValue value = ShibokenContainerValueConverter<value_type>::convertValueToCpp(pyArg); if (!value.has_value()) @@ -171,10 +160,8 @@ public: static PyObject *clear(PyObject *self) { auto *d = get(self); - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); d->m_list->clear(); Py_RETURN_NONE; @@ -183,10 +170,8 @@ public: static PyObject *pop_back(PyObject *self) { auto *d = get(self); - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); d->m_list->pop_back(); Py_RETURN_NONE; @@ -195,10 +180,8 @@ public: static PyObject *pop_front(PyObject *self) { auto *d = get(self); - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); d->m_list->pop_front(); Py_RETURN_NONE; @@ -208,21 +191,16 @@ public: static PyObject *reserve(PyObject *self, PyObject *pyArg) { auto *d = get(self); - if (PyLong_Check(pyArg) == 0) { - PyErr_SetString(PyExc_TypeError, "wrong type passed to reserve()."); - return nullptr; - } - if (d->m_const) { - PyErr_SetString(PyExc_TypeError, msgModifyConstContainer); - return nullptr; - } + if (PyLong_Check(pyArg) == 0) + return PyErr_Format(PyExc_TypeError, "wrong type passed to reserve()."); + if (d->m_const) + return PyErr_Format(PyExc_TypeError, msgModifyConstContainer); if constexpr (ShibokenContainerHasReserve<SequenceContainer>::value) { const Py_ssize_t size = PyLong_AsSsize_t(pyArg); d->m_list->reserve(size); } else { - PyErr_SetString(PyExc_TypeError, "Container does not support reserve()."); - return nullptr; + return PyErr_Format(PyExc_TypeError, "Container does not support reserve()."); } Py_RETURN_NONE; diff --git a/sources/shiboken6/libshiboken/sbkconverter.cpp b/sources/shiboken6/libshiboken/sbkconverter.cpp index 358827aa8..f87aca19a 100644 --- a/sources/shiboken6/libshiboken/sbkconverter.cpp +++ b/sources/shiboken6/libshiboken/sbkconverter.cpp @@ -12,7 +12,12 @@ #include "voidptr.h" #include <string> +#include <cstring> +#include <iostream> #include <unordered_map> +#include <unordered_set> +#include <map> +#include <set> static SbkConverter **PrimitiveTypeConverters; @@ -72,6 +77,99 @@ void init() initArrayConverters(); } +static void dumpPyTypeObject(std::ostream &str, PyTypeObject *t) +{ + str << "\nPython type "; + if (t == nullptr) { + str << "<None>"; + return; + } + str << '"' << t->tp_name << '"'; + if (t->tp_base != nullptr && t->tp_base != &PyBaseObject_Type) + str << '(' << t->tp_base->tp_name << ')'; +} + +static void dumpSbkConverter(std::ostream &str, const SbkConverter *c) +{ + str << "SbkConverter " << static_cast<const void *>(c) << ": "; + if (c->pointerToPython != nullptr) + str << ", C++ pointer->Python"; + if (c->copyToPython != nullptr) + str << ", copy->Python"; + if (c->toCppPointerConversion.second != nullptr) + str << ", Python->C++ pointer"; + if (!c->toCppConversions.empty()) + str << ", " << c->toCppConversions.size() << " Python->C++ conversions"; +} + +// Less than operator for a PyTypeObject for dumping the converter map +static bool pyTypeObjectLessThan(const PyTypeObject *t1, const PyTypeObject *t2) +{ + const bool isNull1 = t1 == nullptr; + const bool isNull2 = t2 == nullptr; + if (isNull1 || isNull2) + return isNull1 && !isNull2; + // Internal types (lower case) first + const bool isInternal1 = std::islower(t1->tp_name[0]); + const bool isInternal2 = std::islower(t2->tp_name[0]); + if (isInternal1 != isInternal2) + return !isInternal2; + return std::strcmp(t1->tp_name, t2->tp_name) < 0; +} + +void dumpConverters() +{ + struct PyTypeObjectLess { + + bool operator()(const PyTypeObject *t1, const PyTypeObject *t2) const { + return pyTypeObjectLessThan(t1, t2); + } + }; + + using StringSet = std::set<std::string>; + using SbkConverterNamesMap = std::unordered_map<SbkConverter *, StringSet>; + using PyTypeObjectConverterMap = std::map<PyTypeObject *, SbkConverterNamesMap, + PyTypeObjectLess>; + + auto &str = std::cerr; + + // Sort the entries by the associated PyTypeObjects and converters + PyTypeObjectConverterMap pyTypeObjectConverterMap; + for (const auto &converter : converters) { + auto *sbkConverter = converter.second; + auto *typeObject = sbkConverter->pythonType; + auto typeIt = pyTypeObjectConverterMap.find(typeObject); + if (typeIt == pyTypeObjectConverterMap.end()) + typeIt = pyTypeObjectConverterMap.insert(std::make_pair(typeObject, + SbkConverterNamesMap{})).first; + SbkConverterNamesMap &sbkConverterMap = typeIt->second; + auto convIt = sbkConverterMap.find(sbkConverter); + if (convIt == sbkConverterMap.end()) + convIt = sbkConverterMap.insert(std::make_pair(sbkConverter, + StringSet{})).first; + convIt->second.insert(converter.first); + } + + for (const auto &tc : pyTypeObjectConverterMap) { + dumpPyTypeObject(str, tc.first); + str << ", " << tc.second.size() << " converter(s):\n"; + for (const auto &cn : tc.second) { + str << " "; + dumpSbkConverter(str, cn.first); + str << ", " << cn.second.size() << " alias(es):"; + int i = 0; + for (const auto &name : cn.second) { + if ((i++ % 5) == 0) + str << "\n "; + str << " \"" << name << '"'; + } + str << '\n'; + } + } + + str << '\n'; +} + SbkConverter *createConverterObject(PyTypeObject *type, PythonToCppFunc toCppPointerConvFunc, IsConvertibleToCppFunc toCppPointerCheckFunc, @@ -422,18 +520,36 @@ void registerConverterName(SbkConverter *converter, const char *typeName) converters.insert(std::make_pair(typeName, converter)); } -static std::string getRealTypeName(const char *name) +static std::string getRealTypeName(const std::string &typeName) { - std::string typeName(name); auto size = typeName.size(); if (std::isalnum(typeName[size - 1]) == 0) return typeName.substr(0, size - 1); return typeName; } -SbkConverter *getConverter(const char *typeName) +// PYSIDE-2404: Build a negative cache of already failed lookups. +// The resulting list must be reset after each new import, +// because that can change results. Also clear the cache after +// reaching some threashold. +static std::unordered_set<std::string> nonExistingTypeNames{}; + +// Arbitrary size limit to prevent random name overflows. +static constexpr std::size_t negativeCacheLimit = 50; + +static void rememberAsNonexistent(const std::string &typeName) +{ + if (nonExistingTypeNames.size() > negativeCacheLimit) + clearNegativeLazyCache(); + converters.insert(std::make_pair(typeName, nullptr)); + nonExistingTypeNames.insert(typeName); +} + +SbkConverter *getConverter(const char *typeNameC) { + std::string typeName = typeNameC; auto it = converters.find(typeName); + // PYSIDE-2404: This can also contain explicit nullptr as a negative cache. if (it != converters.end()) return it->second; // PYSIDE-2404: Did not find the name. Load the lazy classes @@ -442,6 +558,9 @@ SbkConverter *getConverter(const char *typeName) it = converters.find(typeName); if (it != converters.end()) return it->second; + // Cache the negative result. Don't forget to clear the cache for new modules. + rememberAsNonexistent(typeName); + if (Shiboken::pyVerbose() > 0) { const std::string message = std::string("Can't find type resolver for type '") + typeName + "'."; @@ -450,6 +569,15 @@ SbkConverter *getConverter(const char *typeName) return nullptr; } +void clearNegativeLazyCache() +{ + for (const auto &typeName : nonExistingTypeNames) { + auto it = converters.find(typeName); + converters.erase(it); + } + nonExistingTypeNames.clear(); +} + SbkConverter *primitiveTypeConverter(int index) { return PrimitiveTypeConverters[index]; @@ -704,14 +832,7 @@ PyTypeObject *getPythonTypeObject(const SbkConverter *converter) PyTypeObject *getPythonTypeObject(const char *typeName) { - auto *type = getPythonTypeObject(getConverter(typeName)); - if (type == nullptr) { - // PYSIDE-2404: Did not find the name. Load the lazy classes - // which have this name and try again. - Shiboken::Module::loadLazyClassesWithName(getRealTypeName(typeName).c_str()); - type = getPythonTypeObject(getConverter(typeName)); - } - return type; + return getPythonTypeObject(getConverter(typeName)); } bool pythonTypeIsValueType(const SbkConverter *converter) diff --git a/sources/shiboken6/libshiboken/sbkconverter_p.h b/sources/shiboken6/libshiboken/sbkconverter_p.h index c886c9b9f..08fc4c8e1 100644 --- a/sources/shiboken6/libshiboken/sbkconverter_p.h +++ b/sources/shiboken6/libshiboken/sbkconverter_p.h @@ -531,6 +531,12 @@ SbkConverter *createConverterObject(PyTypeObject *type, IsConvertibleToCppFunc toCppPointerCheckFunc, CppToPythonFunc pointerToPythonFunc, CppToPythonFunc copyToPythonFunc); + +LIBSHIBOKEN_API void dumpConverters(); + +/// Interface for sbkmodule which must reset cache when new module is loaded. +LIBSHIBOKEN_API void clearNegativeLazyCache(); + } // namespace Shiboken::Conversions #endif // SBK_CONVERTER_P_H diff --git a/sources/shiboken6/libshiboken/sbkcpptonumpy.cpp b/sources/shiboken6/libshiboken/sbkcpptonumpy.cpp index 44e900f01..7637efa70 100644 --- a/sources/shiboken6/libshiboken/sbkcpptonumpy.cpp +++ b/sources/shiboken6/libshiboken/sbkcpptonumpy.cpp @@ -49,17 +49,17 @@ PyObject *createByteArray1(Py_ssize_t, const uint8_t *) PyObject *createDoubleArray1(Py_ssize_t, const double *) { - return Py_None; + Py_RETURN_NONE; } PyObject *createFloatArray1(Py_ssize_t, const float *) { - return Py_None; + Py_RETURN_NONE; } PyObject *createIntArray1(Py_ssize_t, const int *) { - return Py_None; + Py_RETURN_NONE; } #endif // !HAVE_NUMPY diff --git a/sources/shiboken6/libshiboken/sbkfeature_base.cpp b/sources/shiboken6/libshiboken/sbkfeature_base.cpp index caf3517e5..f31b8f4f7 100644 --- a/sources/shiboken6/libshiboken/sbkfeature_base.cpp +++ b/sources/shiboken6/libshiboken/sbkfeature_base.cpp @@ -65,14 +65,24 @@ void disassembleFrame(const char *marker) static PyObject *dismodule = PyImport_ImportModule("dis"); static PyObject *disco = PyObject_GetAttrString(dismodule, "disco"); static PyObject *const _f_lasti = Shiboken::String::createStaticString("f_lasti"); + static PyObject *const _f_lineno = Shiboken::String::createStaticString("f_lineno"); static PyObject *const _f_code = Shiboken::String::createStaticString("f_code"); - auto *frame = reinterpret_cast<PyObject *>(PyEval_GetFrame()); - AutoDecRef f_lasti(PyObject_GetAttr(frame, _f_lasti)); - AutoDecRef f_code(PyObject_GetAttr(frame, _f_code)); + static PyObject *const _co_filename = Shiboken::String::createStaticString("co_filename"); AutoDecRef ignore{}; - fprintf(stdout, "\n%s BEGIN\n", marker); - ignore.reset(PyObject_CallFunctionObjArgs(disco, f_code.object(), f_lasti.object(), nullptr)); - fprintf(stdout, "%s END\n\n", marker); + auto *frame = reinterpret_cast<PyObject *>(PyEval_GetFrame()); + if (frame == nullptr) { + fprintf(stdout, "\n%s BEGIN no frame END\n\n", marker); + } else { + AutoDecRef f_lasti(PyObject_GetAttr(frame, _f_lasti)); + AutoDecRef f_lineno(PyObject_GetAttr(frame, _f_lineno)); + AutoDecRef f_code(PyObject_GetAttr(frame, _f_code)); + AutoDecRef co_filename(PyObject_GetAttr(f_code, _co_filename)); + long line = PyLong_AsLong(f_lineno); + const char *fname = String::toCString(co_filename); + fprintf(stdout, "\n%s BEGIN line=%ld %s\n", marker, line, fname); + ignore.reset(PyObject_CallFunctionObjArgs(disco, f_code.object(), f_lasti.object(), nullptr)); + fprintf(stdout, "%s END line=%ld %s\n\n", marker, line, fname); + } #if PY_VERSION_HEX >= 0x030C0000 && !Py_LIMITED_API if (error_type) PyErr_DisplayException(error_value); diff --git a/sources/shiboken6/libshiboken/sbkmodule.cpp b/sources/shiboken6/libshiboken/sbkmodule.cpp index b705e8380..ccc7cc2cd 100644 --- a/sources/shiboken6/libshiboken/sbkmodule.cpp +++ b/sources/shiboken6/libshiboken/sbkmodule.cpp @@ -10,18 +10,23 @@ #include <unordered_map> #include <unordered_set> +#include <vector> #include <cstring> +/// This hash maps module objects to arrays of converters. +using ModuleConvertersMap = std::unordered_map<PyObject *, SbkConverter **> ; + /// This hash maps module objects to arrays of Python types. using ModuleTypesMap = std::unordered_map<PyObject *, Shiboken::Module::TypeInitStruct *> ; -/// This hash maps module objects to arrays of converters. -using ModuleConvertersMap = std::unordered_map<PyObject *, SbkConverter **>; +struct TypeCreationStruct +{ + Shiboken::Module::TypeCreationFunction func; + std::vector<std::string> subtypeNames; +}; -/// This hash maps type names to type creation functions. -using TypeCreationFunctionModulePair = - std::pair<Shiboken::Module::TypeCreationFunction, PyObject *>; -using NameToTypeFunctionMap = std::unordered_map<std::string, TypeCreationFunctionModulePair>; +/// This hash maps type names to type creation structs. +using NameToTypeFunctionMap = std::unordered_map<std::string, TypeCreationStruct> ; /// This hash maps module objects to maps of names to functions. using ModuleToFuncsMap = std::unordered_map<PyObject *, NameToTypeFunctionMap> ; @@ -50,7 +55,7 @@ LIBSHIBOKEN_API PyTypeObject *get(TypeInitStruct &typeStruct) // As soon as types[index] gets filled, we can stop. std::string_view names(typeStruct.fullName); - bool usePySide = names.substr(0, 8) == std::string("PySide6."); + const bool usePySide = names.compare(0, 8, "PySide6.") == 0; auto dotPos = usePySide ? names.find('.', 8) : names.find('.'); auto startPos = dotPos + 1; AutoDecRef modName(String::fromCppStringView(names.substr(0, dotPos))); @@ -74,6 +79,39 @@ LIBSHIBOKEN_API PyTypeObject *get(TypeInitStruct &typeStruct) return typeStruct.type; } +static void incarnateHelper(PyObject *module, const std::string_view names, + const NameToTypeFunctionMap &nameToFunc) +{ + auto dotPos = names.find('.'); + std::string::size_type startPos = 0; + auto *modOrType{module}; + while (dotPos != std::string::npos) { + auto typeName = names.substr(startPos, dotPos - startPos); + AutoDecRef obTypeName(String::fromCppStringView(typeName)); + modOrType = PyObject_GetAttr(modOrType, obTypeName); + startPos = dotPos + 1; + dotPos = names.find('.', startPos); + } + // now we have the type to create. + auto funcIter = nameToFunc.find(std::string(names)); + // - call this function that returns a PyTypeObject + auto tcStruct = funcIter->second; + auto initFunc = tcStruct.func; + PyTypeObject *type = initFunc(modOrType); + auto name = names.substr(startPos); + PyObject_SetAttrString(modOrType, name.data(), reinterpret_cast<PyObject *>(type)); +} + +static void incarnateSubtypes(PyObject *module, + const std::vector<std::string> &nameList, + NameToTypeFunctionMap &nameToFunc) +{ + for (auto const & tableIter : nameList) { + std::string_view names(tableIter); + incarnateHelper(module, names, nameToFunc); + } +} + static PyTypeObject *incarnateType(PyObject *module, const char *name, NameToTypeFunctionMap &nameToFunc) { @@ -85,13 +123,15 @@ static PyTypeObject *incarnateType(PyObject *module, const char *name, return nullptr; } // - call this function that returns a PyTypeObject - auto pair = funcIter->second; - auto initFunc = pair.first; - auto *modOrType = pair.second; + auto tcStruct = funcIter->second; + auto initFunc = tcStruct.func; + auto *modOrType{module}; // PYSIDE-2404: Make sure that no switching happens during type creation. auto saveFeature = initSelectableFeature(nullptr); PyTypeObject *type = initFunc(modOrType); + if (!tcStruct.subtypeNames.empty()) + incarnateSubtypes(module, tcStruct.subtypeNames, nameToFunc); initSelectableFeature(saveFeature); // - assign this object to the name in the module @@ -164,7 +204,7 @@ static PyObject *PyModule_lazyGetAttro(PyObject *module, PyObject *name) // - locate the name and retrieve the generating function const char *attrNameStr = Shiboken::String::toCString(name); auto &nameToFunc = tableIter->second; - // - create the real type (incarnateType checks this) + // - create the real type and handle subtypes auto *type = incarnateType(module, attrNameStr, nameToFunc); auto *ret = reinterpret_cast<PyObject *>(type); // - if attribute does really not exist use the original @@ -172,7 +212,6 @@ static PyObject *PyModule_lazyGetAttro(PyObject *module, PyObject *name) PyErr_Clear(); return origModuleGetattro(module, name); } - return ret; } @@ -292,25 +331,12 @@ static bool shouldLazyLoad(PyObject *module) return std::strncmp(modName, "PySide6.", 8) == 0; } -void AddTypeCreationFunction(PyObject *module, - const char *name, - TypeCreationFunction func) +void checkIfShouldLoadImmediately(PyObject *module, const std::string &name, + const NameToTypeFunctionMap &nameToFunc) { static const char *flag = getenv("PYSIDE6_OPTION_LAZY"); static const int value = flag != nullptr ? std::atoi(flag) : 1; - // - locate the module in the moduleTofuncs mapping - auto tableIter = moduleToFuncs.find(module); - assert(tableIter != moduleToFuncs.end()); - // - Assign the name/generating function pair. - auto &nameToFunc = tableIter->second; - TypeCreationFunctionModulePair pair{func, module}; - auto nit = nameToFunc.find(name); - if (nit == nameToFunc.end()) - nameToFunc.insert(std::make_pair(name, pair)); - else - nit->second = pair; - // PYSIDE-2404: Lazy Loading // // Options: @@ -319,56 +345,56 @@ void AddTypeCreationFunction(PyObject *module, // 3 - lazy loading for any module. // // By default we lazy load all known modules (option = 1). - if (value == 0 // completely disabled || canNotLazyLoad(module) // for some reason we cannot lazy load || (value == 1 && !shouldLazyLoad(module)) // not a known module ) { - PyTypeObject *type = func(module); - PyModule_AddObject(module, name, reinterpret_cast<PyObject *>(type)); // steals reference + incarnateHelper(module, name, nameToFunc); } } void AddTypeCreationFunction(PyObject *module, const char *name, - TypeCreationFunction func, - const char *containerName) + TypeCreationFunction func) { - // This version could be delayed as well, but for the few cases - // we simply fetch the container type and insert directly. - AutoDecRef obContainerType(PyObject_GetAttrString(module, containerName)); - PyTypeObject *type = func(obContainerType); - PyObject_SetAttrString(obContainerType, name, reinterpret_cast<PyObject *>(type)); // steals reference -} + // - locate the module in the moduleTofuncs mapping + auto tableIter = moduleToFuncs.find(module); + assert(tableIter != moduleToFuncs.end()); + // - Assign the name/generating function tcStruct. + auto &nameToFunc = tableIter->second; + TypeCreationStruct tcStruct{func, {}}; + auto nit = nameToFunc.find(name); + if (nit == nameToFunc.end()) + nameToFunc.insert(std::make_pair(name, tcStruct)); + else + nit->second = tcStruct; -void AddTypeCreationFunction(PyObject *module, - const char *name, - TypeCreationFunction func, - const char *outerContainerName, - const char *innerContainerName) -{ - // This version has even more indirection. It is very rare, and - // we handle it directly. - AutoDecRef obOuterType(PyObject_GetAttrString(module, outerContainerName)); - AutoDecRef obInnerType(PyObject_GetAttrString(obOuterType, innerContainerName)); - PyTypeObject *type = func(obInnerType); - PyObject_SetAttrString(obInnerType, name, reinterpret_cast<PyObject *>(type)); // steals reference + checkIfShouldLoadImmediately(module, name, nameToFunc); } void AddTypeCreationFunction(PyObject *module, - const char *name, + const char *containerName, TypeCreationFunction func, - const char *containerName3, - const char *containerName2, - const char *containerName) + const char *namePath) { - // This version has even mode indirection. It is very rare, and - // we handle it directly. - AutoDecRef obContainerType3(PyObject_GetAttrString(module, containerName3)); - AutoDecRef obContainerType2(PyObject_GetAttrString(obContainerType3, containerName2)); - AutoDecRef obContainerType(PyObject_GetAttrString(obContainerType2, containerName)); - PyTypeObject *type = func(obContainerType); - PyObject_SetAttrString(obContainerType, name, reinterpret_cast<PyObject *>(type)); // steals reference + // - locate the module in the moduleTofuncs mapping + auto tableIter = moduleToFuncs.find(module); + assert(tableIter != moduleToFuncs.end()); + // - Assign the name/generating function tcStruct. + auto &nameToFunc = tableIter->second; + auto nit = nameToFunc.find(containerName); + + // - insert namePath into the subtype vector of the main type. + nit->second.subtypeNames.push_back(namePath); + // - insert it also as its own entry. + nit = nameToFunc.find(namePath); + TypeCreationStruct tcStruct{func, {}}; + if (nit == nameToFunc.end()) + nameToFunc.insert(std::make_pair(namePath, tcStruct)); + else + nit->second = tcStruct; + + checkIfShouldLoadImmediately(module, namePath, nameToFunc); } PyObject *import(const char *moduleName) @@ -455,6 +481,8 @@ PyObject *create(const char * /* modName */, void *moduleData) // into `sys.modules`. This can cause a race condition. // Insert the module early into the module dict to prevend recursion. PyDict_SetItemString(sysModules, PyModule_GetName(module), module); + // Clear the non-existing name cache because we have a new module. + Shiboken::Conversions::clearNegativeLazyCache(); return module; } diff --git a/sources/shiboken6/libshiboken/sbknumpy.cpp b/sources/shiboken6/libshiboken/sbknumpy.cpp index 2e1c64d73..b6422e73f 100644 --- a/sources/shiboken6/libshiboken/sbknumpy.cpp +++ b/sources/shiboken6/libshiboken/sbknumpy.cpp @@ -29,10 +29,8 @@ static void initNumPy() // Expanded from macro "import_array" in __multiarray_api.h // Make sure to read about the magic defines PY_ARRAY_UNIQUE_SYMBOL etc., // when changing this or spreading the code over several source files. - if (_import_array() < 0) { + if (_import_array() < 0) PyErr_Print(); - PyErr_Clear(); - } } #endif // HAVE_NUMPY diff --git a/sources/shiboken6/libshiboken/voidptr.cpp b/sources/shiboken6/libshiboken/voidptr.cpp index 7045b08b1..8bb3f6ac8 100644 --- a/sources/shiboken6/libshiboken/voidptr.cpp +++ b/sources/shiboken6/libshiboken/voidptr.cpp @@ -156,10 +156,9 @@ PyObject *SbkVoidPtrObject_int(PyObject *v) PyObject *toBytes(PyObject *self, PyObject * /* args */) { auto *sbkObject = reinterpret_cast<SbkVoidPtrObject *>(self); - if (sbkObject->size < 0) { - PyErr_SetString(PyExc_IndexError, "VoidPtr does not have a size set."); - return nullptr; - } + if (sbkObject->size < 0) + return PyErr_Format(PyExc_IndexError, "VoidPtr does not have a size set."); + PyObject *bytes = PyBytes_FromStringAndSize(reinterpret_cast<const char *>(sbkObject->cptr), sbkObject->size); Py_XINCREF(bytes); diff --git a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py index 11bda9779..5650e2bc1 100644 --- a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py +++ b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/enum_sig.py @@ -167,7 +167,7 @@ class ExactEnumerator(object): # find out how many functions create a signature sigs = list(_ for _ in functions if get_sig(_[1])) self.fmt.have_body = bool(subclasses or sigs or properties or enums or # noqa W:504 - init_signature or signals) + init_signature or signals or attributes) with self.fmt.klass(class_name, class_str): self.fmt.level += 1 diff --git a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py index 644f49ff6..ce12dd6c8 100644 --- a/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py +++ b/sources/shiboken6/shibokenmodule/files.dir/shibokensupport/signature/lib/pyi_generator.py @@ -179,6 +179,7 @@ FROM_IMPORTS = [ (None, ["builtins"]), (None, ["os"]), (None, ["enum"]), + ("collections.abc", ["Iterable"]), ("typing", sorted(typing.__all__)), ("PySide6.QtCore", ["PyClassProperty", "Signal", "SignalInstance"]), ("shiboken6", ["Shiboken"]), @@ -199,6 +200,10 @@ def filter_from_imports(from_struct, text): if (f"class {each}(") not in text: if re.search(rf"(\b|@){each}\b([^\s\(:]|\n)", text): lis.append(each) + # Search if a type is present in the return statement + # of function declarations: '... -> here:' + if re.search(rf"->.*{each}.*:", text): + lis.append(each) if not lis: nfs.pop() return nfs @@ -282,7 +287,7 @@ def generate_pyi(import_name, outpath, options): wr.print(f"from {mod} import {import_args}") wr.print() wr.print() - wr.print("NoneType = type(None)") + wr.print("NoneType: TypeAlias = type[None]") wr.print() else: wr.print(line) diff --git a/sources/shiboken6/shibokenmodule/shibokenmodule.cpp b/sources/shiboken6/shibokenmodule/shibokenmodule.cpp index 6feca9ea8..5c6219885 100644 --- a/sources/shiboken6/shibokenmodule/shibokenmodule.cpp +++ b/sources/shiboken6/shibokenmodule/shibokenmodule.cpp @@ -91,6 +91,19 @@ for (auto *o : setAll) { return listAll; // @snippet getallvalidwrappers +// @snippet dumptypegraph +const bool ok = Shiboken::BindingManager::instance().dumpTypeGraph(%1); +%PYARG_0 = %CONVERTTOPYTHON[bool](ok); +// @snippet dumptypegraph + +// @snippet dumpwrappermap +Shiboken::BindingManager::instance().dumpWrapperMap(); +// @snippet dumpwrappermap + +// @snippet dumpconverters +Shiboken::Conversions::dumpConverters(); +// @snippet dumpconverters + // @snippet init // Add __version__ and __version_info__ attributes to the module PyObject* version = PyTuple_New(5); diff --git a/sources/shiboken6/shibokenmodule/typesystem_shiboken.xml b/sources/shiboken6/shibokenmodule/typesystem_shiboken.xml index 2288ca7a4..acb522ecc 100644 --- a/sources/shiboken6/shibokenmodule/typesystem_shiboken.xml +++ b/sources/shiboken6/shibokenmodule/typesystem_shiboken.xml @@ -49,9 +49,22 @@ <inject-code file="shibokenmodule.cpp" snippet="getallvalidwrappers"/> </add-function> + <add-function signature="dumpTypeGraph(const char *@fileName@)" return-type="bool"> + <inject-code file="shibokenmodule.cpp" snippet="dumptypegraph"/> + </add-function> + + <add-function signature="dumpWrapperMap()"> + <inject-code file="shibokenmodule.cpp" snippet="dumpwrappermap"/> + </add-function> + + <add-function signature="dumpConverters()"> + <inject-code file="shibokenmodule.cpp" snippet="dumpconverters"/> + </add-function> + <extra-includes> <include file-name="sbkversion.h" location="local"/> <include file-name="voidptr.h" location="local"/> + <include file-name="sbkconverter_p.h" location="local"/> </extra-includes> <inject-code position="end" file="shibokenmodule.cpp" snippet="init"/> </typesystem> diff --git a/sources/shiboken6/tests/libother/othermultiplederived.h b/sources/shiboken6/tests/libother/othermultiplederived.h index a8e265388..cd9910687 100644 --- a/sources/shiboken6/tests/libother/othermultiplederived.h +++ b/sources/shiboken6/tests/libother/othermultiplederived.h @@ -10,7 +10,7 @@ class ObjectType; -class LIBOTHER_API OtherMultipleDerived : public MDerived1 +class LIBOTHER_API OtherMultipleDerived : public OtherBase, public MDerived1 { public: // this will use CppCopier from other module (bug#142) diff --git a/sources/shiboken6/tests/libsample/abstract.cpp b/sources/shiboken6/tests/libsample/abstract.cpp index 648cedcd0..0d67d8630 100644 --- a/sources/shiboken6/tests/libsample/abstract.cpp +++ b/sources/shiboken6/tests/libsample/abstract.cpp @@ -49,6 +49,18 @@ void Abstract::show(PrintFormat format) const std::cout << '>'; } +void Abstract::virtualWithOutParameter(int &x) const +{ + x = 42; +} + +int Abstract::callVirtualWithOutParameter() const +{ + int x; + virtualWithOutParameter(x); + return x; +} + void Abstract::callVirtualGettingEnum(PrintFormat p) { virtualGettingAEnum(p); diff --git a/sources/shiboken6/tests/libsample/abstract.h b/sources/shiboken6/tests/libsample/abstract.h index 1e62a9c51..4c1b98d90 100644 --- a/sources/shiboken6/tests/libsample/abstract.h +++ b/sources/shiboken6/tests/libsample/abstract.h @@ -74,6 +74,9 @@ public: virtual void hideFunction(HideType *arg) = 0; + virtual void virtualWithOutParameter(int &x) const; + int callVirtualWithOutParameter() const; + protected: virtual const char *className() const { return "Abstract"; } diff --git a/sources/shiboken6/tests/otherbinding/typediscovery_test.py b/sources/shiboken6/tests/otherbinding/typediscovery_test.py index 791d3bdce..39dc5cf0f 100644 --- a/sources/shiboken6/tests/otherbinding/typediscovery_test.py +++ b/sources/shiboken6/tests/otherbinding/typediscovery_test.py @@ -13,7 +13,8 @@ sys.path.append(os.fspath(Path(__file__).resolve().parents[1])) from shiboken_paths import init_paths init_paths() -from sample import Abstract, Base1, Derived +from sample import (Abstract, Base1, Derived, + MDerived1, SonOfMDerived1, MDerived3) from other import OtherMultipleDerived @@ -32,14 +33,18 @@ class TypeDiscoveryTest(unittest.TestCase): def testMultipleInheritance(self): obj = OtherMultipleDerived.createObject("Base1") self.assertEqual(type(obj), Base1) - # PYSIDE-868: In case of multiple inheritance, a factory - # function will return the base class wrapper. + # PYSIDE-868: In case of single line direct inheritance, + # a factory function will return the class wrapper + # of the derived class. obj = OtherMultipleDerived.createObject("MDerived1") - self.assertEqual(type(obj), Base1) + self.assertEqual(type(obj), MDerived1) obj = OtherMultipleDerived.createObject("SonOfMDerived1") - self.assertEqual(type(obj), Base1) + self.assertEqual(type(obj), SonOfMDerived1) obj = OtherMultipleDerived.createObject("MDerived3") - self.assertEqual(type(obj), Base1) + self.assertEqual(type(obj), MDerived3) + # PYSIDE-868: OtherMultipleDerived inherits + # OtherBase, Base1. In this case, a factory + # function will return the base class wrapper. obj = OtherMultipleDerived.createObject("OtherMultipleDerived") self.assertEqual(type(obj), Base1) diff --git a/sources/shiboken6/tests/samplebinding/derived_test.py b/sources/shiboken6/tests/samplebinding/derived_test.py index 418c990d3..346f29136 100644 --- a/sources/shiboken6/tests/samplebinding/derived_test.py +++ b/sources/shiboken6/tests/samplebinding/derived_test.py @@ -33,6 +33,15 @@ class Deviant(Derived): return 'Deviant' +class ImplementVirtualWithOutParameter(Derived): + def __init__(self, value): + super().__init__() + self._value = value + + def virtualWithOutParameter(self): + return self._value + + class DerivedTest(unittest.TestCase): '''Test case for Derived class''' @@ -122,6 +131,13 @@ class DerivedTest(unittest.TestCase): obj = DerivedUsingCt(42) self.assertEqual(obj.value(), 42) + def testVirtualWithOutParameter(self): + d = Derived() + self.assertEqual(d.callVirtualWithOutParameter(), 42) + + d = ImplementVirtualWithOutParameter(1) + self.assertEqual(d.callVirtualWithOutParameter(), 1) + if __name__ == '__main__': unittest.main() diff --git a/sources/shiboken6/tests/samplebinding/typesystem_sample.xml b/sources/shiboken6/tests/samplebinding/typesystem_sample.xml index 36134e649..e315e599e 100644 --- a/sources/shiboken6/tests/samplebinding/typesystem_sample.xml +++ b/sources/shiboken6/tests/samplebinding/typesystem_sample.xml @@ -571,6 +571,14 @@ <modify-function signature="hideFunction(HideType*)" remove="all"/> <modify-field name="toBeRenamedField" rename="renamedField"/> <modify-field name="readOnlyField" write="false"/> + <modify-function signature="virtualWithOutParameter(int&)const"> + <inject-code class="shell" position="override"> + x = virtualWithOutParameterPyOverride(gil, pyOverride.object()); + return; + </inject-code> + </modify-function> + <add-function signature="virtualWithOutParameterPyOverride()" + return-type="int" python-override="true"/> </object-type> <object-type name="Derived" polymorphic-id-expression="%1->type() == Derived::TpDerived"> |