shiboken6: Lazy-initialize the dependency graph6.7

Change the dependency graph in the BindingManager from
PyTypeObject *-based nodes to nodes which are wrapping the
TypeInitStruct * and hashing on the type name. This allows for
creating the types on demand when walking along the edges and calling
the type discovery functions.

This only creates the required types instead of the entire
lazy group of polymorphic classes.

The graph is now populated by from the generated code using a
function named initInheritance() instead of dynamically
from introduceWrapperType.

Task-number: PYSIDE-2404
Task-number: PYSIDE-2675
Change-Id: I030d4957c221f4defbb7cc52a6927287b70d9864
Reviewed-by: Christian Tismer <tismer@stackless.com>
(cherry picked from commit b5cbb40f4593e8576deb4632c70399cf30b9bc35)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

5 files changed, 98 insertions, 26 deletions

diff --git a/sources/shiboken6/generator/shiboken/cppgenerator.cpp b/sources/shiboken6/generator/shiboken/cppgenerator.cpp
index ad4071c35..6594a7365 100644
--- a/sources/shiboken6/generator/shiboken/cppgenerator.cpp
+++ b/sources/shiboken6/generator/shiboken/cppgenerator.cpp
@@ -61,6 +61,7 @@ static const char shibokenErrorsOccurred[] = "Shiboken::Errors::occurred() != nu
 static constexpr auto virtualMethodStaticReturnVar = "result"_L1;
 
 static constexpr auto sbkObjectTypeF = "SbkObject_TypeF()"_L1;
+static const char initInheritanceFunction[] = "initInheritance";
 
 static QString mangleName(QString name)
 {
@@ -4260,6 +4261,12 @@ QString CppGenerator::writeContainerConverterInitialization(TextStream &s,
     return converter;
 }
 
+QString CppGenerator::typeInitStruct(const TypeEntryCPtr &te)
+{
+    return cppApiVariableName(te->targetLangPackage()) + u'['
+        + getTypeIndexVariableName(te) + u']';
+}
+
 void CppGenerator::writeExtendedConverterInitialization(TextStream &s,
                                                         const TypeEntryCPtr &externalType,
                                                         const AbstractMetaClassCList &conversions)
@@ -4267,15 +4274,13 @@ void CppGenerator::writeExtendedConverterInitialization(TextStream &s,
     s << "// Extended implicit conversions for " << externalType->qualifiedTargetLangName()
       << ".\n";
     for (const auto &sourceClass : conversions) {
-        const QString converterVar = cppApiVariableName(externalType->targetLangPackage()) + u'['
-            + getTypeIndexVariableName(externalType) + u']';
         QString sourceTypeName = fixedCppTypeName(sourceClass->typeEntry());
         QString targetTypeName = fixedCppTypeName(externalType);
         QString toCpp = pythonToCppFunctionName(sourceTypeName, targetTypeName);
         QString isConv = convertibleToCppFunctionName(sourceTypeName, targetTypeName);
         if (!externalType->isPrimitive())
             s << cpythonTypeNameExt(externalType) << ";\n";
-        writeAddPythonToCppConversion(s, converterVar, toCpp, isConv);
+        writeAddPythonToCppConversion(s, typeInitStruct(externalType), toCpp, isConv);
     }
 }
 
@@ -5473,6 +5478,27 @@ QStringList CppGenerator::pyBaseTypes(const AbstractMetaClassCPtr &metaClass)
     return result;
 }
 
+void CppGenerator::writeInitInheritance(TextStream &s) const
+{
+    s << "static void " << initInheritanceFunction << "()\n{\n" << indent
+        << "auto &bm = Shiboken::BindingManager::instance();\n"
+        << sbkUnusedVariableCast("bm");
+    for (const auto &cls : api().classes()){
+        auto te = cls->typeEntry();
+        if (shouldGenerate(te)) {
+            const auto &baseEntries = pyBaseTypeEntries(cls);
+            if (!baseEntries.isEmpty()) {
+                const QString childTypeInitStruct = typeInitStruct(cls->typeEntry());
+                for (const auto &baseEntry : baseEntries) {
+                    s << "bm.addClassInheritance(&" << typeInitStruct(baseEntry) << ",\n"
+                      << Pad(' ', 23) << '&' << childTypeInitStruct << ");\n";
+                }
+            }
+        }
+    }
+    s << outdent << "}\n\n";
+}
+
 void CppGenerator::writeClassRegister(TextStream &s,
                                       const AbstractMetaClassCPtr &metaClass,
                                       const GeneratorContext &classContext,
@@ -6337,6 +6363,8 @@ bool CppGenerator::finishGeneration()
     // PYSIDE-510: Create a signatures string for the introspection feature.
     writeSignatureStrings(s, signatureStream.toString(), moduleName(), "global functions");
 
+    writeInitInheritance(s);
+
     // Write module init function
     const QString globalModuleVar = pythonModuleObjectName();
     s << "extern \"C\" LIBSHIBOKEN_EXPORT PyObject *PyInit_"
@@ -6474,7 +6502,8 @@ bool CppGenerator::finishGeneration()
         }
     }
 
-    s << "\nif (" << shibokenErrorsOccurred << ") {\n" << indent
+    s << '\n' << initInheritanceFunction << "();\n"
+        << "\nif (" << shibokenErrorsOccurred << ") {\n" << indent
         << "PyErr_Print();\n"
         << "Py_FatalError(\"can't initialize module " << moduleName() << "\");\n"
         << outdent << "}\n";
diff --git a/sources/shiboken6/generator/shiboken/cppgenerator.h b/sources/shiboken6/generator/shiboken/cppgenerator.h
index 1ad576895..a31c2ca14 100644
--- a/sources/shiboken6/generator/shiboken/cppgenerator.h
+++ b/sources/shiboken6/generator/shiboken/cppgenerator.h
@@ -395,6 +395,7 @@ private:
     static void writeSignatureStrings(TextStream &s, const QString &signatures,
                                       const QString &arrayName,
                                       const char *comment);
+    void writeInitInheritance(TextStream &s) const;
     void writeClassRegister(TextStream &s,
                             const AbstractMetaClassCPtr &metaClass,
                             const GeneratorContext &classContext,
@@ -480,6 +481,8 @@ private:
                                                          const AbstractMetaType &type,
                                                          const ApiExtractorResult &api);
     void writeSmartPointerConverterInitialization(TextStream &s, const AbstractMetaType &ype) const;
+
+    static QString typeInitStruct(const TypeEntryCPtr &te);
     static void writeExtendedConverterInitialization(TextStream &s,
                                                      const TypeEntryCPtr &externalType,
                                                      const AbstractMetaClassCList &conversions);
diff --git a/sources/shiboken6/libshiboken/basewrapper.cpp b/sources/shiboken6/libshiboken/basewrapper.cpp
index f34e053ca..15c79479e 100644
--- a/sources/shiboken6/libshiboken/basewrapper.cpp
+++ b/sources/shiboken6/libshiboken/basewrapper.cpp
@@ -1007,11 +1007,6 @@ introduceWrapperType(PyObject *enclosingObject,
 
     auto *type = SbkType_FromSpecBasesMeta(typeSpec, bases, SbkObjectType_TypeF());
 
-    for (Py_ssize_t i = 0; i < basesSize; ++i) {
-        auto *st = reinterpret_cast<PyTypeObject *>(PySequence_Fast_GET_ITEM(bases, i));
-        BindingManager::instance().addClassInheritance(st, type);
-    }
-
     auto sotp = PepType_SOTP(type);
     if (wrapperFlags & DeleteInMainThread)
         sotp->delete_in_main_thread = 1;
diff --git a/sources/shiboken6/libshiboken/bindingmanager.cpp b/sources/shiboken6/libshiboken/bindingmanager.cpp
index 60460c6ab..83c927ae5 100644
--- a/sources/shiboken6/libshiboken/bindingmanager.cpp
+++ b/sources/shiboken6/libshiboken/bindingmanager.cpp
@@ -7,6 +7,7 @@
 #include "bindingmanager.h"
 #include "gilstate.h"
 #include "helper.h"
+#include "sbkmodule.h"
 #include "sbkstring.h"
 #include "sbkstaticstrings.h"
 #include "sbkfeature_base.h"
@@ -21,6 +22,29 @@
 #include <unordered_map>
 #include <unordered_set>
 
+// GraphNode for the dependency graph. It keeps a pointer to
+// the TypeInitStruct to be able to lazily create the type and hashes
+// by the full type name.
+struct GraphNode
+{
+    explicit GraphNode(Shiboken::Module::TypeInitStruct *i) : name(i->fullName), initStruct(i) {}
+    explicit GraphNode(const char *n) : name(n), initStruct(nullptr) {} // Only for searching
+
+    std::string_view name;
+    Shiboken::Module::TypeInitStruct *initStruct;
+
+    friend bool operator==(const GraphNode &n1, const GraphNode &n2) { return n1.name == n2.name; }
+    friend bool operator!=(const GraphNode &n1, const GraphNode &n2) { return n1.name != n2.name; }
+};
+
+template <>
+struct std::hash<GraphNode> {
+    size_t operator()(const GraphNode &n) const noexcept
+    {
+        return std::hash<std::string_view>{}(n.name);
+    }
+};
+
 namespace Shiboken
 {
 
@@ -56,28 +80,45 @@ public:
     }
 };
 
-class Graph : public BaseGraph<PyTypeObject *>
+class Graph : public BaseGraph<GraphNode>
 {
 public:
-    Graph() = default;
+    using TypeCptrPair = BindingManager::TypeCptrPair;
 
-    BindingManager::TypeCptrPair identifyType(void *cptr, PyTypeObject *type, PyTypeObject *baseType) const;
+    TypeCptrPair identifyType(void *cptr, PyTypeObject *type, PyTypeObject *baseType) const
+    {
+        return identifyType(cptr, GraphNode(type->tp_name), type, baseType);
+    }
 
     bool dumpTypeGraph(const char *fileName) const;
+
+private:
+    TypeCptrPair identifyType(void *cptr, const GraphNode &typeNode, PyTypeObject *type,
+                              PyTypeObject *baseType) const;
 };
 
-BindingManager::TypeCptrPair Graph::identifyType(void *cptr, PyTypeObject *type, PyTypeObject *baseType) const
+Graph::TypeCptrPair Graph::identifyType(void *cptr,
+                                        const GraphNode &typeNode, PyTypeObject *type,
+                                        PyTypeObject *baseType) const
 {
-    auto edgesIt = m_edges.find(type);
+    assert(typeNode.initStruct != nullptr || type != nullptr);
+    auto edgesIt = m_edges.find(typeNode);
     if (edgesIt != m_edges.end()) {
         const NodeList &adjNodes = edgesIt->second;
-        for (PyTypeObject *node : adjNodes) {
-            auto newType = identifyType(cptr, node, baseType);
+        for (const auto &node : adjNodes) {
+            auto newType = identifyType(cptr, node, nullptr, baseType);
             if (newType.first != nullptr)
                 return newType;
         }
     }
 
+    if (type == nullptr) {
+        if (typeNode.initStruct->type == nullptr) // Layzily create type
+            type = Shiboken::Module::get(*typeNode.initStruct);
+        else
+            type = typeNode.initStruct->type;
+    }
+
     auto *sotp = PepType_SOTP(type);
     if (sotp->type_discovery != nullptr) {
         if (void *derivedCPtr = sotp->type_discovery(cptr, baseType))
@@ -86,9 +127,8 @@ BindingManager::TypeCptrPair Graph::identifyType(void *cptr, PyTypeObject *type,
     return {nullptr, nullptr};
 }
 
-static void formatDotNode(const char *nameC, std::ostream &file)
+static void formatDotNode(std::string_view name, std::ostream &file)
 {
-    std::string_view name(nameC);
     auto lastDot = name.rfind('.');
     file << "    \"" << name << "\" [ label=";
     if (lastDot != std::string::npos) {
@@ -109,15 +149,15 @@ bool Graph::dumpTypeGraph(const char *fileName) const
     file << "digraph D {\n";
 
     // Define nodes with short names
-    for (const auto *node : nodeSet())
-        formatDotNode(node->tp_name, file);
+    for (const auto &node : nodeSet())
+        formatDotNode(node.name, file);
 
     // Write edges
     for (const auto &p : m_edges) {
-        auto *node1 = p.first;
+        const auto &node1 = p.first;
         const NodeList &nodeList = p.second;
-        for (const PyTypeObject *node2 : nodeList)
-            file << "    \"" << node2->tp_name << "\" -> \"" << node1->tp_name << "\"\n";
+        for (const auto &node2 : nodeList)
+            file << "    \"" << node2.name << "\" -> \"" << node1.name << "\"\n";
     }
     file << "}\n";
     return true;
@@ -380,9 +420,10 @@ PyObject *BindingManager::getOverride(const void *cptr,
     return nullptr;
 }
 
-void BindingManager::addClassInheritance(PyTypeObject *parent, PyTypeObject *child)
+void BindingManager::addClassInheritance(Module::TypeInitStruct *parent,
+                                         Module::TypeInitStruct *child)
 {
-    m_d->classHierarchy.addEdge(parent, child);
+    m_d->classHierarchy.addEdge(GraphNode(parent), GraphNode(child));
 }
 
 BindingManager::TypeCptrPair BindingManager::findDerivedType(void *cptr, PyTypeObject *type) const
diff --git a/sources/shiboken6/libshiboken/bindingmanager.h b/sources/shiboken6/libshiboken/bindingmanager.h
index 281de2c73..54c4e486a 100644
--- a/sources/shiboken6/libshiboken/bindingmanager.h
+++ b/sources/shiboken6/libshiboken/bindingmanager.h
@@ -15,6 +15,10 @@ struct SbkObject;
 namespace Shiboken
 {
 
+namespace Module {
+struct TypeInitStruct;
+}
+
 struct DestructorEntry;
 
 using ObjectVisitor = void (*)(SbkObject *, void *);
@@ -40,7 +44,7 @@ public:
     SbkObject *retrieveWrapper(const void *cptr);
     PyObject *getOverride(const void *cptr, PyObject *nameCache[], const char *methodName);
 
-    void addClassInheritance(PyTypeObject *parent, PyTypeObject *child);
+    void addClassInheritance(Module::TypeInitStruct *parent, Module::TypeInitStruct *child);
     /// Try to find the correct type of cptr via type discovery knowing that it's at least
     /// of type \p type. If a derived class is found, it returns a cptr cast to the type
     /// (which may be different in case of  multiple inheritance.