Lazy Load: Fix polymorphic classes by identifying lazy groups

Classes with a polymorphicIdValue have an expression which
may reference a related class. We use that to identify
a lazy group, which has to be initialized at once.

This is now completely solved.

Fixes: PYSIDE-2675
Change-Id: I957a1b2b95d37b96cc2e98082fc7f92e601322cb
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
(cherry picked from commit 9f09e1dda0f167432110a22db6f9a5accf800734)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

5 files changed, 250 insertions, 21 deletions

diff --git a/sources/pyside6/tests/QtCore/qobject_event_filter_test.py b/sources/pyside6/tests/QtCore/qobject_event_filter_test.py
index 1cf07fbab..ab7a1b6ad 100644
--- a/sources/pyside6/tests/QtCore/qobject_event_filter_test.py
+++ b/sources/pyside6/tests/QtCore/qobject_event_filter_test.py
@@ -63,6 +63,17 @@ class FilteredObject(QObject):
             self.app.quit()
 
 
+class PolymorphicIdFilterObject(QObject):
+    """PYSIDE-2675: Check whether QChildEvent.added() is accessible via PolymorphicId"""
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self.added = False
+
+    def event(self, event):
+        self.added = event.added()
+        return False
+
+
 class TestQObjectEventFilterPython(UsesQApplication):
     '''QObject.eventFilter - Reimplemented in python
     Filters 5 TimerEvents and then bypasses the other events to the
@@ -93,6 +104,11 @@ class TestQObjectEventFilterPython(UsesQApplication):
         self.assertEqual(filtered.times_called, 5)
         self.assertEqual(self.obj_filter.events_handled, 5)
 
+    def testPolymorphicId(self):
+        testObject = PolymorphicIdFilterObject()
+        t2 = QObject(testObject)  # noqa: F841
+        self.assertTrue(testObject.added)
+
     @unittest.skipUnless(hasattr(sys, "getrefcount"), f"{sys.implementation.name} has no refcount")
     def testInstallEventFilterRefCountAfterDelete(self):
         '''Bug 910 - installEventFilter() increments reference count on target object
diff --git a/sources/shiboken6/generator/shiboken/cppgenerator.cpp b/sources/shiboken6/generator/shiboken/cppgenerator.cpp
index 059ab7be3..bfde1ff53 100644
--- a/sources/shiboken6/generator/shiboken/cppgenerator.cpp
+++ b/sources/shiboken6/generator/shiboken/cppgenerator.cpp
@@ -5759,6 +5759,18 @@ void CppGenerator::writeInitQtMetaTypeFunctionBody(TextStream &s, const Generato
     }
 }
 
+// Note: This is an incomplete function that does not work without a
+// surrounding functionality. It's purpose is to make sure that exactly
+// this expression always is used, although the first clause is never true in PySide.
+// Without it can create false positives.
+static AbstractMetaClassCPtr getPolymorphicBaseClass(const AbstractMetaClassCPtr &metaClass)
+{
+  auto baseClass = metaClass;
+  while (!baseClass->typeEntry()->isPolymorphicBase() && baseClass->baseClass())
+        baseClass = baseClass->baseClass();
+  return baseClass;
+}
+
 void CppGenerator::replacePolymorphicIdPlaceHolders(const AbstractMetaClassCPtr &metaClass,
                                                     QString *id)
 {
@@ -5768,9 +5780,7 @@ void CppGenerator::replacePolymorphicIdPlaceHolders(const AbstractMetaClassCPtr
         id->replace("%1"_L1, replacement);
     }
     if (id->contains("%B"_L1)) {
-        auto baseClass = metaClass;
-        while (!baseClass->typeEntry()->isPolymorphicBase() && baseClass->baseClass())
-            baseClass = baseClass->baseClass();
+        auto baseClass = getPolymorphicBaseClass(metaClass);
         QString replacement = " reinterpret_cast< "_L1 + m_gsp + baseClass->qualifiedCppName()
                               + " *>(cptr)"_L1;
         id->replace("%B"_L1, replacement);
@@ -5999,19 +6009,15 @@ void CppGenerator::writeNbBoolFunction(const GeneratorContext &context,
 void CppGenerator::writeInitFunc(TextStream &declStr, TextStream &callStr,
                                  const QString &initFunctionName,
                                  const TypeEntryCPtr &enclosingEntry,
-                                 const QString &pythonName, bool lazy)
+                                 const QString &pythonName,
+                                 const QString &lazyGroup)
 {
+
     const QString functionName = "init_"_L1 + initFunctionName;
     const bool hasParent = enclosingEntry && enclosingEntry->type() != TypeEntry::TypeSystemType;
     declStr << "PyTypeObject *" << functionName << "(PyObject *"
         << (hasParent ? "enclosingClass" : "module") << ");\n";
-
-    if (!lazy) {
-        const QString enclosing = hasParent
-            ? "reinterpret_cast<PyObject *>("_L1 + cpythonTypeNameExt(enclosingEntry) + u')'
-            : "module"_L1;
-        callStr << functionName << '(' << enclosing << ");\n";
-    } else if (hasParent) {
+    if (hasParent) {
         const QString &enclosingName = enclosingEntry->name();
         const auto parts = QStringView{enclosingName}.split(u"::", Qt::SkipEmptyParts);
         callStr << "Shiboken::Module::AddTypeCreationFunction("
@@ -6023,9 +6029,13 @@ void CppGenerator::writeInitFunc(TextStream &declStr, TextStream &callStr,
         }
         callStr << "\");\n";
     } else {
-        callStr << "Shiboken::Module::AddTypeCreationFunction("
-            << "module, \"" << pythonName << "\", "
-            << "init_" << initFunctionName << ");\n";
+        const char *funcName = lazyGroup.isEmpty()
+            ? "AddTypeCreationFunction" : "AddGroupedTypeCreationFunction";
+        callStr << "Shiboken::Module::" << funcName  << "(module, \""
+                << pythonName << "\", " << functionName;
+        if (!lazyGroup.isEmpty())
+            callStr << ", \"" << lazyGroup << "\"";
+        callStr << ");\n";
     }
 }
 
@@ -6042,6 +6052,12 @@ static void writeSubModuleHandling(TextStream &s, const QString &moduleName,
         << indent << "return nullptr;\n" << outdent << outdent << "}\n";
 }
 
+static AbstractMetaClassCPtr getLazyGroupBaseClass(const AbstractMetaClassCPtr &metaClass)
+{
+    return needsTypeDiscoveryFunction(metaClass) && !isQObject(metaClass)
+           ? getPolymorphicBaseClass(metaClass) : nullptr;
+}
+
 bool CppGenerator::finishGeneration()
 {
     //Generate CPython wrapper file
@@ -6073,6 +6089,18 @@ bool CppGenerator::finishGeneration()
         s_globalFunctionDef << methodDefinitionEntries(overloadData);
     }
 
+    // Collect the lazy group base classes first, because we need to add
+    // these base classes into the group, too.
+    std::set<AbstractMetaClassCPtr> lazyGroupBaseClasses{};
+    for (const auto &cls : api().classes()){
+        auto te = cls->typeEntry();
+        if (shouldGenerate(te)) {
+            auto lazyGroupCls = getLazyGroupBaseClass(cls);
+            if (lazyGroupCls)
+                lazyGroupBaseClasses.insert(lazyGroupCls);
+        }
+    }
+
     AbstractMetaClassCList classesWithStaticFields;
     for (const auto &cls : api().classes()){
         auto te = cls->typeEntry();
@@ -6082,9 +6110,17 @@ bool CppGenerator::finishGeneration()
                 s_classInitDecl << te->configCondition() << '\n';
                 s_classPythonDefines << te->configCondition() << '\n';
             }
+            auto lazyGroupCls = getLazyGroupBaseClass(cls);
+            if (!lazyGroupCls) {
+                auto it = lazyGroupBaseClasses.find(cls);
+                if (it != lazyGroupBaseClasses.end())
+                    lazyGroupCls = cls;
+            }
             writeInitFunc(s_classInitDecl, s_classPythonDefines,
                           getSimpleClassInitFunctionName(cls),
-                          targetLangEnclosingEntry(te), cls->name());
+                          targetLangEnclosingEntry(te), cls->name(),
+                          lazyGroupCls ? lazyGroupCls->typeEntry()->qualifiedTargetLangName()
+                                       : QString());
             if (cls->hasStaticFields()) {
                 s_classInitDecl << "PyTypeObject *"
                     << getSimpleClassStaticFieldsInitFunctionName(cls) << "(PyObject *module);\n";
diff --git a/sources/shiboken6/generator/shiboken/cppgenerator.h b/sources/shiboken6/generator/shiboken/cppgenerator.h
index 00ae31f9a..1b4ebf803 100644
--- a/sources/shiboken6/generator/shiboken/cppgenerator.h
+++ b/sources/shiboken6/generator/shiboken/cppgenerator.h
@@ -63,7 +63,8 @@ private:
     static void writeInitFunc(TextStream &declStr, TextStream &callStr,
                               const QString &initFunctionName,
                               const TypeEntryCPtr &enclosingEntry,
-                              const QString &pythonName, bool lazy = true);
+                              const QString &pythonName,
+                              const QString &lazyGroup = {});
     static void writeCacheResetNative(TextStream &s, const GeneratorContext &classContext);
     void writeConstructorNative(TextStream &s, const GeneratorContext &classContext,
                                 const AbstractMetaFunctionCPtr &func) const;
diff --git a/sources/shiboken6/libshiboken/sbkmodule.cpp b/sources/shiboken6/libshiboken/sbkmodule.cpp
index 4153df27f..428ebc5f0 100644
--- a/sources/shiboken6/libshiboken/sbkmodule.cpp
+++ b/sources/shiboken6/libshiboken/sbkmodule.cpp
@@ -31,6 +31,82 @@ static ModuleTypesMap moduleTypes;
 static ModuleConvertersMap moduleConverters;
 static ModuleToFuncsMap moduleToFuncs;
 
+/*****************************************************************************
+
+    How Do Lazy Groups Work?
+    ------------------------
+
+When polymorphic classes are in use, then we have to deal with classes
+which might not yet be visible. They are located by type discovery functions.
+
+In order to allow these functions to do their work, the needed classes
+must be existent in time. That is what lazy groups are doing:
+
+    They provide the minimum set of sufficient classes that might be
+    needed by the type discovery functions.
+
+Lazy groups are determined by the cppgenerator when polymorphic functions
+are analyzed. They are communicated to sbkmodule by a LazyGroup parameter.
+
+
+The Idea
+--------
+
+When the creating functions of a module are collected for lazy evaluation,
+we build a data structure that keeps the lazy groups together. In this
+phase, there is no other special action.
+
+As soon as a single element of a group gets used by getattr, the whole action
+takes place:
+
+- All elements in the same group are touched by getattr as well, meaning
+- The whole group becomes existent at once.
+
+After that action, a group is not evaluated again because it is switched
+to immediate mode.
+
+
+Importing Another Module
+------------------------
+
+If a group has not been touched and a new module with new group members
+is added, the elements are simply accumulated in the group as before.
+
+If a group has already been touched, then it is in immediate mode, and all
+new elements must be created as well.
+
+
+The Implementation
+------------------
+
+There is a structure LazyPool which contains
+
+    - classToGroup  members->group      n:1
+
+    - groupState    groups->state       1:1
+
+The classToGroup is the central structure that controls group membership.
+The groupState enum makes sure that the group members are initialized
+together at once and only once.
+
+*****************************************************************************/
+
+/// Lazy Groups
+///
+/// Accumulated in lists, but completely incarnated if a member is accessed.
+struct LazyGroupStructure {
+    enum State {
+        NoGroup,                // No group at all
+        FirstSeen,              // Seen first by getattr
+        Handled                 // Normal processing like no group
+    };
+
+    std::unordered_map<std::string, std::string> classToGroup;
+    std::unordered_map<std::string, State> groupState;
+};
+
+static LazyGroupStructure LazyPool;
+
 namespace Shiboken
 {
 namespace Module
@@ -99,7 +175,8 @@ static PyTypeObject *incarnateType(PyObject *module, const char *name,
     Py_INCREF(res);
     PyModule_AddObject(module, name, res);   // steals reference
     // - remove the entry, if not by something cleared.
-    if (!nameToFunc.empty())
+    funcIter = nameToFunc.find(name);
+    if (funcIter != nameToFunc.end())
         nameToFunc.erase(funcIter);
     // - return the PyTypeObject.
     return type;
@@ -144,10 +221,34 @@ void resolveLazyClasses(PyObject *module)
 // PYSIDE-2404: Override the gettattr function of modules.
 static getattrofunc origModuleGetattro{};
 
+static LazyGroupStructure::State getGroupStateAndLock(const std::string &groupName)
+{
+    if (groupName.empty())
+        return LazyGroupStructure::NoGroup;
+
+    auto stateIt = LazyPool.groupState.find(groupName);
+    if (stateIt == LazyPool.groupState.end()) {
+        LazyPool.groupState.insert(std::make_pair(groupName, LazyGroupStructure::FirstSeen));
+        return LazyGroupStructure::FirstSeen;
+    }
+
+    auto result = stateIt->second;
+    if (stateIt->second == LazyGroupStructure::FirstSeen)
+        stateIt->second = LazyGroupStructure::Handled;
+    return result;
+}
+
+static std::string getGroupName(const std::string &key)
+{
+    auto git = LazyPool.classToGroup.find(key);
+    return git != LazyPool.classToGroup.end() ? git->second : std::string{};
+}
+
 // PYSIDE-2404: Use the patched module getattr to do on-demand initialization.
 //              This modifies _all_ modules but should have no impact.
 static PyObject *PyModule_lazyGetAttro(PyObject *module, PyObject *name)
 {
+    static auto *sysModules = PyImport_GetModuleDict();
     // - check if the attribute is present and return it.
     auto *attr = PyObject_GenericGetAttr(module, name);
     // - we handle AttributeError, only.
@@ -161,9 +262,19 @@ static PyObject *PyModule_lazyGetAttro(PyObject *module, PyObject *name)
     if (tableIter == moduleToFuncs.end())
         return origModuleGetattro(module, name);
 
-    // - locate the name and retrieve the generating function
     const char *attrNameStr = Shiboken::String::toCString(name);
+    auto key = std::string(PyModule_GetName(module)) + '.' + attrNameStr;
+
+    // - see if we have a group. Initializes the process if seen first.
+    const auto &groupName = getGroupName(key);
+    auto state = getGroupStateAndLock(groupName);
+
+    // - retrieve the generating function
     auto &nameToFunc = tableIter->second;
+
+    // - make sure that the state gets past possible action
+    getGroupStateAndLock(groupName);
+
     // - create the real type (incarnateType checks this)
     auto *type = incarnateType(module, attrNameStr, nameToFunc);
     auto *ret = reinterpret_cast<PyObject *>(type);
@@ -173,6 +284,31 @@ static PyObject *PyModule_lazyGetAttro(PyObject *module, PyObject *name)
         return origModuleGetattro(module, name);
     }
 
+    if (state != LazyPool.FirstSeen)
+        return ret;
+
+    // The state is now FirstSeen. So we are the one instance who handles it
+    // and no one else again.
+    // This was crucial to avoid duplication in recursive calls.
+
+    // - incarnate the whole group
+    for (auto it = LazyPool.classToGroup.cbegin(), end = LazyPool.classToGroup.cend();
+         it != end; ++it) {
+        if (it->second == groupName) {
+            // - obtain the module name
+            std::string_view names(it->first);
+            const bool usePySide = names.compare(0, 8, "PySide6.") == 0;
+            auto dotPos = usePySide ? names.find('.', 8) : names.find('.');
+            auto startPos = dotPos + 1;
+            AutoDecRef modName(String::fromCppStringView(names.substr(0, dotPos)));
+            module = PyDict_GetItem(sysModules, modName);
+            assert(module != nullptr);
+            // - isolate the type name
+            auto typeName = names.substr(startPos);
+            // - create the type
+            PyModule_lazyGetAttro(module, String::fromCString(typeName.data()));
+        }
+    }
     return ret;
 }
 
@@ -292,9 +428,24 @@ static bool shouldLazyLoad(PyObject *module)
     return std::strncmp(modName, "PySide6.", 8) == 0;
 }
 
-void AddTypeCreationFunction(PyObject *module,
-                             const char *name,
-                             TypeCreationFunction func)
+static bool groupMaterialized(const char *group)
+{
+    auto iter = LazyPool.groupState.find(group);
+    return iter != LazyPool.groupState.end();
+}
+
+static void addToGroup(PyObject *module, const char *shortName, const char *group)
+{
+    auto name = std::string(PyModule_GetName(module)) + '.' + shortName;
+
+    // - insert into the group members
+    LazyPool.classToGroup.insert(std::make_pair(name, group));
+}
+
+static void addTypeCreationFunction(PyObject *module,
+                                    const char *name,
+                                    TypeCreationFunction func,
+                                    const char *lazyGroup = nullptr)
 {
     static const char *flag = getenv("PYSIDE6_OPTION_LAZY");
     static const int value = flag != nullptr ? std::atoi(flag) : 1;
@@ -311,6 +462,10 @@ void AddTypeCreationFunction(PyObject *module,
     else
         nit->second = pair;
 
+    const bool hasLazyGroup = lazyGroup != nullptr;
+    if (hasLazyGroup)
+        addToGroup(module, name, lazyGroup);
+
     // PYSIDE-2404: Lazy Loading
     //
     // Options:
@@ -323,6 +478,7 @@ void AddTypeCreationFunction(PyObject *module,
     if (value == 0                                  // completely disabled
         || canNotLazyLoad(module)                   // for some reason we cannot lazy load
         || (value == 1 && !shouldLazyLoad(module))  // not a known module
+        || (hasLazyGroup && groupMaterialized(lazyGroup))
         ) {
         PyTypeObject *type = func(module);
         PyModule_AddObject(module, name, reinterpret_cast<PyObject *>(type));   // steals reference
@@ -331,6 +487,21 @@ void AddTypeCreationFunction(PyObject *module,
 
 void AddTypeCreationFunction(PyObject *module,
                              const char *name,
+                             TypeCreationFunction func)
+{
+    addTypeCreationFunction(module, name, func);
+}
+
+void AddGroupedTypeCreationFunction(PyObject *module,
+                                    const char *name,
+                                    TypeCreationFunction func,
+                                    const char *lazyGroup)
+{
+    addTypeCreationFunction(module, name, func, lazyGroup);
+}
+
+void AddTypeCreationFunction(PyObject *module,
+                             const char *name,
                              TypeCreationFunction func,
                              const char *containerName)
 {
diff --git a/sources/shiboken6/libshiboken/sbkmodule.h b/sources/shiboken6/libshiboken/sbkmodule.h
index 1b3de33b7..7acce68dd 100644
--- a/sources/shiboken6/libshiboken/sbkmodule.h
+++ b/sources/shiboken6/libshiboken/sbkmodule.h
@@ -51,6 +51,11 @@ LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module,
                                              const char *name,
                                              TypeCreationFunction func);
 
+LIBSHIBOKEN_API void AddGroupedTypeCreationFunction(PyObject *module,
+                                                    const char *name,
+                                                    TypeCreationFunction func,
+                                                    const char *lazyGroup);
+
 LIBSHIBOKEN_API void AddTypeCreationFunction(PyObject *module,
                                              const char *name,
                                              TypeCreationFunction func,