shiboken: Clean up handling of enclosing class

Add a EnclosingClassMixin which provides
targetLangEnclosingClass() by walking up the
hierarchy and remove duplicated code.

Task-number: PYSIDE-990
Task-number: PYSIDE-1074
Change-Id: Ia69b687a34e735f74265d72727e6bad3216c33f0
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>

5 files changed, 32 insertions, 62 deletions

diff --git a/sources/shiboken2/ApiExtractor/abstractmetalang.cpp b/sources/shiboken2/ApiExtractor/abstractmetalang.cpp
index 39364b3f5..19a4c7849 100644
--- a/sources/shiboken2/ApiExtractor/abstractmetalang.cpp
+++ b/sources/shiboken2/ApiExtractor/abstractmetalang.cpp
@@ -1858,6 +1858,18 @@ const AbstractMetaFunction *AbstractMetaField::setter() const
     return m_setter;
 }
 
+const AbstractMetaClass *EnclosingClassMixin::targetLangEnclosingClass() const
+{
+    auto result = m_enclosingClass;
+    while (result) {
+        auto typeEntry = result->typeEntry();
+        if (typeEntry->codeGeneration() != TypeEntry::GenerateForSubclass)
+            break;
+        result = result->enclosingClass();
+    }
+    return result;
+}
+
 const AbstractMetaFunction *AbstractMetaField::getter() const
 {
     if (!m_getter) {
diff --git a/sources/shiboken2/ApiExtractor/abstractmetalang.h b/sources/shiboken2/ApiExtractor/abstractmetalang.h
index beafec314..3ef135cf9 100644
--- a/sources/shiboken2/ApiExtractor/abstractmetalang.h
+++ b/sources/shiboken2/ApiExtractor/abstractmetalang.h
@@ -704,25 +704,26 @@ private:
     friend class AbstractMetaClass;
 };
 
+class EnclosingClassMixin {
+public:
+    const AbstractMetaClass *enclosingClass() const { return m_enclosingClass; }
+    void setEnclosingClass(const AbstractMetaClass *cls) { m_enclosingClass = cls; }
+    const AbstractMetaClass *targetLangEnclosingClass() const;
+
+private:
+     const AbstractMetaClass *m_enclosingClass = nullptr;
+};
+
 #ifndef QT_NO_DEBUG_STREAM
 QDebug operator<<(QDebug d, const AbstractMetaArgument *aa);
 #endif
 
-class AbstractMetaField : public AbstractMetaVariable, public AbstractMetaAttributes
+class AbstractMetaField : public AbstractMetaVariable, public AbstractMetaAttributes, public EnclosingClassMixin
 {
 public:
     AbstractMetaField();
     ~AbstractMetaField();
 
-    const AbstractMetaClass *enclosingClass() const
-    {
-        return m_class;
-    }
-    void setEnclosingClass(const AbstractMetaClass *cls)
-    {
-        m_class = cls;
-    }
-
     const AbstractMetaFunction *getter() const;
     const AbstractMetaFunction *setter() const;
 
@@ -741,7 +742,6 @@ public:
 private:
     mutable AbstractMetaFunction *m_getter = nullptr;
     mutable AbstractMetaFunction *m_setter = nullptr;
-    const AbstractMetaClass *m_class = nullptr;
 };
 
 #ifndef QT_NO_DEBUG_STREAM
@@ -1179,7 +1179,7 @@ private:
     Documentation m_doc;
 };
 
-class AbstractMetaEnum : public AbstractMetaAttributes
+class AbstractMetaEnum : public AbstractMetaAttributes, public EnclosingClassMixin
 {
 public:
     AbstractMetaEnum();
@@ -1234,23 +1234,12 @@ public:
         m_typeEntry = entry;
     }
 
-    AbstractMetaClass *enclosingClass() const
-    {
-        return m_class;
-    }
-
-    void setEnclosingClass(AbstractMetaClass *c)
-    {
-        m_class = c;
-    }
-
     bool isSigned() const { return m_signed; }
     void setSigned(bool s) { m_signed = s; }
 
 private:
     AbstractMetaEnumValueList m_enumValues;
     EnumTypeEntry *m_typeEntry = nullptr;
-    AbstractMetaClass *m_class = nullptr;
 
     EnumKind m_enumKind = CEnum;
     uint m_hasQenumsDeclaration : 1;
@@ -1261,7 +1250,7 @@ private:
 QDebug operator<<(QDebug d, const AbstractMetaEnum *ae);
 #endif
 
-class AbstractMetaClass : public AbstractMetaAttributes
+class AbstractMetaClass : public AbstractMetaAttributes, public EnclosingClassMixin
 {
     Q_GADGET
 public:
@@ -1477,22 +1466,12 @@ public:
 
     void setBaseClass(AbstractMetaClass *base_class);
 
-    const AbstractMetaClass *enclosingClass() const
-    {
-        return m_enclosingClass;
-    }
-
     /**
      *   \return the namespace from another package which this namespace extends.
      */
     AbstractMetaClass *extendedNamespace() const { return m_extendedNamespace; }
     void setExtendedNamespace(AbstractMetaClass *e) { m_extendedNamespace = e; }
 
-    void setEnclosingClass(AbstractMetaClass *cl)
-    {
-        m_enclosingClass = cl;
-    }
-
     const AbstractMetaClassList& innerClasses() const
     {
         return m_innerClasses;
diff --git a/sources/shiboken2/generator/shiboken2/cppgenerator.cpp b/sources/shiboken2/generator/shiboken2/cppgenerator.cpp
index 5460fd7c7..6d41a7c4d 100644
--- a/sources/shiboken2/generator/shiboken2/cppgenerator.cpp
+++ b/sources/shiboken2/generator/shiboken2/cppgenerator.cpp
@@ -4628,9 +4628,8 @@ static QString mangleName(QString name)
 
 void CppGenerator::writeEnumInitialization(QTextStream &s, const AbstractMetaEnum *cppEnum)
 {
-    const AbstractMetaClass *enclosingClass = getProperEnclosingClassForEnum(cppEnum);
-    const AbstractMetaClass *upper = enclosingClass ? enclosingClass->enclosingClass() : nullptr;
-    bool hasUpperEnclosingClass = upper && upper->typeEntry()->codeGeneration() != TypeEntry::GenerateForSubclass;
+    const AbstractMetaClass *enclosingClass = cppEnum->targetLangEnclosingClass();
+    bool hasUpperEnclosingClass = enclosingClass && enclosingClass->targetLangEnclosingClass() != nullptr;
     const EnumTypeEntry *enumTypeEntry = cppEnum->typeEntry();
     QString enclosingObjectVariable;
     if (enclosingClass)
@@ -4925,9 +4924,8 @@ void CppGenerator::writeClassRegister(QTextStream &s,
 {
     const ComplexTypeEntry *classTypeEntry = metaClass->typeEntry();
 
-    const AbstractMetaClass *enc = metaClass->enclosingClass();
-    bool hasEnclosingClass = enc && enc->typeEntry()->codeGeneration() != TypeEntry::GenerateForSubclass;
-    QString enclosingObjectVariable = hasEnclosingClass ? QLatin1String("enclosingClass") : QLatin1String("module");
+    const AbstractMetaClass *enc = metaClass->targetLangEnclosingClass();
+    QString enclosingObjectVariable = enc ? QLatin1String("enclosingClass") : QLatin1String("module");
 
     QString pyTypeName = cpythonTypeName(metaClass);
     QString initFunctionName = getInitFunctionName(classContext);
@@ -5027,7 +5025,7 @@ void CppGenerator::writeClassRegister(QTextStream &s,
 
         // 9:wrapperflags
         QByteArrayList wrapperFlags;
-        if (hasEnclosingClass)
+        if (enc)
             wrapperFlags.append(QByteArrayLiteral("Shiboken::ObjectType::WrapperFlags::InnerClass"));
         if (metaClass->deleteInMainThread())
             wrapperFlags.append(QByteArrayLiteral("Shiboken::ObjectType::WrapperFlags::DeleteInMainThread"));
@@ -5461,10 +5459,9 @@ bool CppGenerator::finishGeneration()
         s_classInitDecl << "void " << initFunctionName << "(PyObject *module);" << endl;
 
         s_classPythonDefines << INDENT << initFunctionName;
-        if (cls->enclosingClass()
-            && (cls->enclosingClass()->typeEntry()->codeGeneration() != TypeEntry::GenerateForSubclass)) {
+        if (auto enclosing = cls->targetLangEnclosingClass()) {
             s_classPythonDefines << "(reinterpret_cast<PyTypeObject *>("
-                << cpythonTypeNameExt(cls->enclosingClass()->typeEntry()) << ")->tp_dict);";
+                << cpythonTypeNameExt(enclosing->typeEntry()) << ")->tp_dict);";
         } else {
             s_classPythonDefines << "(module);";
         }
diff --git a/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp b/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp
index fd75c620e..07c031d9b 100644
--- a/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp
+++ b/sources/shiboken2/generator/shiboken2/shibokengenerator.cpp
@@ -337,22 +337,6 @@ void ShibokenGenerator::lookForEnumsInClassesNotToBeGenerated(AbstractMetaEnumLi
     }
 }
 
-static const AbstractMetaClass *getProperEnclosingClass(const AbstractMetaClass *metaClass)
-{
-    if (!metaClass)
-        return nullptr;
-
-    if (metaClass->typeEntry()->codeGeneration() != TypeEntry::GenerateForSubclass)
-        return metaClass;
-
-    return getProperEnclosingClass(metaClass->enclosingClass());
-}
-
-const AbstractMetaClass *ShibokenGenerator::getProperEnclosingClassForEnum(const AbstractMetaEnum *metaEnum)
-{
-    return getProperEnclosingClass(metaEnum->enclosingClass());
-}
-
 QString ShibokenGenerator::wrapperName(const AbstractMetaClass *metaClass) const
 {
     if (shouldGenerateCppWrapper(metaClass)) {
diff --git a/sources/shiboken2/generator/shiboken2/shibokengenerator.h b/sources/shiboken2/generator/shiboken2/shibokengenerator.h
index 7970ceb94..4501b902d 100644
--- a/sources/shiboken2/generator/shiboken2/shibokengenerator.h
+++ b/sources/shiboken2/generator/shiboken2/shibokengenerator.h
@@ -206,8 +206,6 @@ protected:
 
     /// Adds enums eligible for generation from classes/namespaces marked not to be generated.
     static void lookForEnumsInClassesNotToBeGenerated(AbstractMetaEnumList &enumList, const AbstractMetaClass *metaClass);
-    /// Returns the enclosing class for an enum, or nullptr if it should be global.
-    const AbstractMetaClass *getProperEnclosingClassForEnum(const AbstractMetaEnum *metaEnum);
 
     QString wrapperName(const AbstractMetaClass *metaClass) const;
     QString wrapperName(const AbstractMetaType *metaType) const;