shiboken: Move detection of template arguments to ClangBuilder

Obtain the template arguments from Clang and fall back to parsing
the type name where this is not possible (which may be the case
inside a template declaration).

The string-based formatting and re-parsing of the type in
AbstractMetaBuilder::translateType() can then be removed, opening
the way to passing up more complex types from Clang into the
MetaBuilder.

Task-number: PYSIDE-672
Change-Id: I43ff285c5f3720319bf40c65b1c27302ef1b934e
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Christian Tismer <tismer@stackless.com>

6 files changed, 163 insertions, 16 deletions

diff --git a/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp b/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp
index 9ada2993a..20f201ec6 100644
--- a/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp
+++ b/sources/shiboken2/ApiExtractor/abstractmetabuilder.cpp
@@ -2281,15 +2281,13 @@ AbstractMetaType *AbstractMetaBuilderPrivate::translateTypeStatic(const TypeInfo
 {
     // 1. Test the type info without resolving typedefs in case this is present in the
     //    type system
-    TypeInfo typei;
     if (resolveType) {
         if (AbstractMetaType *resolved = translateTypeStatic(_typei, currentClass, d, false, errorMessageIn))
             return resolved;
     }
 
-    if (!resolveType) {
-        typei = _typei;
-    } else {
+    TypeInfo typeInfo = _typei;
+    if (resolveType) {
         // Go through all parts of the current scope (including global namespace)
         // to resolve typedefs. The parser does not properly resolve typedefs in
         // the global scope when they are referenced from inside a namespace.
@@ -2297,29 +2295,20 @@ AbstractMetaType *AbstractMetaBuilderPrivate::translateTypeStatic(const TypeInfo
         // seemed non-trivial
         int i = d ? d->m_scopes.size() - 1 : -1;
         while (i >= 0) {
-            typei = TypeInfo::resolveType(_typei, d->m_scopes.at(i--));
-            if (typei.qualifiedName().join(colonColon()) != _typei.qualifiedName().join(colonColon()))
+            typeInfo = TypeInfo::resolveType(_typei, d->m_scopes.at(i--));
+            if (typeInfo.qualifiedName().join(colonColon()) != _typei.qualifiedName().join(colonColon()))
                 break;
         }
 
     }
 
-    if (typei.isFunctionPointer()) {
+    if (typeInfo.isFunctionPointer()) {
         if (errorMessageIn)
             *errorMessageIn = msgUnableToTranslateType(_typei, QLatin1String("Unsupported function pointer."));
         return nullptr;
     }
 
     QString errorMessage;
-    TypeInfo typeInfo = TypeParser::parse(typei.toString(), &errorMessage);
-    if (typeInfo.qualifiedName().isEmpty()) {
-        errorMessage = msgUnableToTranslateType(_typei, errorMessage);
-        if (errorMessageIn)
-            *errorMessageIn = errorMessage;
-        else
-            qCWarning(lcShiboken,"%s", qPrintable(errorMessage));
-        return 0;
-    }
 
     // 2. Handle arrays.
     // 2.1 Handle char arrays with unspecified size (aka "const char[]") as "const char*" with
diff --git a/sources/shiboken2/ApiExtractor/clangparser/clangbuilder.cpp b/sources/shiboken2/ApiExtractor/clangparser/clangbuilder.cpp
index 23ccf5d88..9d0d69c27 100644
--- a/sources/shiboken2/ApiExtractor/clangparser/clangbuilder.cpp
+++ b/sources/shiboken2/ApiExtractor/clangparser/clangbuilder.cpp
@@ -183,6 +183,10 @@ public:
     TypeInfo createTypeInfo(const CXType &type) const;
     TypeInfo createTypeInfo(const CXCursor &cursor) const
     { return createTypeInfo(clang_getCursorType(cursor)); }
+    void addTemplateInstantiations(const CXType &type,
+                                   QString *typeName,
+                                   TypeInfo *t) const;
+    bool addTemplateInstantiationsRecursion(const CXType &type, TypeInfo *t) const;
 
     TemplateParameterModelItem createTemplateParameter(const CXCursor &cursor) const;
     TemplateParameterModelItem createNonTypeTemplateParameter(const CXCursor &cursor) const;
@@ -391,6 +395,54 @@ static bool isPointerType(CXTypeKind k)
     return k == CXType_Pointer || k == CXType_LValueReference || k == CXType_RValueReference;
 }
 
+bool BuilderPrivate::addTemplateInstantiationsRecursion(const CXType &type, TypeInfo *t) const
+{
+    // Template arguments
+    switch (type.kind) {
+    case CXType_Elaborated:
+    case CXType_Record:
+    case CXType_Unexposed:
+        if (const int numTemplateArguments = qMax(0, clang_Type_getNumTemplateArguments(type))) {
+            for (unsigned tpl = 0; tpl < unsigned(numTemplateArguments); ++tpl) {
+                const CXType argType = clang_Type_getTemplateArgumentAsType(type, tpl);
+                // CXType_Invalid is returned when hitting on a specialization
+                // of a non-type template (template <int v>).
+                if (argType.kind == CXType_Invalid)
+                    return false;
+                t->addInstantiation(createTypeInfo(argType));
+            }
+        }
+        break;
+    default:
+        break;
+    }
+    return true;
+}
+
+static void dummyTemplateArgumentHandler(int, const QStringRef &) {}
+
+void BuilderPrivate::addTemplateInstantiations(const CXType &type,
+                                               QString *typeName,
+                                               TypeInfo *t) const
+{
+    // In most cases, for templates like "Vector<A>", Clang will give us the
+    // arguments by recursing down the type. However this will fail for example
+    // within template classes (for functions like the copy constructor):
+    // template <class T>
+    // class Vector {
+    //    Vector(const Vector&);
+    // };
+    // In that case, have TypeInfo parse the list from the spelling.
+    // Finally, remove the list "<>" from the type name.
+    const bool parsed = addTemplateInstantiationsRecursion(type, t)
+        && !t->instantiations().isEmpty();
+    const QPair<int, int> pos = parsed
+        ? parseTemplateArgumentList(*typeName, dummyTemplateArgumentHandler)
+        : t->parseTemplateArgumentList(*typeName);
+    if (pos.first != -1 && pos.second != -1 && pos.second > pos.first)
+        typeName->remove(pos.first, pos.second - pos.first);
+}
+
 TypeInfo BuilderPrivate::createTypeInfo(const CXType &type) const
 {
     if (type.kind == CXType_Pointer) { // Check for function pointers, first.
@@ -440,6 +492,11 @@ TypeInfo BuilderPrivate::createTypeInfo(const CXType &type) const
            || TypeInfo::stripLeadingVolatile(&typeName)) {
     }
 
+    // Obtain template instantiations if the name has '<' (thus excluding
+    // typedefs like "std::string".
+    if (typeName.contains(QLatin1Char('<')))
+        addTemplateInstantiations(nestedType, &typeName, &typeInfo);
+
     typeInfo.setQualifiedName(qualifiedName(typeName));
     // 3320:CINDEX_LINKAGE int clang_getNumArgTypes(CXType T); function ptr types?
     return typeInfo;
diff --git a/sources/shiboken2/ApiExtractor/clangparser/clangutils.cpp b/sources/shiboken2/ApiExtractor/clangparser/clangutils.cpp
index 2ff18b23b..a68c5b5ee 100644
--- a/sources/shiboken2/ApiExtractor/clangparser/clangutils.cpp
+++ b/sources/shiboken2/ApiExtractor/clangparser/clangutils.cpp
@@ -160,6 +160,43 @@ QVector<Diagnostic> getDiagnostics(CXTranslationUnit tu)
     return result;
 }
 
+QPair<int, int> parseTemplateArgumentList(const QString &l,
+                                          const TemplateArgumentHandler &handler,
+                                          int from)
+{
+    const int ltPos = l.indexOf(QLatin1Char('<'), from);
+    if (ltPos == - 1)
+        return qMakePair(-1, -1);
+    int startPos = ltPos + 1;
+    int level = 1;
+    for (int p = startPos, end = l.size(); p < end; ) {
+        const char c = l.at(p).toLatin1();
+        switch (c) {
+        case ',':
+        case '>':
+            handler(level, l.midRef(startPos, p - startPos).trimmed());
+            ++p;
+            if (c == '>') {
+                if (--level == 0)
+                    return qMakePair(ltPos, p);
+                // Skip over next ',': "a<b<c,d>,e>"
+                for (; p < end && (l.at(p).isSpace() || l.at(p) == QLatin1Char(',')); ++p) {}
+            }
+            startPos = p;
+            break;
+        case '<':
+            handler(level, l.midRef(startPos, p - startPos).trimmed());
+            ++level;
+            startPos = ++p;
+            break;
+        default:
+             ++p;
+            break;
+        }
+    }
+    return qMakePair(-1, -1);
+}
+
 CXDiagnosticSeverity maxSeverity(const QVector<Diagnostic> &ds)
 {
     CXDiagnosticSeverity result = CXDiagnostic_Ignored;
diff --git a/sources/shiboken2/ApiExtractor/clangparser/clangutils.h b/sources/shiboken2/ApiExtractor/clangparser/clangutils.h
index 98d0c9752..ab6f0ef91 100644
--- a/sources/shiboken2/ApiExtractor/clangparser/clangutils.h
+++ b/sources/shiboken2/ApiExtractor/clangparser/clangutils.h
@@ -34,6 +34,8 @@
 #include <QtCore/QString>
 #include <QtCore/QVector>
 
+#include <functional>
+
 QT_FORWARD_DECLARE_CLASS(QDebug)
 
 bool operator==(const CXCursor &c1, const CXCursor &c2);
@@ -92,6 +94,14 @@ struct Diagnostic {
 QVector<Diagnostic> getDiagnostics(CXTranslationUnit tu);
 CXDiagnosticSeverity maxSeverity(const QVector<Diagnostic> &ds);
 
+// Parse a template argument list "a<b<c,d>,e>" and invoke a handler
+// with each match (level and string). Return begin and end of the list.
+typedef std::function<void(int /*level*/, const QStringRef &)> TemplateArgumentHandler;
+
+QPair<int, int> parseTemplateArgumentList(const QString &l,
+                                          const TemplateArgumentHandler &handler,
+                                          int from = 0);
+
 #ifndef QT_NO_DEBUG_STREAM
 QDebug operator<<(QDebug, const SourceLocation &);
 QDebug operator<<(QDebug, const Diagnostic &);
diff --git a/sources/shiboken2/ApiExtractor/parser/codemodel.cpp b/sources/shiboken2/ApiExtractor/parser/codemodel.cpp
index d3ada9235..a560c0faf 100644
--- a/sources/shiboken2/ApiExtractor/parser/codemodel.cpp
+++ b/sources/shiboken2/ApiExtractor/parser/codemodel.cpp
@@ -29,11 +29,15 @@
 
 
 #include "codemodel.h"
+
+#include <clangparser/clangutils.h>
+
 #include <algorithm>
 #include <functional>
 #include <iostream>
 #include <QDebug>
 #include <QDir>
+#include <QtCore/QStack>
 
 // Predicate to find an item by name in a list of QSharedPointer<Item>
 template <class T> class ModelItemNamePredicate : public std::unary_function<bool, QSharedPointer<T> >
@@ -195,6 +199,51 @@ TypeInfo TypeInfo::resolveType(CodeModelItem __item, TypeInfo const &__type, Cod
     return otherType;
 }
 
+// Handler for clang::parseTemplateArgumentList() that populates
+// TypeInfo::m_instantiations
+class TypeInfoTemplateArgumentHandler :
+    public std::binary_function<void, int, const QStringRef &>
+{
+public:
+    explicit TypeInfoTemplateArgumentHandler(TypeInfo *t)
+    {
+        m_parseStack.append(t);
+    }
+
+    void operator()(int level, const QStringRef &name)
+    {
+        if (level > m_parseStack.size()) {
+            Q_ASSERT(!top()->m_instantiations.isEmpty());
+            m_parseStack.push(&top()->m_instantiations.back());
+        }
+        while (level < m_parseStack.size())
+            m_parseStack.pop();
+        TypeInfo instantiation;
+        instantiation.setQualifiedName(qualifiedName(name));
+        top()->addInstantiation(instantiation);
+   }
+
+private:
+    TypeInfo *top() const { return m_parseStack.back(); }
+
+    static QStringList qualifiedName(const QStringRef &name)
+    {
+        QStringList result;
+        const QVector<QStringRef> nameParts = name.split(QLatin1String("::"));
+        result.reserve(nameParts.size());
+        for (const QStringRef &p : nameParts)
+            result.append(p.toString());
+        return result;
+    }
+
+    QStack<TypeInfo *> m_parseStack;
+};
+
+QPair<int, int> TypeInfo::parseTemplateArgumentList(const QString &l, int from)
+{
+    return clang::parseTemplateArgumentList(l, clang::TemplateArgumentHandler(TypeInfoTemplateArgumentHandler(this)), from);
+}
+
 QString TypeInfo::toString() const
 {
     QString tmp;
diff --git a/sources/shiboken2/ApiExtractor/parser/codemodel.h b/sources/shiboken2/ApiExtractor/parser/codemodel.h
index d0381c4e3..c61f640e1 100644
--- a/sources/shiboken2/ApiExtractor/parser/codemodel.h
+++ b/sources/shiboken2/ApiExtractor/parser/codemodel.h
@@ -36,6 +36,7 @@
 #include "enumvalue.h"
 
 #include <QtCore/QHash>
+#include <QtCore/QPair>
 #include <QtCore/QSet>
 #include <QtCore/QString>
 #include <QtCore/QStringList>
@@ -185,6 +186,8 @@ public:
     void addInstantiation(const TypeInfo &i) { m_instantiations.append(i); }
     void clearInstantiations() { m_instantiations.clear(); }
 
+    QPair<int, int> parseTemplateArgumentList(const QString &l, int from = 0);
+
     bool operator==(const TypeInfo &other) const;
 
     bool operator!=(const TypeInfo &other) const
@@ -210,6 +213,8 @@ public:
     static bool stripLeadingQualifier(const QString &qualifier, QString *s);
 
 private:
+    friend class TypeInfoTemplateArgumentHandler;
+
     static TypeInfo resolveType(CodeModelItem item, TypeInfo const &__type, CodeModelItem __scope);
 
     QStringList m_qualifiedName;