ClangCodeModel: Fix template highlighting bug

We need to take into account that the ">>" in constructs such as
std::vector<std::pair<int, int>> is only one token on the clang side.

Change-Id: I90f002ca56f236032f6d39c338593a2ff7590061
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: David Schulz <david.schulz@qt.io>

8 files changed, 30 insertions, 7 deletions

diff --git a/src/libs/clangsupport/clangsupport_global.h b/src/libs/clangsupport/clangsupport_global.h
index 4420115165..281dd05806 100644
--- a/src/libs/clangsupport/clangsupport_global.h
+++ b/src/libs/clangsupport/clangsupport_global.h
@@ -110,6 +110,7 @@ enum class HighlightingType : quint8
     TemplateTemplateParameter,
     AngleBracketOpen,
     AngleBracketClose,
+    DoubleAngleBracketClose, // clang parses ">>" as one token, even if it's closing a nested template
     TernaryIf,
     TernaryElse,
 };
diff --git a/src/plugins/clangcodemodel/clanghighlightingresultreporter.cpp b/src/plugins/clangcodemodel/clanghighlightingresultreporter.cpp
index dc78a4d7d3..451a6465d0 100644
--- a/src/plugins/clangcodemodel/clanghighlightingresultreporter.cpp
+++ b/src/plugins/clangcodemodel/clanghighlightingresultreporter.cpp
@@ -115,6 +115,7 @@ bool ignore(ClangBackEnd::HighlightingType type)
     case HighlightingType::TemplateTemplateParameter:
     case HighlightingType::AngleBracketOpen:
     case HighlightingType::AngleBracketClose:
+    case HighlightingType::DoubleAngleBracketClose:
     case HighlightingType::TernaryIf:
     case HighlightingType::TernaryElse:
         return true;
@@ -149,6 +150,8 @@ TextEditor::HighlightingResult toHighlightingResult(
         result.kind = CppTools::SemanticHighlighter::AngleBracketOpen;
     else if (tokenInfo.types.mixinHighlightingTypes.contains(HighlightingType::AngleBracketClose))
         result.kind = CppTools::SemanticHighlighter::AngleBracketClose;
+    else if (tokenInfo.types.mixinHighlightingTypes.contains(HighlightingType::DoubleAngleBracketClose))
+        result.kind = CppTools::SemanticHighlighter::DoubleAngleBracketClose;
     else if (tokenInfo.types.mixinHighlightingTypes.contains(HighlightingType::TernaryIf))
         result.kind = CppTools::SemanticHighlighter::TernaryIf;
     else if (tokenInfo.types.mixinHighlightingTypes.contains(HighlightingType::TernaryElse))
diff --git a/src/plugins/cpptools/semantichighlighter.cpp b/src/plugins/cpptools/semantichighlighter.cpp
index d4d250166c..7b40efe561 100644
--- a/src/plugins/cpptools/semantichighlighter.cpp
+++ b/src/plugins/cpptools/semantichighlighter.cpp
@@ -178,6 +178,7 @@ void SemanticHighlighter::onHighlighterResultAvailable(int from, int to)
     for (int i = from; i < to; ++i) {
         const HighlightingResult &result = m_watcher->future().resultAt(i);
         if (result.kind != AngleBracketOpen && result.kind != AngleBracketClose
+                && result.kind != DoubleAngleBracketClose
                 && result.kind != TernaryIf && result.kind != TernaryElse) {
             const QTextBlock block =
                     m_baseTextDocument->document()->findBlockByNumber(result.line - 1);
@@ -193,14 +194,18 @@ void SemanticHighlighter::onHighlighterResultAvailable(int from, int to)
             parentheses.second = getClearedParentheses(parentheses.first);
         }
         Parenthesis paren;
-        if (result.kind == AngleBracketOpen)
+        if (result.kind == AngleBracketOpen) {
             paren = {Parenthesis::Opened, '<', result.column - 1};
-        else if (result.kind == AngleBracketClose)
+        } else if (result.kind == AngleBracketClose) {
             paren = {Parenthesis::Closed, '>', result.column - 1};
-        else if (result.kind == TernaryIf)
+        } else if (result.kind == DoubleAngleBracketClose) {
+            parentheses.second.append({Parenthesis::Closed, '>', result.column - 1});
+            paren = {Parenthesis::Closed, '>', result.column};
+        } else if (result.kind == TernaryIf) {
             paren = {Parenthesis::Opened, '?', result.column - 1};
-        else if (result.kind == TernaryElse)
+        } else if (result.kind == TernaryElse) {
             paren = {Parenthesis::Closed, ':', result.column - 1};
+        }
         QTC_ASSERT(paren.pos != -1, continue);
         paren.source = parenSource();
         parentheses.second << paren;
diff --git a/src/plugins/cpptools/semantichighlighter.h b/src/plugins/cpptools/semantichighlighter.h
index 81b690eb07..6ece6b9530 100644
--- a/src/plugins/cpptools/semantichighlighter.h
+++ b/src/plugins/cpptools/semantichighlighter.h
@@ -61,6 +61,7 @@ public:
         VirtualFunctionDeclarationUse,
         AngleBracketOpen,
         AngleBracketClose,
+        DoubleAngleBracketClose,
         TernaryIf,
         TernaryElse,
     };
diff --git a/src/tools/clangbackend/source/tokeninfo.cpp b/src/tools/clangbackend/source/tokeninfo.cpp
index 1ca8f4cac0..8f63de4187 100644
--- a/src/tools/clangbackend/source/tokeninfo.cpp
+++ b/src/tools/clangbackend/source/tokeninfo.cpp
@@ -597,14 +597,17 @@ void TokenInfo::punctuationOrOperatorKind()
             break;
     }
 
-    if (m_types.mixinHighlightingTypes.empty()
-        && kind != CXCursor_InclusionDirective
-        && kind != CXCursor_PreprocessingDirective) {
+    if (m_types.mainHighlightingType == HighlightingType::Punctuation
+            && m_types.mixinHighlightingTypes.empty()
+            && kind != CXCursor_InclusionDirective
+            && kind != CXCursor_PreprocessingDirective) {
         const ClangString spelling = m_token->spelling();
         if (spelling == "<")
             m_types.mixinHighlightingTypes.push_back(HighlightingType::AngleBracketOpen);
         else if (spelling == ">")
             m_types.mixinHighlightingTypes.push_back(HighlightingType::AngleBracketClose);
+        else if (spelling == ">>")
+            m_types.mixinHighlightingTypes.push_back(HighlightingType::DoubleAngleBracketClose);
     }
 
     if (isOutputArgument())
diff --git a/tests/unit/unittest/data/highlightingmarks.cpp b/tests/unit/unittest/data/highlightingmarks.cpp
index f3b624f320..8221afc29f 100644
--- a/tests/unit/unittest/data/highlightingmarks.cpp
+++ b/tests/unit/unittest/data/highlightingmarks.cpp
@@ -769,3 +769,5 @@ template<typename T>
 void func(T v) {
     GlobalVar = 5;
 }
+
+static std::vector<std::pair<int, int>> pv;
diff --git a/tests/unit/unittest/gtest-creator-printing.cpp b/tests/unit/unittest/gtest-creator-printing.cpp
index cc10dcfa9d..2f0afc841c 100644
--- a/tests/unit/unittest/gtest-creator-printing.cpp
+++ b/tests/unit/unittest/gtest-creator-printing.cpp
@@ -868,6 +868,7 @@ static const char *highlightingTypeToCStringLiteral(HighlightingType type)
         RETURN_TEXT_FOR_CASE(TemplateTemplateParameter);
         RETURN_TEXT_FOR_CASE(AngleBracketOpen);
         RETURN_TEXT_FOR_CASE(AngleBracketClose);
+        RETURN_TEXT_FOR_CASE(DoubleAngleBracketClose);
         RETURN_TEXT_FOR_CASE(TernaryIf);
         RETURN_TEXT_FOR_CASE(TernaryElse);
     }
diff --git a/tests/unit/unittest/tokenprocessor-test.cpp b/tests/unit/unittest/tokenprocessor-test.cpp
index 94e1b83b15..562f852abe 100644
--- a/tests/unit/unittest/tokenprocessor-test.cpp
+++ b/tests/unit/unittest/tokenprocessor-test.cpp
@@ -1794,6 +1794,13 @@ TEST_F(TokenProcessor, TemplateSeparateDeclDef)
     ASSERT_THAT(infos[37], IsHighlightingMark(764u, 5u, 9u, HighlightingType::GlobalVariable));
 }
 
+TEST_F(TokenProcessor, NestedTemplate)
+{
+    const auto infos = translationUnit.tokenInfosInRange(sourceRange(773, 44));
+    ASSERT_THAT(infos[12], HasTwoTypes(HighlightingType::Punctuation,
+                                       HighlightingType::DoubleAngleBracketClose));
+}
+
 Data *TokenProcessor::d;
 
 void TokenProcessor::SetUpTestCase()