ClangCodeModel: Fix template highlighting bug

We need to take into account that the ">>" in constructs such as
std::vector<std::pair<int, int>> is only one token on the clang side.

Change-Id: I90f002ca56f236032f6d39c338593a2ff7590061
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: David Schulz <david.schulz@qt.io>

src/libs/clangsupport/clangsupport_global.h

@@ -110,6 +110,7 @@ enum class HighlightingType : quint8
     TemplateTemplateParameter,
     AngleBracketOpen,
     AngleBracketClose,
+    DoubleAngleBracketClose, // clang parses ">>" as one token, even if it's closing a nested template
     TernaryIf,
     TernaryElse,
 };

src/plugins/clangcodemodel/clanghighlightingresultreporter.cpp

@@ -115,6 +115,7 @@ bool ignore(ClangBackEnd::HighlightingType type)
     case HighlightingType::TemplateTemplateParameter:
     case HighlightingType::AngleBracketOpen:
     case HighlightingType::AngleBracketClose:
+    case HighlightingType::DoubleAngleBracketClose:
     case HighlightingType::TernaryIf:
     case HighlightingType::TernaryElse:
         return true;
@@ -149,6 +150,8 @@ TextEditor::HighlightingResult toHighlightingResult(
         result.kind = CppTools::SemanticHighlighter::AngleBracketOpen;
     else if (tokenInfo.types.mixinHighlightingTypes.contains(HighlightingType::AngleBracketClose))
         result.kind = CppTools::SemanticHighlighter::AngleBracketClose;
+    else if (tokenInfo.types.mixinHighlightingTypes.contains(HighlightingType::DoubleAngleBracketClose))
+        result.kind = CppTools::SemanticHighlighter::DoubleAngleBracketClose;
     else if (tokenInfo.types.mixinHighlightingTypes.contains(HighlightingType::TernaryIf))
         result.kind = CppTools::SemanticHighlighter::TernaryIf;
     else if (tokenInfo.types.mixinHighlightingTypes.contains(HighlightingType::TernaryElse))

src/plugins/cpptools/semantichighlighter.cpp

@@ -178,6 +178,7 @@ void SemanticHighlighter::onHighlighterResultAvailable(int from, int to)
     for (int i = from; i < to; ++i) {
         const HighlightingResult &result = m_watcher->future().resultAt(i);
         if (result.kind != AngleBracketOpen && result.kind != AngleBracketClose
+                && result.kind != DoubleAngleBracketClose
                 && result.kind != TernaryIf && result.kind != TernaryElse) {
             const QTextBlock block =
                     m_baseTextDocument->document()->findBlockByNumber(result.line - 1);
@@ -193,14 +194,18 @@ void SemanticHighlighter::onHighlighterResultAvailable(int from, int to)
             parentheses.second = getClearedParentheses(parentheses.first);
         }
         Parenthesis paren;
-        if (result.kind == AngleBracketOpen)
+        if (result.kind == AngleBracketOpen) {
             paren = {Parenthesis::Opened, '<', result.column - 1};
-        else if (result.kind == AngleBracketClose)
+        } else if (result.kind == AngleBracketClose) {
             paren = {Parenthesis::Closed, '>', result.column - 1};
-        else if (result.kind == TernaryIf)
+        } else if (result.kind == DoubleAngleBracketClose) {
+            parentheses.second.append({Parenthesis::Closed, '>', result.column - 1});
+            paren = {Parenthesis::Closed, '>', result.column};
+        } else if (result.kind == TernaryIf) {
             paren = {Parenthesis::Opened, '?', result.column - 1};
-        else if (result.kind == TernaryElse)
+        } else if (result.kind == TernaryElse) {
             paren = {Parenthesis::Closed, ':', result.column - 1};
+        }
         QTC_ASSERT(paren.pos != -1, continue);
         paren.source = parenSource();
         parentheses.second << paren;

src/plugins/cpptools/semantichighlighter.h

@@ -61,6 +61,7 @@ public:
         VirtualFunctionDeclarationUse,
         AngleBracketOpen,
         AngleBracketClose,
+        DoubleAngleBracketClose,
         TernaryIf,
         TernaryElse,
     };

src/tools/clangbackend/source/tokeninfo.cpp

@@ -597,7 +597,8 @@ void TokenInfo::punctuationOrOperatorKind()
             break;
     }
 
-    if (m_types.mixinHighlightingTypes.empty()
+    if (m_types.mainHighlightingType == HighlightingType::Punctuation
+            && m_types.mixinHighlightingTypes.empty()
             && kind != CXCursor_InclusionDirective
             && kind != CXCursor_PreprocessingDirective) {
         const ClangString spelling = m_token->spelling();
@@ -605,6 +606,8 @@ void TokenInfo::punctuationOrOperatorKind()
             m_types.mixinHighlightingTypes.push_back(HighlightingType::AngleBracketOpen);
         else if (spelling == ">")
             m_types.mixinHighlightingTypes.push_back(HighlightingType::AngleBracketClose);
+        else if (spelling == ">>")
+            m_types.mixinHighlightingTypes.push_back(HighlightingType::DoubleAngleBracketClose);
     }
 
     if (isOutputArgument())

tests/unit/unittest/data/highlightingmarks.cpp

@@ -769,3 +769,5 @@ template<typename T>
 void func(T v) {
     GlobalVar = 5;
 }
+
+static std::vector<std::pair<int, int>> pv;

tests/unit/unittest/gtest-creator-printing.cpp

@@ -868,6 +868,7 @@ static const char *highlightingTypeToCStringLiteral(HighlightingType type)
         RETURN_TEXT_FOR_CASE(TemplateTemplateParameter);
         RETURN_TEXT_FOR_CASE(AngleBracketOpen);
         RETURN_TEXT_FOR_CASE(AngleBracketClose);
+        RETURN_TEXT_FOR_CASE(DoubleAngleBracketClose);
         RETURN_TEXT_FOR_CASE(TernaryIf);
         RETURN_TEXT_FOR_CASE(TernaryElse);
     }

tests/unit/unittest/tokenprocessor-test.cpp

@@ -1794,6 +1794,13 @@ TEST_F(TokenProcessor, TemplateSeparateDeclDef)
     ASSERT_THAT(infos[37], IsHighlightingMark(764u, 5u, 9u, HighlightingType::GlobalVariable));
 }
 
+TEST_F(TokenProcessor, NestedTemplate)
+{
+    const auto infos = translationUnit.tokenInfosInRange(sourceRange(773, 44));
+    ASSERT_THAT(infos[12], HasTwoTypes(HighlightingType::Punctuation,
+                                       HighlightingType::DoubleAngleBracketClose));
+}
+
 Data *TokenProcessor::d;
 
 void TokenProcessor::SetUpTestCase()