ClangCodeModel: Provide highlighting via clangd

Note that we lose the highlighting for virtual function calls. We need
to amend clangd to add the respective information to its semantic tokens
message.
Also, Qt properties are no longer highlighted as class members. We'll
investigate how to best restore this feature.

Change-Id: I403712aada3d7a8e1c7b7c1277f43f7f64f8450b
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: David Schulz <david.schulz@qt.io>

src/libs/languageserverprotocol/semantictokens.cpp

@@ -104,8 +104,10 @@ QList<SemanticToken> SemanticTokens::toTokens(const QList<int> &tokenTypes,
         token.deltaLine = *(it);
         token.deltaStart = *(it + 1);
         token.length = *(it + 2);
-        token.tokenType = tokenTypes.value(*(it + 3), -1);
+        token.tokenIndex = *(it + 3);
-        token.tokenModifiers = convertModifiers(*(it + 4), tokenModifiers);
+        token.tokenType = tokenTypes.value(token.tokenIndex, -1);
+        token.rawTokenModifiers = *(it + 4);
+        token.tokenModifiers = convertModifiers(token.rawTokenModifiers, tokenModifiers);
         tokens << token;
     }
     return tokens;

src/libs/languageserverprotocol/semantictokens.h

@@ -38,7 +38,9 @@ struct LANGUAGESERVERPROTOCOL_EXPORT SemanticToken
     int deltaLine = 0;
     int deltaStart = 0;
     int length = 0;
+    int tokenIndex = 0;
     int tokenType = 0;
+    int rawTokenModifiers = 0;
     int tokenModifiers = 0;
 };
 

src/plugins/clangcodemodel/clangcodemodelplugin.cpp

@@ -207,6 +207,7 @@ QVector<QObject *> ClangCodeModelPlugin::createTestObjects() const
         new Tests::ClangCodeCompletionTest,
         new Tests::ClangdTestFindReferences,
         new Tests::ClangdTestFollowSymbol,
+        new Tests::ClangdTestHighlighting,
         new Tests::ClangdTestLocalReferences,
         new Tests::ClangdTestTooltips,
     };

src/plugins/clangcodemodel/clangdclient.cpp

@@ -38,6 +38,7 @@
 #include <cpptools/cpptoolsreuse.h>
 #include <cpptools/cppvirtualfunctionassistprovider.h>
 #include <cpptools/cppvirtualfunctionproposalitem.h>
+#include <cpptools/semantichighlighter.h>
 #include <languageclient/languageclientinterface.h>
 #include <languageclient/languageclientutils.h>
 #include <projectexplorer/project.h>
@@ -49,6 +50,7 @@
 #include <texteditor/codeassist/iassistprovider.h>
 #include <texteditor/texteditor.h>
 #include <utils/algorithm.h>
+#include <utils/runextensions.h>
 
 #include <QCheckBox>
 #include <QFile>
@@ -58,6 +60,7 @@
 #include <QRegularExpression>
 
 #include <set>
+#include <unordered_map>
 
 using namespace CPlusPlus;
 using namespace Core;
@@ -231,10 +234,12 @@ public:
         QString theType = type();
         if (theType.endsWith("const"))
             theType.chop(5);
-        const int ptrRefCount = theType.count('*') + theType.count('&');
+        const int xrefCount = theType.count("&&");
+        const int refCount = theType.count('&') - 2 * xrefCount;
+        const int ptrRefCount = theType.count('*') + refCount;
         const int constCount = theType.count("const");
         if (ptrRefCount == 0)
-            return constCount > 0 || detailIs("LValueToRValue");
+            return constCount > 0 || detailIs("LValueToRValue") || arcanaContains("xvalue");
         return ptrRefCount <= constCount;
     }
 
@@ -245,6 +250,13 @@ public:
                 && childList->at(index).range().contains(range);
     }
 
+    bool hasChildWithRole(const QString &role) const
+    {
+        return Utils::contains(children().value_or(QList<AstNode>()), [&role](const AstNode &c) {
+            return c.role() == role;
+        });
+    }
+
     QString operatorString() const
     {
         if (kind() == "BinaryOperator")
@@ -688,6 +700,9 @@ public:
                                HelpItem::Category category = HelpItem::Unknown,
                                const QString &type = {});
 
+    void handleSemanticTokens(TextEditor::TextDocument *doc,
+                              const QList<ExpandedSemanticToken> &tokens);
+
     ClangdClient * const q;
     const CppTools::ClangdSettings::Data settings;
     QHash<quint64, ReferencesData> runningFindUsages;
@@ -695,6 +710,7 @@ public:
     Utils::optional<SwitchDeclDefData> switchDeclDefData;
     Utils::optional<LocalRefsData> localRefsData;
     Utils::optional<QVersionNumber> versionNumber;
+    std::unordered_map<TextEditor::TextDocument *, CppTools::SemanticHighlighter> highlighters;
     quint64 nextJobId = 0;
     bool isFullyIndexed = false;
     bool isTesting = false;
@@ -757,6 +773,11 @@ ClangdClient::ClangdClient(Project *project, const Utils::FilePath &jsonDbDir)
     };
     setSymbolStringifier(symbolStringifier);
 
+    setSemanticTokensHandler([this](TextEditor::TextDocument *doc,
+                                    const QList<ExpandedSemanticToken> &tokens) {
+        d->handleSemanticTokens(doc, tokens);
+    });
+
     hoverHandler()->setHelpItemProvider([this](const HoverRequest::Response &response,
                                                const DocumentUri &uri) {
         gatherHelpItemForTooltip(response, uri);
@@ -908,6 +929,11 @@ void ClangdClient::handleDiagnostics(const PublishDiagnosticsParams &params)
     }
 }
 
+void ClangdClient::handleDocumentClosed(TextEditor::TextDocument *doc)
+{
+    d->highlighters.erase(doc);
+}
+
 QVersionNumber ClangdClient::versionNumber() const
 {
     if (d->versionNumber)
@@ -1715,6 +1741,653 @@ void ClangdClient::Private::setHelpItemForTooltip(const MessageId &token, const
         q->hoverHandler()->setHelpItem(token, helpItem);
 }
 
+static void collectExtraResults(QFutureInterface<TextEditor::HighlightingResult> &future,
+                                TextEditor::HighlightingResults &results, const AstNode &ast,
+                                QTextDocument *doc, const QString &docContent)
+{
+    if (!ast.isValid())
+        return;
+
+    static const auto lessThan = [](const TextEditor::HighlightingResult &r1,
+                                    const TextEditor::HighlightingResult &r2) {
+        return r1.line < r2.line || (r1.line == r2.line && r1.column < r2.column)
+                || (r1.line == r2.line && r1.column == r2.column && r1.length < r2.length);
+    };
+    const auto insert = [&](const TextEditor::HighlightingResult &result) {
+        if (!result.isValid()) // Some nodes don't have a range.
+            return;
+        const auto it = std::lower_bound(results.begin(), results.end(), result, lessThan);
+        if (it == results.end() || *it != result) {
+            qCDebug(clangdLog) << "adding additional highlighting result"
+                               << result.line << result.column << result.length;
+            results.insert(it, result);
+            return;
+        }
+
+        // This is for conversion operators, whose type part is only reported as a type by clangd.
+        if ((it->textStyles.mainStyle == TextEditor::C_TYPE
+             || it->textStyles.mainStyle == TextEditor::C_PRIMITIVE_TYPE)
+                && !result.textStyles.mixinStyles.empty()
+                && result.textStyles.mixinStyles.at(0) == TextEditor::C_OPERATOR) {
+            it->textStyles.mixinStyles = result.textStyles.mixinStyles;
+        }
+    };
+    const auto setFromRange = [doc](TextEditor::HighlightingResult &result, const Range &range) {
+        if (!range.isValid())
+            return;
+        const Position startPos = range.start();
+        const Position endPos = range.end();
+        result.line = startPos.line() + 1;
+        result.column = startPos.character() + 1;
+        result.length = endPos.toPositionInDocument(doc) - startPos.toPositionInDocument(doc);
+    };
+    static const auto onlyIndexOf = [](const QStringView &view, const QStringView &s,
+                                       int from = 0) {
+        const int firstIndex = view.indexOf(s, from);
+        if (firstIndex == -1)
+            return -1;
+        const int nextIndex = view.indexOf(s, firstIndex + 1);
+
+        // The second condion deals with the off-by-one error in TemplateSpecialization nodes;
+        // see below.
+        return nextIndex == -1 || nextIndex == firstIndex + 1 ? firstIndex : -1;
+    };
+
+    QList<AstNode> nodes = {ast};
+    while (!nodes.isEmpty()) {
+        if (future.isCanceled())
+            return;
+        const AstNode node = nodes.takeFirst();
+        const QList<AstNode> children = node.children().value_or(QList<AstNode>());
+        nodes << children;
+
+        if (node.kind().endsWith("Literal")) {
+            TextEditor::HighlightingResult result;
+            result.useTextSyles = true;
+            const bool isStringLike = node.kind().startsWith("String")
+                    || node.kind().startsWith("Character");
+            result.textStyles.mainStyle = isStringLike
+                    ? TextEditor::C_STRING : TextEditor::C_NUMBER;
+            setFromRange(result, node.range());
+            insert(result);
+            continue;
+        }
+        if (node.role() == "type" && node.kind() == "Builtin") {
+            TextEditor::HighlightingResult result;
+            result.useTextSyles = true;
+            result.textStyles.mainStyle = TextEditor::C_PRIMITIVE_TYPE;
+            setFromRange(result, node.range());
+            insert(result);
+            continue;
+        }
+
+        const bool isExpression = node.role() == "expression";
+        const bool isDeclaration = node.role() == "declaration";
+
+        // Unfortunately, the exact position of a specific token is usually not
+        // recorded in the AST, so if we need that, we have to search for it textually.
+        // In corner cases, this might get sabotaged by e.g. comments, in which case we give up.
+        const auto posForNodeStart = [doc](const AstNode &node) {
+            return Utils::Text::positionInText(doc, node.range().start().line() + 1,
+                                               node.range().start().character() + 1);
+        };
+        const auto posForNodeEnd = [doc](const AstNode &node) {
+            return Utils::Text::positionInText(doc, node.range().end().line() + 1,
+                                               node.range().end().character() + 1);
+        };
+        const int nodeStartPos = posForNodeStart(node);
+        const int nodeEndPos = posForNodeEnd(node);
+
+        // Match question mark and colon in ternary operators.
+        if (isExpression && node.kind() == "ConditionalOperator") {
+            if (children.size() != 3)
+                continue;
+
+            // The question mark is between sub-expressions 1 and 2, the colon is between
+            // sub-expressions 2 and 3.
+            const int searchStartPosQuestionMark = posForNodeEnd(children.first());
+            const int searchEndPosQuestionMark = posForNodeStart(children.at(1));
+            QStringView content = QStringView(docContent).mid(searchStartPosQuestionMark,
+                searchEndPosQuestionMark - searchStartPosQuestionMark);
+            const int questionMarkPos = onlyIndexOf(content, QStringView(QStringLiteral("?")));
+            if (questionMarkPos == -1)
+                continue;
+            const int searchStartPosColon = posForNodeEnd(children.at(1));
+            const int searchEndPosColon = posForNodeStart(children.at(2));
+            content = QStringView(docContent).mid(searchStartPosColon,
+                searchEndPosColon - searchStartPosColon);
+            const int colonPos = onlyIndexOf(content, QStringView(QStringLiteral(":")));
+            if (colonPos == -1)
+                continue;
+
+            const int absQuestionMarkPos = searchStartPosQuestionMark + questionMarkPos;
+            const int absColonPos = searchStartPosColon + colonPos;
+            if (absQuestionMarkPos > absColonPos)
+                continue;
+
+            TextEditor::HighlightingResult result;
+            result.useTextSyles = true;
+            result.textStyles.mainStyle = TextEditor::C_PUNCTUATION;
+            result.textStyles.mixinStyles.push_back(TextEditor::C_OPERATOR);
+            Utils::Text::convertPosition(doc, absQuestionMarkPos, &result.line, &result.column);
+            result.length = 1;
+            result.kind = CppTools::SemanticHighlighter::TernaryIf;
+            insert(result);
+            Utils::Text::convertPosition(doc, absColonPos, &result.line, &result.column);
+            result.kind = CppTools::SemanticHighlighter::TernaryElse;
+            insert(result);
+            continue;
+        }
+
+        // The following functions are for matching the "<" and ">" brackets of template
+        // declarations, specializations and instantiations.
+        const auto insertAngleBracketInfo = [&docContent, doc, &insert](
+                int searchStart1, int searchEnd1, int searchStart2, int searchEnd2) {
+            const int openingAngleBracketPos = onlyIndexOf(
+                        QStringView(docContent).mid(searchStart1, searchEnd1 - searchStart1),
+                        QStringView(QStringLiteral("<")));
+            if (openingAngleBracketPos == -1)
+                return;
+            const int absOpeningAngleBracketPos = searchStart1 + openingAngleBracketPos;
+            if (absOpeningAngleBracketPos > searchStart2)
+                searchStart2 = absOpeningAngleBracketPos + 1;
+            if (searchStart2 >= searchEnd2)
+                return;
+            const int closingAngleBracketPos = onlyIndexOf(
+                        QStringView(docContent).mid(searchStart2, searchEnd2 - searchStart2),
+                        QStringView(QStringLiteral(">")));
+            if (closingAngleBracketPos == -1)
+                return;
+
+            const int absClosingAngleBracketPos = searchStart2 + closingAngleBracketPos;
+            if (absOpeningAngleBracketPos > absClosingAngleBracketPos)
+                return;
+
+            TextEditor::HighlightingResult result;
+            result.useTextSyles = true;
+            result.textStyles.mainStyle = TextEditor::C_PUNCTUATION;
+            Utils::Text::convertPosition(doc, absOpeningAngleBracketPos,
+                                         &result.line, &result.column);
+            result.length = 1;
+            result.kind = CppTools::SemanticHighlighter::AngleBracketOpen;
+            insert(result);
+            Utils::Text::convertPosition(doc, absClosingAngleBracketPos,
+                                         &result.line, &result.column);
+            result.kind = CppTools::SemanticHighlighter::AngleBracketClose;
+            insert(result);
+        };
+
+        if (isDeclaration && (node.kind() == "FunctionTemplate"
+                              || node.kind() == "ClassTemplate")) {
+            // The child nodes are the template parameters and and the function or class.
+            // The opening angle bracket is before the first child node, the closing angle
+            // bracket is before the function child node and after the last param node.
+            const QString classOrFunctionKind = QLatin1String(node.kind() == "FunctionTemplate"
+                    ? "Function" : "CXXRecord");
+            const auto functionOrClassIt = std::find_if(children.begin(), children.end(),
+                                                 [&classOrFunctionKind](const AstNode &n) {
+                return n.role() == "declaration" && n.kind() == classOrFunctionKind;
+            });
+            if (functionOrClassIt == children.end() || functionOrClassIt == children.begin())
+                continue;
+            const int firstTemplateParamStartPos = posForNodeStart(children.first());
+            const int lastTemplateParamEndPos = posForNodeEnd(*(functionOrClassIt - 1));
+            const int functionOrClassStartPos = posForNodeStart(*functionOrClassIt);
+            insertAngleBracketInfo(nodeStartPos, firstTemplateParamStartPos,
+                                   lastTemplateParamEndPos, functionOrClassStartPos);
+            continue;
+        }
+
+        static const auto findTemplateParam = [](const AstNode &n) {
+            return n.role() == "declaration" && (n.kind() == "TemplateTypeParm"
+                                                 || n.kind() == "NonTypeTemplateParm");
+        };
+
+        if (isDeclaration && node.kind() == "TypeAliasTemplate") {
+            // Children are one node of type TypeAlias and the template parameters.
+            // The opening angle bracket is before the first parameter and the closing
+            // angle bracket is after the last parameter.
+            // The TypeAlias node seems to appear first in the AST, even though lexically
+            // is comes after the parameters. We don't rely on the order here.
+            // Note that there is a second pair of angle brackets. That one is part of
+            // a TemplateSpecialization, which is handled further below.
+            const auto firstTemplateParam = std::find_if(children.begin(), children.end(),
+                                                         findTemplateParam);
+            if (firstTemplateParam == children.end())
+                continue;
+            const auto lastTemplateParam = std::find_if(children.rbegin(), children.rend(),
+                                                        findTemplateParam);
+            QTC_ASSERT(lastTemplateParam != children.rend(), continue);
+            const auto typeAlias = std::find_if(children.begin(), children.end(),
+                [](const AstNode &n) { return n.kind() == "TypeAlias"; });
+            if (typeAlias == children.end())
+                continue;
+
+            const int firstTemplateParamStartPos = posForNodeStart(*firstTemplateParam);
+            const int lastTemplateParamEndPos = posForNodeEnd(*lastTemplateParam);
+            const int searchEndPos = posForNodeStart(*typeAlias);
+            insertAngleBracketInfo(nodeStartPos, firstTemplateParamStartPos,
+                                   lastTemplateParamEndPos, searchEndPos);
+            continue;
+        }
+
+        if (isDeclaration && node.kind() == "ClassTemplateSpecialization") {
+            // There is one child of kind TemplateSpecialization. The first pair
+            // of angle brackets comes before that.
+            if (children.size() == 1) {
+                const int childNodePos = posForNodeStart(children.first());
+                insertAngleBracketInfo(nodeStartPos, childNodePos, nodeStartPos, childNodePos);
+            }
+            continue;
+        }
+
+        if (isDeclaration && node.kind() == "TemplateTemplateParm") {
+            // The child nodes are template arguments and template parameters.
+            // Arguments seem to appear before parameters in the AST, even though they
+            // come after them in the source code. We don't rely on the order here.
+            const auto firstTemplateParam = std::find_if(children.begin(), children.end(),
+                                                         findTemplateParam);
+            if (firstTemplateParam == children.end())
+                continue;
+            const auto lastTemplateParam = std::find_if(children.rbegin(), children.rend(),
+                                                        findTemplateParam);
+            QTC_ASSERT(lastTemplateParam != children.rend(), continue);
+            const auto templateArg = std::find_if(children.begin(), children.end(),
+                [](const AstNode &n) { return n.role() == "template argument"; });
+
+            const int firstTemplateParamStartPos = posForNodeStart(*firstTemplateParam);
+            const int lastTemplateParamEndPos = posForNodeEnd(*lastTemplateParam);
+            const int searchEndPos = templateArg == children.end()
+                    ? nodeEndPos : posForNodeStart(*templateArg);
+            insertAngleBracketInfo(nodeStartPos, firstTemplateParamStartPos,
+                                   lastTemplateParamEndPos, searchEndPos);
+            continue;
+        }
+
+        // {static,dynamic,reinterpret}_cast<>().
+        if (isExpression && node.kind().startsWith("CXX") && node.kind().endsWith("Cast")) {
+            // First child is type, second child is expression.
+            // The opening angle bracket is before the first child, the closing angle bracket
+            // is between the two children.
+            if (children.size() == 2) {
+                insertAngleBracketInfo(nodeStartPos, posForNodeStart(children.first()),
+                                       posForNodeEnd(children.first()),
+                                       posForNodeStart(children.last()));
+            }
+            continue;
+        }
+
+        if (node.kind() == "TemplateSpecialization") {
+            // First comes the template type, then the template arguments.
+            // The opening angle bracket is before the first template argument,
+            // the closing angle bracket is after the last template argument.
+            // The first child node has no range, so we start searching at the parent node.
+            if (children.size() >= 2) {
+                int searchStart2 = posForNodeEnd(children.last());
+                int searchEnd2 = nodeEndPos;
+
+                // There is a weird off-by-one error on the clang side: If there is a
+                // nested template instantiation *and* there is no space between
+                // the closing angle brackets, then the inner TemplateSpecialization node's range
+                // will extend one character too far, covering the outer's closing angle bracket.
+                // This is what we are correcting for here.
+                // TODO: Can we fix this in clang?
+                if (searchStart2 == searchEnd2)
+                    --searchStart2;
+                insertAngleBracketInfo(nodeStartPos, posForNodeStart(children.at(1)),
+                                       searchStart2, searchEnd2);
+            }
+            continue;
+        }
+
+        if (!isExpression && !isDeclaration)
+            continue;
+
+        // Operators, overloaded ones in particular.
+        static const QString operatorPrefix = "operator";
+        QString detail = node.detail().value_or(QString());
+        const bool isCallToNew = node.kind() == "CXXNew";
+        const bool isCallToDelete = node.kind() == "CXXDelete";
+        if (!isCallToNew && !isCallToDelete
+                && (!detail.startsWith(operatorPrefix) || detail == operatorPrefix)) {
+            continue;
+        }
+
+        if (!isCallToNew && !isCallToDelete)
+            detail.remove(0, operatorPrefix.length());
+
+        TextEditor::HighlightingResult result;
+        result.useTextSyles = true;
+        const bool isConversionOp = node.kind() == "CXXConversion";
+        const bool isOverloaded = !isConversionOp
+                && (isDeclaration || ((!isCallToNew && !isCallToDelete)
+                                      || node.arcanaContains("CXXMethod")));
+        result.textStyles.mainStyle = isConversionOp
+                ? TextEditor::C_PRIMITIVE_TYPE
+                : isCallToNew || isCallToDelete || detail.at(0).isSpace()
+                  ? TextEditor::C_KEYWORD : TextEditor::C_PUNCTUATION;
+        result.textStyles.mixinStyles.push_back(TextEditor::C_OPERATOR);
+        if (isOverloaded)
+            result.textStyles.mixinStyles.push_back(TextEditor::C_OVERLOADED_OPERATOR);
+        if (isDeclaration)
+            result.textStyles.mixinStyles.push_back(TextEditor::C_DECLARATION);
+
+        const QStringView nodeText = QStringView(docContent)
+                .mid(nodeStartPos, nodeEndPos - nodeStartPos);
+
+        if (isCallToNew || isCallToDelete) {
+            result.line = node.range().start().line() + 1;
+            result.column = node.range().start().character() + 1;
+            result.length = isCallToNew ? 3 : 6;
+            insert(result);
+            if (node.arcanaContains("array")) {
+                const int openingBracketOffset = nodeText.indexOf('[');
+                if (openingBracketOffset == -1)
+                    continue;
+                const int closingBracketOffset = nodeText.lastIndexOf(']');
+                if (closingBracketOffset == -1 || closingBracketOffset < openingBracketOffset)
+                    continue;
+
+                result.textStyles.mainStyle = TextEditor::C_PUNCTUATION;
+                result.length = 1;
+                Utils::Text::convertPosition(doc,
+                                             nodeStartPos + openingBracketOffset,
+                                             &result.line, &result.column);
+                insert(result);
+                Utils::Text::convertPosition(doc,
+                                             nodeStartPos + closingBracketOffset,
+                                             &result.line, &result.column);
+                insert(result);
+            }
+            continue;
+        }
+
+        if (isExpression && (detail == QLatin1String("()") || detail == QLatin1String("[]"))) {
+            result.line = node.range().start().line() + 1;
+            result.column = node.range().start().character() + 1;
+            result.length = 1;
+            insert(result);
+            result.line = node.range().end().line() + 1;
+            result.column = node.range().end().character();
+            insert(result);
+            continue;
+        }
+
+        const int opStringLen = detail.at(0).isSpace() ? detail.length() - 1 : detail.length();
+
+        // The simple case: Call to operator+, +=, * etc.
+        if (nodeEndPos - nodeStartPos == opStringLen) {
+            setFromRange(result, node.range());
+            insert(result);
+            continue;
+        }
+
+        const int prefixOffset = nodeText.indexOf(operatorPrefix);
+        if (prefixOffset == -1)
+            continue;
+
+        const bool isArray = detail == "[]";
+        const bool isCall = detail == "()";
+        const bool isArrayNew = detail == " new[]";
+        const bool isArrayDelete = detail == " delete[]";
+        const QStringView searchTerm = isArray || isCall
+                ? QStringView(detail).chopped(1) : isArrayNew || isArrayDelete
+                  ? QStringView(detail).chopped(2) : detail;
+        const int opStringOffset = nodeText.indexOf(searchTerm, prefixOffset
+                                                    + operatorPrefix.length());
+        if (opStringOffset == -1 || nodeText.indexOf(operatorPrefix, opStringOffset) != -1)
+            continue;
+
+        const int opStringOffsetInDoc = nodeStartPos + opStringOffset
+                + detail.length() - opStringLen;
+        Utils::Text::convertPosition(doc, opStringOffsetInDoc, &result.line, &result.column);
+        result.length = opStringLen;
+        if (isArray || isCall)
+            result.length = 1;
+        else if (isArrayNew || isArrayDelete)
+            result.length -= 2;
+        if (!isArray && !isCall)
+            insert(result);
+        if (!isArray && !isCall && !isArrayNew && !isArrayDelete)
+            continue;
+
+        result.textStyles.mainStyle = TextEditor::C_PUNCTUATION;
+        result.length = 1;
+        const int openingParenOffset = nodeText.indexOf(
+                    isCall ? '(' : '[', prefixOffset + operatorPrefix.length());
+        if (openingParenOffset == -1)
+            continue;
+        const int closingParenOffset = nodeText.indexOf(isCall ? ')' : ']', openingParenOffset + 1);
+        if (closingParenOffset == -1 || closingParenOffset < openingParenOffset)
+            continue;
+        Utils::Text::convertPosition(doc, nodeStartPos + openingParenOffset,
+                                     &result.line, &result.column);
+        insert(result);
+        Utils::Text::convertPosition(doc, nodeStartPos + closingParenOffset,
+                                     &result.line, &result.column);
+        insert(result);
+    }
+}
+
+// clangd reports also the #ifs, #elses and #endifs around the disabled code as disabled,
+// and not even in a consistent manner. We don't want this, so we have to clean up here.
+// TODO: Fix in clangd?
+static void cleanupDisabledCode(TextEditor::HighlightingResults &results, QTextDocument *doc,
+                                const QString &docContent)
+{
+    bool inDisabled = false;
+    for (auto it = results.begin(); it != results.end();) {
+        const bool wasInDisabled = inDisabled;
+        if (it->textStyles.mainStyle != TextEditor::C_DISABLED_CODE) {
+            inDisabled = false;
+            ++it;
+            continue;
+        }
+
+        inDisabled = true;
+        const int pos = Utils::Text::positionInText(doc, it->line, it->column);
+        const QStringView content(QStringView(docContent).mid(pos, it->length).trimmed());
+        if (!content.startsWith(QLatin1String("#if"))
+                && !content.startsWith(QLatin1String("#elif"))
+                && !content.startsWith(QLatin1String("#else"))
+                && !content.startsWith(QLatin1String("#endif"))) {
+            ++it;
+            continue;
+        }
+
+        if (!wasInDisabled) {
+            // The #if or #else that starts disabled code should not be disabled.
+            it = results.erase(it);
+            continue;
+        }
+
+        if (wasInDisabled && (it == results.end()
+                || (it + 1)->textStyles.mainStyle != TextEditor::C_DISABLED_CODE)) {
+            // The #else or #endif that ends disabled code should not be disabled.
+            it = results.erase(it);
+            continue;
+        }
+        ++it;
+    }
+}
+
+static void semanticHighlighter(QFutureInterface<TextEditor::HighlightingResult> &future,
+                                const QList<ExpandedSemanticToken> &tokens,
+                                const QString &docContents, const AstNode &ast)
+{
+    if (future.isCanceled()) {
+        future.reportFinished();
+        return;
+    }
+
+    QTextDocument doc(docContents);
+    const auto isOutputParameter = [&ast](const ExpandedSemanticToken &token) {
+        if (token.type != "variable" && token.type != "property" && token.type != "parameter")
+            return false;
+        const Position pos(token.line - 1, token.column - 1);
+        const QList<AstNode> path = getAstPath(ast, Range(pos, pos));
+        if (path.size() < 2)
+            return false;
+        if (path.last().hasConstType())
+            return false;
+        for (auto it = path.rbegin() + 1; it != path.rend(); ++it) {
+            if (it->kind() == "Call" || it->kind() == "CXXConstruct"
+                    || it->kind() == "MemberInitializer") {
+                return true;
+            }
+            if (it->kind().endsWith("Cast") && it->hasConstType())
+                return false;
+        }
+        return false;
+    };
+
+    const auto toResult = [&ast, &isOutputParameter](const ExpandedSemanticToken &token) {
+        TextEditor::TextStyles styles;
+        if (token.type == "variable") {
+            if (token.modifiers.contains("functionScope")) {
+                styles.mainStyle = TextEditor::C_LOCAL;
+            } else if (token.modifiers.contains("classScope")) {
+                styles.mainStyle = TextEditor::C_FIELD;
+            } else if (token.modifiers.contains("fileScope")
+                       || token.modifiers.contains("globalScope")) {
+                styles.mainStyle = TextEditor::C_GLOBAL;
+            }
+        } else if (token.type == "function" || token.type == "method") {
+            styles.mainStyle = TextEditor::C_FUNCTION;
+            if (ast.isValid()) {
+                const Position pos(token.line - 1, token.column - 1);
+                const QList<AstNode> path = getAstPath(ast, Range(pos, pos));
+                if (path.length() > 1) {
+                    const AstNode declNode = path.at(path.length() - 2);
+                    if (declNode.kind() == "Function" || declNode.kind() == "CXXMethod") {
+                        if (declNode.arcanaContains("' virtual"))
+                            styles.mainStyle = TextEditor::C_VIRTUAL_METHOD;
+                        if (declNode.hasChildWithRole("statement"))
+                            styles.mixinStyles.push_back(TextEditor::C_FUNCTION_DEFINITION);
+                    }
+                }
+            }
+        } else if (token.type == "class") {
+            styles.mainStyle = TextEditor::C_TYPE;
+
+            // clang hardly ever differentiates between constructors and the associated class,
+            // whereas we highlight constructors as functions.
+            if (ast.isValid()) {
+                const Position pos(token.line - 1, token.column - 1);
+                const QList<AstNode> path = getAstPath(ast, Range(pos, pos));
+                if (!path.isEmpty()) {
+                    if (path.last().kind() == "CXXConstructor") {
+                        if (!path.last().arcanaContains("implicit"))
+                            styles.mainStyle = TextEditor::C_FUNCTION;
+                    } else if (path.last().kind() == "Record" && path.length() > 1) {
+                        const AstNode node = path.at(path.length() - 2);
+                        if (node.kind() == "CXXDestructor" && !node.arcanaContains("implicit")) {
+                            styles.mainStyle = TextEditor::C_FUNCTION;
+                            // TODO: "declaration" modifier is missing for destructors; fix in clangd
+                            // (the scope is also wrong)
+                            if (node.role() == "declaration")
+                                styles.mixinStyles.push_back(TextEditor::C_DECLARATION);
+                        }
+                    }
+                }
+            }
+        } else if (token.type == "comment") { // "comment" means code disabled via the preprocessor
+            styles.mainStyle = TextEditor::C_DISABLED_CODE;
+        } else if (token.type == "namespace") {
+            styles.mainStyle = TextEditor::C_TYPE;
+        } else if (token.type == "property") {
+            styles.mainStyle = TextEditor::C_FIELD;
+        } else if (token.type == "enum") {
+            styles.mainStyle = TextEditor::C_TYPE;
+            styles.mixinStyles.push_back(TextEditor::C_ENUMERATION);
+        } else if (token.type == "enumMember") {
+            styles.mainStyle = TextEditor::C_ENUMERATION;
+        } else if (token.type == "parameter") {
+            styles.mainStyle = TextEditor::C_PARAMETER;
+        } else if (token.type == "macro") {
+            styles.mainStyle = TextEditor::C_PREPROCESSOR;
+        } else if (token.type == "type") {
+            styles.mainStyle = TextEditor::C_TYPE;
+        } else if (token.type == "typeParameter") {
+            styles.mainStyle = TextEditor::C_TYPE;
+        }
+        if (token.modifiers.contains("declaration"))
+            styles.mixinStyles.push_back(TextEditor::C_DECLARATION);
+        if (isOutputParameter(token))
+            styles.mixinStyles.push_back(TextEditor::C_OUTPUT_ARGUMENT);
+        qCDebug(clangdLog) << "adding highlighting result"
+                           << token.line << token.column << token.length << int(styles.mainStyle);
+        return TextEditor::HighlightingResult(token.line, token.column, token.length, styles);
+    };
+
+    TextEditor::HighlightingResults results = Utils::transform(tokens, toResult);
+    cleanupDisabledCode(results, &doc, docContents);
+    collectExtraResults(future, results, ast, &doc, docContents);
+    if (!future.isCanceled()) {
+        qCDebug(clangdLog) << "reporting" << results.size() << "highlighting results";
+        future.reportResults(QVector<TextEditor::HighlightingResult>(results.cbegin(),
+                                                                     results.cend()));
+    }
+    future.reportFinished();
+}
+
+// Unfortunately, clangd ignores almost everything except symbols when sending
+// semantic token info, so we need to consult the AST for additional information.
+// In particular, we inspect the following constructs:
+//    - Raw string literals, because our built-in lexer does not parse them properly.
+//      While we're at it, we also handle other types of literals.
+//    - Ternary expressions (for the matching of "?" and ":").
+//    - Template declarations and instantiations (for the matching of "<" and ">").
+//    - Function declarations, to find out whether a declaration is also a definition.
+//    - Function arguments, to find out whether they correspond to output parameters.
+//    - We consider most other tokens to be simple enough to be handled by the built-in code model.
+//      Sometimes we have no choice, as for #include directives, which appear neither
+//      in the semantic tokens nor in the AST.
+void ClangdClient::Private::handleSemanticTokens(TextEditor::TextDocument *doc,
+                                                 const QList<ExpandedSemanticToken> &tokens)
+{
+    qCDebug(clangdLog()) << "handling LSP tokens" << tokens.size();
+    for (const ExpandedSemanticToken &t : tokens)
+        qCDebug(clangdLog) << '\t' << t.line << t.column << t.length << t.type << t.modifiers;
+
+    // TODO: Cache ASTs
+    AstParams params(TextDocumentIdentifier(DocumentUri::fromFilePath(doc->filePath())));
+    AstRequest astReq(params);
+    astReq.setResponseCallback([this, tokens, doc](const AstRequest::Response &response) {
+        if (!q->documentOpen(doc))
+            return;
+        const Utils::optional<AstNode> ast = response.result();
+        if (ast && clangdLog().isDebugEnabled())
+            ast->print();
+
+        const auto runner = [tokens, text = doc->document()->toPlainText(),
+                             theAst = ast ? *ast : AstNode()] {
+            return Utils::runAsync(semanticHighlighter, tokens, text, theAst);
+        };
+
+        if (isTesting) {
+            const auto watcher = new QFutureWatcher<TextEditor::HighlightingResult>(q);
+            connect(watcher, &QFutureWatcher<TextEditor::HighlightingResult>::finished,
+                    q, [this, watcher] {
+                emit q->highlightingResultsReady(watcher->future().results());
+                watcher->deleteLater();
+            });
+            watcher->setFuture(runner());
+            return;
+        }
+
+        auto it = highlighters.find(doc);
+        if (it == highlighters.end()) {
+            it = highlighters.emplace(doc, doc).first;
+        } else {
+            it->second.updateFormatMapFromFontSettings();
+        }
+        it->second.setHighlightingRunner(runner);
+        it->second.run();
+    });
+    q->sendContent(astReq);
+}
+
 void ClangdClient::VirtualFunctionAssistProcessor::cancel()
 {
     resetData();

src/plugins/clangcodemodel/clangdclient.h

@@ -83,9 +83,11 @@ signals:
     void foundReferences(const QList<Core::SearchResultItem> &items);
     void findUsagesDone();
     void helpItemGathered(const Core::HelpItem &helpItem);
+    void highlightingResultsReady(const TextEditor::HighlightingResults &results);
 
 private:
     void handleDiagnostics(const LanguageServerProtocol::PublishDiagnosticsParams &params) override;
+    void handleDocumentClosed(TextEditor::TextDocument *doc) override;
 
     class Private;
     class FollowSymbolData;

src/plugins/clangcodemodel/clangeditordocumentprocessor.cpp

@@ -137,6 +137,8 @@ void ClangEditorDocumentProcessor::semanticRehighlight()
     };
     if (!Utils::contains(Core::EditorManager::visibleEditors(), matchesEditor))
         return;
+    if (ClangModelManagerSupport::instance()->clientForFile(m_document.filePath()))
+        return;
 
     m_semanticHighlighter.updateFormatMapFromFontSettings();
     if (m_projectPart)
@@ -254,6 +256,8 @@ void ClangEditorDocumentProcessor::updateHighlighting(
         const QVector<ClangBackEnd::SourceRangeContainer> &skippedPreprocessorRanges,
         uint documentRevision)
 {
+    if (ClangModelManagerSupport::instance()->clientForFile(m_document.filePath()))
+        return;
     if (documentRevision == revision()) {
         const auto skippedPreprocessorBlocks = toTextEditorBlocks(textDocument(), skippedPreprocessorRanges);
         emit ifdefedOutBlocksUpdated(documentRevision, skippedPreprocessorBlocks);

src/plugins/clangcodemodel/test/clangdtests.cpp

@@ -35,12 +35,14 @@
 #include <cpptools/cppcodemodelsettings.h>
 #include <cpptools/cpptoolsreuse.h>
 #include <cpptools/cpptoolstestcase.h>
+#include <cpptools/semantichighlighter.h>
 #include <coreplugin/editormanager/editormanager.h>
 #include <projectexplorer/kitmanager.h>
 #include <projectexplorer/project.h>
 #include <projectexplorer/projectexplorer.h>
 #include <qtsupport/qtkitinformation.h>
 #include <utils/algorithm.h>
+#include <utils/textutils.h>
 
 #include <QEventLoop>
 #include <QFileInfo>
@@ -54,6 +56,7 @@ using namespace CPlusPlus;
 using namespace Core;
 using namespace CppTools::Tests;
 using namespace ProjectExplorer;
+using namespace TextEditor;
 
 namespace ClangCodeModel {
 namespace Internal {
@@ -601,7 +604,7 @@ void ClangdTestTooltips::test()
     QCOMPARE(editor->document(), doc);
     QVERIFY(editor->editorWidget());
 
-    if (QLatin1String(QTest::currentDataTag()) == "IncludeDirective")
+    if (QLatin1String(QTest::currentDataTag()) == QLatin1String("IncludeDirective"))
         QSKIP("FIXME: clangd sends empty or no hover data for includes");
 
     QTimer timer;
@@ -633,6 +636,689 @@ void ClangdTestTooltips::test()
     QCOMPARE(helpItem.docMark(), expectedMark);
 }
 
+ClangdTestHighlighting::ClangdTestHighlighting()
+{
+    setProjectFileName("highlighting.pro");
+    setSourceFileNames({"highlighting.cpp"});
+    setMinimumVersion(13);
+}
+
+void ClangdTestHighlighting::initTestCase()
+{
+    ClangdTest::initTestCase();
+
+    QTimer timer;
+    timer.setSingleShot(true);
+    QEventLoop loop;
+    QObject::connect(&timer, &QTimer::timeout, &loop, &QEventLoop::quit);
+    const auto handler = [this, &loop](const TextEditor::HighlightingResults &results) {
+        m_results = results;
+        loop.quit();
+    };
+    connect(client(), &ClangdClient::highlightingResultsReady, handler);
+    timer.start(10000);
+    loop.exec();
+    QVERIFY(timer.isActive());
+    QVERIFY(!m_results.isEmpty());
+}
+
+void ClangdTestHighlighting::test_data()
+{
+    QTest::addColumn<int>("firstLine");
+    QTest::addColumn<int>("startColumn");
+    QTest::addColumn<int>("lastLine");
+    QTest::addColumn<int>("endColumn");
+    QTest::addColumn<QList<int>>("expectedStyles");
+    QTest::addColumn<int>("expectedKind");
+
+    QTest::newRow("string literal") << 1 << 24 << 1 << 34 << QList<int>{C_STRING} << 0;
+    QTest::newRow("UTF-8 string literal") << 2 << 24 << 2 << 36 << QList<int>{C_STRING} << 0;
+    QTest::newRow("raw string literal") << 3 << 24 << 4 << 9 << QList<int>{C_STRING} << 0;
+    QTest::newRow("character literal") << 5 << 24 << 5 << 27 << QList<int>{C_STRING} << 0;
+    QTest::newRow("integer literal") << 23 << 24 << 23 << 25 << QList<int>{C_NUMBER} << 0;
+    QTest::newRow("float literal") << 24 << 24 << 24 << 28 << QList<int>{C_NUMBER} << 0;
+    QTest::newRow("function definition") << 45 << 5 << 45 << 13
+        << QList<int>{C_FUNCTION, C_FUNCTION_DEFINITION, C_DECLARATION} << 0;
+    QTest::newRow("member function definition") << 52 << 10 << 52 << 24
+        << QList<int>{C_FUNCTION, C_FUNCTION_DEFINITION, C_DECLARATION} << 0;
+    QTest::newRow("virtual member function definition outside of class body")
+        << 586 << 17 << 586 << 32
+        << QList<int>{C_VIRTUAL_METHOD, C_FUNCTION_DEFINITION, C_DECLARATION} << 0;
+    QTest::newRow("virtual member function definition inside class body")
+        << 589 << 16 << 589 << 41
+        << QList<int>{C_VIRTUAL_METHOD, C_FUNCTION_DEFINITION, C_DECLARATION} << 0;
+    QTest::newRow("function declaration") << 55 << 5 << 55 << 24
+        << QList<int>{C_FUNCTION, C_DECLARATION} << 0;
+    QTest::newRow("member function declaration") << 59 << 10 << 59 << 24
+        << QList<int>{C_FUNCTION, C_DECLARATION} << 0;
+    QTest::newRow("member function call") << 104 << 9 << 104 << 32
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("function call") << 64 << 5 << 64 << 13 << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("type conversion function (struct)") << 68 << 14 << 68 << 17
+        << QList<int>{C_TYPE, C_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("type conversion function (built-in)") << 69 << 14 << 69 << 17
+        << QList<int>{C_PRIMITIVE_TYPE, C_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("type reference") << 74 << 5 << 74 << 8 << QList<int>{C_TYPE} << 0;
+    QTest::newRow("local variable declaration") << 79 << 9 << 79 << 12
+        << QList<int>{C_LOCAL, C_DECLARATION} << 0;
+    QTest::newRow("local variable reference") << 81 << 5 << 81 << 8 << QList<int>{C_LOCAL} << 0;
+    QTest::newRow("function parameter declaration") << 84 << 41 << 84 << 44
+        << QList<int>{C_PARAMETER, C_DECLARATION} << 0;
+    QTest::newRow("function parameter reference") << 86 << 5 << 86 << 8
+        << QList<int>{C_PARAMETER} << 0;
+    QTest::newRow("member declaration") << 90 << 9 << 90 << 20
+        << QList<int>{C_FIELD, C_DECLARATION} << 0;
+    QTest::newRow("member reference") << 94 << 9 << 94 << 20 << QList<int>{C_FIELD} << 0;
+    QTest::newRow("static member function declaration") << 110 << 10 << 110 << 22
+        << QList<int>{C_FUNCTION, C_DECLARATION} << 0;
+    QTest::newRow("static member function call") << 114 << 15 << 114 << 27
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("enum declaration") << 118 << 6 << 118 << 17
+        << QList<int>{C_TYPE, C_ENUMERATION, C_DECLARATION} << 0;
+    QTest::newRow("enumerator declaration") << 120 << 5 << 120 << 15
+        << QList<int>{C_ENUMERATION, C_DECLARATION} << 0;
+    QTest::newRow("enum in variable declaration") << 125 << 5 << 125 << 16
+        << QList<int>{C_TYPE, C_ENUMERATION} << 0;
+    QTest::newRow("enum variable declaration") << 125 << 17 << 125 << 28
+        << QList<int>{C_LOCAL, C_DECLARATION} << 0;
+    QTest::newRow("enum variable reference") << 127 << 5 << 127 << 16 << QList<int>{C_LOCAL} << 0;
+    QTest::newRow("enumerator reference") << 127 << 19 << 127 << 29
+        << QList<int>{C_ENUMERATION} << 0;
+    QTest::newRow("forward declaration") << 130 << 7 << 130 << 23
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("constructor declaration") << 134 << 5 << 134 << 10
+        << QList<int>{C_FUNCTION, C_DECLARATION} << 0;
+    QTest::newRow("destructor declaration") << 135 << 6 << 135 << 11
+        << QList<int>{C_FUNCTION, C_DECLARATION} << 0;
+    QTest::newRow("reference to forward-declared class") << 138 << 1 << 138 << 17
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("class in variable declaration") << 140 << 5 << 140 << 10
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("class variable declaration") << 140 << 11 << 140 << 31
+        << QList<int>{C_LOCAL, C_DECLARATION} << 0;
+    QTest::newRow("union declaration") << 145 << 7 << 145 << 12
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("union in global variable declaration") << 150 << 1 << 150 << 6
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("global variable declaration") << 150 << 7 << 150 << 32
+        << QList<int>{C_GLOBAL, C_DECLARATION} << 0;
+    QTest::newRow("struct declaration") << 50 << 8 << 50 << 11
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("namespace declaration") << 160 << 11 << 160 << 20
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("namespace alias declaration") << 164 << 11 << 164 << 25
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("struct in namespaced using declaration") << 165 << 18 << 165 << 35
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("namespace reference") << 166 << 1 << 166 << 10 << QList<int>{C_TYPE} << 0;
+    QTest::newRow("namespaced struct in global variable declaration") << 166 << 12 << 166 << 29
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("virtual function declaration") << 170 << 18 << 170 << 33
+        << QList<int>{C_VIRTUAL_METHOD, C_DECLARATION} << 0;
+    QTest::newRow("virtual function call via pointer") << 192 << 33 << 192 << 48
+        << QList<int>{C_VIRTUAL_METHOD} << 0;
+    QTest::newRow("final virtual function call via pointer") << 202 << 38 << 202 << 58
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("non-final virtual function call via pointer") << 207 << 41 << 207 << 61
+        << QList<int>{C_VIRTUAL_METHOD} << 0;
+    QTest::newRow("operator+ declaration") << 220 << 18 << 220 << 19
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator+ call") << 224 << 36 << 224 << 37
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator+= call") << 226 << 24 << 226 << 26
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator* member declaration") << 604 << 18 << 604 << 19
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator* non-member declaration") << 607 << 14 << 607 << 15
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator* member call") << 613 << 7 << 613 << 8
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator* non-member call") << 614 << 7 << 614 << 8
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator<<= member declaration") << 618 << 19 << 618 << 22
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator<<= call") << 629 << 12 << 629 << 15
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("integer literal 2") << 629 << 16 << 629 << 17 << QList<int>{C_NUMBER} << 0;
+    QTest::newRow("operator(int) member declaration (opening paren") << 619 << 19 << 619 << 20
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator(int) member declaration (closing paren") << 619 << 20 << 619 << 21
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator(int) call (opening parenthesis)") << 632 << 12 << 632 << 13
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator(int) call (argument)") << 632 << 13 << 632 << 14
+        << QList<int>{C_NUMBER} << 0;
+    QTest::newRow("operator(int) call (closing parenthesis)") << 632 << 14 << 632 << 15
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator[] member declaration (opening bracket") << 620 << 18 << 620 << 19
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator[] member declaration (closing bracket") << 620 << 20 << 620 << 21
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator[] call (opening bracket)") << 633 << 12 << 633 << 13
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator[] call (argument)") << 633 << 13 << 633 << 14
+        << QList<int>{C_NUMBER} << 0;
+    QTest::newRow("operator[] call (closing bracket)") << 633 << 14 << 633 << 15
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator new member declaration") << 621 << 20 << 621 << 23
+        << QList<int>{C_KEYWORD, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator new member call") << 635 << 22 << 635 << 25
+        << QList<int>{C_KEYWORD, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator delete member declaration") << 622 << 19 << 622 << 25
+        << QList<int>{C_KEYWORD, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator delete member call") << 636 << 5 << 636 << 11
+        << QList<int>{C_KEYWORD, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("var after operator delete member call") << 636 << 12 << 636 << 19
+        << QList<int>{C_LOCAL} << 0;
+    QTest::newRow("operator new[] member declaration (keyword)") << 623 << 20 << 623 << 23
+        << QList<int>{C_KEYWORD, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator new[] member declaration (opening bracket)") << 623 << 23 << 623 << 24
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator new[] member declaration (closing bracket)") << 623 << 24 << 623 << 25
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator new[] member call (keyword") << 637 << 19 << 637 << 22
+        << QList<int>{C_KEYWORD, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator new[] member call (type argument)") << 637 << 23 << 637 << 28
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("operator new[] member call (opening bracket)") << 637 << 28 << 637 << 29
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator new[] member call (size argument)") << 637 << 29 << 637 << 31
+        << QList<int>{C_NUMBER} << 0;
+    QTest::newRow("operator new[] member call (closing bracket)") << 637 << 31 << 637 << 32
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator delete[] member declaration (keyword)") << 624 << 19 << 624 << 25
+        << QList<int>{C_KEYWORD, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator delete[] member declaration (opening bracket)")
+        << 624 << 25 << 624 << 26
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator delete[] member declaration (closing bracket)")
+        << 624 << 26 << 624 << 27
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator delete[] member call (keyword") << 638 << 5 << 638 << 11
+        << QList<int>{C_KEYWORD, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator delete[] member call (opening bracket)") << 638 << 12 << 638 << 13
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator delete[] member call (closing bracket)") << 638 << 13 << 638 << 14
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator new built-in call") << 634 << 14 << 634 << 17
+        << QList<int>{C_KEYWORD, C_OPERATOR} << 0;
+    QTest::newRow("operator() member declaration (opening paren") << 654 << 20 << 654 << 21
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator() member declaration (closing paren") << 654 << 21 << 654 << 22
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator() call (opening parenthesis)") << 662 << 11 << 662 << 12
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator() call (closing parenthesis)") << 662 << 12 << 662 << 13
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator* member declaration (2)") << 655 << 17 << 655 << 18
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator* member call (2)") << 663 << 5 << 663 << 6
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("operator= member declaration") << 656 << 20 << 656 << 21
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR, C_DECLARATION} << 0;
+    QTest::newRow("operator= call") << 664 << 12 << 664 << 13
+        << QList<int>{C_PUNCTUATION, C_OPERATOR, C_OVERLOADED_OPERATOR} << 0;
+    QTest::newRow("ternary operator (question mark)") << 668 << 18 << 668 << 19
+        << QList<int>{C_PUNCTUATION, C_OPERATOR} << int(CppTools::SemanticHighlighter::TernaryIf);
+    QTest::newRow("ternary operator (colon)") << 668 << 23 << 668 << 24
+        << QList<int>{C_PUNCTUATION, C_OPERATOR} << int(CppTools::SemanticHighlighter::TernaryElse);
+    QTest::newRow("opening angle bracket in function template declaration")
+        << 247 << 10 << 247 << 11
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("closing angle bracket in function template declaration")
+        << 247 << 18 << 247 << 19
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("opening angle bracket in class template declaration") << 261 << 10 << 261 << 11
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("closing angle bracket in class template declaration") << 261 << 18 << 261 << 19
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("macro definition") << 231 << 9 << 231 << 31
+        << QList<int>{C_PREPROCESSOR, C_DECLARATION} << 0;
+    QTest::newRow("function-like macro definition") << 232 << 9 << 232 << 24
+        << QList<int>{C_PREPROCESSOR, C_DECLARATION} << 0;
+    QTest::newRow("function-like macro call") << 236 << 5 << 236 << 20
+        << QList<int>{C_PREPROCESSOR} << 0;
+    QTest::newRow("function-like macro call argument 1") << 236 << 21 << 236 << 22
+        << QList<int>{C_NUMBER} << 0;
+    QTest::newRow("function-like macro call argument 2") << 236 << 24 << 236 << 25
+        << QList<int>{C_NUMBER} << 0;
+    QTest::newRow("function template call") << 254 << 5 << 254 << 21 << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("template type parameter") << 265 << 17 << 265 << 38
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("template parameter default argument") << 265 << 41 << 265 << 44
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("template non-type parameter") << 265 << 50 << 265 << 74
+        << QList<int>{C_LOCAL, C_DECLARATION} << 0;
+    QTest::newRow("template non-type parameter default argument") << 265 << 77 << 265 << 78
+        << QList<int>{C_NUMBER} << 0;
+    QTest::newRow("template template parameter") << 265 << 103 << 265 << 128
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("template template parameter default argument") << 265 << 131 << 265 << 142
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("outer opening angle bracket in nested template declaration")
+        << 265 << 10 << 265 << 11
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("inner opening angle bracket in nested template declaration")
+        << 265 << 89 << 265 << 90
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("inner closing angle bracket in nested template declaration")
+        << 265 << 95 << 265 << 96
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("outer closing angle bracket in nested template declaration")
+        << 265 << 142 << 265 << 143
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("function template declaration") << 266 << 6 << 266 << 22
+        << QList<int>{C_FUNCTION, C_FUNCTION_DEFINITION, C_DECLARATION} << 0;
+    QTest::newRow("template type parameter reference") << 268 << 5 << 268 << 26
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("local var declaration of template parameter type") << 268 << 27 << 268 << 57
+        << QList<int>{C_LOCAL, C_DECLARATION} << 0;
+    QTest::newRow("reference to non-type template parameter") << 269 << 46 << 269 << 70
+        << QList<int>{C_LOCAL} << 0;
+    QTest::newRow("local var declaration initialized with non-type template parameter")
+        << 269 << 10 << 269 << 43
+        << QList<int>{C_LOCAL, C_DECLARATION} << 0;
+    QTest::newRow("template template parameter reference") << 270 << 5 << 270 << 30
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("template type parameter reference in template instantiation")
+        << 270 << 31 << 270 << 52
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("local var declaration of template template parameter type")
+        << 270 << 54 << 270 << 88
+        << QList<int>{C_LOCAL, C_DECLARATION} << 0;
+    QTest::newRow("local variable declaration as argument to function-like macro call")
+        << 302 << 18 << 302 << 23
+        << QList<int>{C_LOCAL, C_DECLARATION} << 0;
+    QTest::newRow("local variable as argument to function-like macro call")
+        << 302 << 25 << 302 << 34
+        << QList<int>{C_LOCAL} << 0;
+    QTest::newRow("class member as argument to function-like macro call")
+        << 310 << 29 << 310 << 38
+        << QList<int>{C_FIELD} << 0;
+    QTest::newRow("enum declaration with underlying type") << 316 << 6 << 316 << 21
+        << QList<int>{C_TYPE, C_ENUMERATION, C_DECLARATION} << 0;
+    QTest::newRow("type in static_cast") << 328 << 23 << 328 << 33
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("opening angle bracket in static_cast") << 328 << 16 << 328 << 17
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("closing angle bracket in static_cast") << 328 << 39 << 328 << 40
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("type in reinterpret_cast") << 329 << 28 << 329 << 38
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("integer alias declaration") << 333 << 7 << 333 << 25
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("integer alias in declaration") << 341 << 5 << 341 << 17
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("recursive integer alias in declaration") << 342 << 5 << 342 << 23
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("integer typedef in declaration") << 343 << 5 << 343 << 19
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("call to function pointer alias") << 344 << 5 << 344 << 13
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("friend class declaration") << 350 << 18 << 350 << 27
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("friend class reference") << 351 << 34 << 351 << 43
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("function parameter of friend class type") << 351 << 45 << 351 << 50
+        << QList<int>{C_PARAMETER, C_DECLARATION} << 0;
+    QTest::newRow("constructor member initialization") << 358 << 9 << 358 << 15
+        << QList<int>{C_FIELD} << 0;
+    QTest::newRow("call to function template") << 372 << 5 << 372 << 25
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("class template declaration") << 377 << 7 << 377 << 20
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("class template instantiation (name)") << 384 << 5 << 384 << 18
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("class template instantiation (opening angle bracket)") << 384 << 18 << 384 << 19
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("class template instantiation (closing angle bracket)") << 384 << 22 << 384 << 23
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("namespace in declaration") << 413 << 4 << 413 << 26 << QList<int>{C_TYPE} << 0;
+    QTest::newRow("namespaced class in declaration") << 413 << 28 << 413 << 41
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("class as template argument in declaration") << 413 << 42 << 413 << 52
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("local variable declaration of template instance type") << 413 << 54 << 413 << 77
+        << QList<int>{C_LOCAL, C_DECLARATION} << 0;
+    QTest::newRow("local typedef declaration") << 418 << 17 << 418 << 35
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("local typedef in variable declaration") << 419 << 5 << 419 << 23
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("non-const reference argument") << 455 << 31 << 455 << 32
+        << QList<int>{C_LOCAL, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("const reference argument") << 464 << 28 << 464 << 29
+        << QList<int>{C_LOCAL} << 0;
+    QTest::newRow("rvalue reference argument") << 473 << 48 << 473 << 49
+        << QList<int>{C_LOCAL} << 0;
+    QTest::newRow("non-const pointer argument") << 482 << 29 << 482 << 30
+        << QList<int>{C_LOCAL, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("pointer to const argument") << 490 << 28 << 490 << 29
+        << QList<int>{C_LOCAL} << 0;
+    QTest::newRow("const pointer argument") << 491 << 26 << 491 << 27
+        << QList<int>{C_LOCAL, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("non-const reference via member function call as output argument (object)")
+        << 580 << 29 << 580 << 30
+        << QList<int>{C_LOCAL, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("non-const reference via member function call as output argument (function)")
+        << 580 << 31 << 580 << 37
+        << QList<int>{C_FUNCTION, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("value argument") << 501 << 57 << 501 << 58
+        << QList<int>{C_LOCAL} << 0;
+    QTest::newRow("non-const ref argument as second arg") << 501 << 61 << 501 << 62
+        << QList<int>{C_LOCAL, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("non-const ref argument from function parameter") << 506 << 31 << 506 << 40
+        << QList<int>{C_PARAMETER, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("non-const pointer argument expression") << 513 << 30 << 513 << 31
+        << QList<int>{C_LOCAL, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("non-const ref argument from qualified member (object)") << 525 << 31 << 525 << 39
+        << QList<int>{C_LOCAL, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("non-const ref argument from qualified member (member)") << 525 << 40 << 525 << 46
+        << QList<int>{C_FIELD, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("non-const ref argument to constructor") << 540 << 47 << 540 << 55
+        << QList<int>{C_LOCAL, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("non-const ref argument to member initialization") << 546 << 15 << 546 << 18
+        << QList<int>{C_PARAMETER, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("typedef as underlying type in enum declaration") << 424 << 21 << 424 << 39
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("argument to user-defined subscript operator") << 434 << 12 << 434 << 17
+        << QList<int>{C_PARAMETER} << 0;
+    QTest::newRow("partial class template specialization") << 553 << 25 << 553 << 28
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("using declaration for function") << 556 << 10 << 556 << 13
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("variable in operator() call") << 566 << 7 << 566 << 10
+        << QList<int>{C_PARAMETER} << 0;
+    QTest::newRow("using declaration for function template") << 584 << 10 << 584 << 16
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("Q_PROPERTY (macro name)") << 599 << 5 << 599 << 15
+        << QList<int>{C_PREPROCESSOR} << 0;
+    QTest::newRow("Q_PROPERTY (property name)") << 599 << 52 << 599 << 56
+        << QList<int>{C_FIELD} << 0;
+    QTest::newRow("Q_PROPERTY (getter)") << 599 << 62 << 599 << 69
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("Q_PROPERTY (notifier)") << 599 << 91 << 599 << 102
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("Q_PROPERTY (type)") << 600 << 22 << 600 << 29
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("multi-line Q_PROPERTY (macro name)") << 704 << 5 << 704 << 15
+        << QList<int>{C_PREPROCESSOR} << 0;
+    QTest::newRow("multi-line Q_PROPERTY (property name)") << 718 << 13 << 718 << 17
+        << QList<int>{C_FIELD} << 0;
+    QTest::newRow("multi-line Q_PROPERTY (getter)") << 722 << 13 << 722 << 20
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("multi-line Q_PROPERTY (notifier)") << 730 << 13 << 730 << 24
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("old-style signal (macro)") << 672 << 5 << 672 << 11
+        << QList<int>{C_PREPROCESSOR} << 0;
+    QTest::newRow("old-style signal (signal)") << 672 << 12 << 672 << 21
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("old-style signal (signal parameter)") << 672 << 22 << 672 << 29
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("old-style slot (macro)") << 673 << 5 << 673 << 9
+        << QList<int>{C_PREPROCESSOR} << 0;
+    QTest::newRow("old-style slot (slot)") << 673 << 10 << 673 << 19
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("old-style slot (slot parameter)") << 673 << 20 << 673 << 27
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("old-style signal with complex parameter (macro)") << 674 << 5 << 674 << 11
+        << QList<int>{C_PREPROCESSOR} << 0;
+    QTest::newRow("old-style signal with complex parameter (signal)") << 674 << 12 << 674 << 21
+        << QList<int>{C_FUNCTION} << 0;
+    QTest::newRow("old-style signal with complex parameter (signal parameter part 1)")
+        << 674 << 22 << 674 << 29
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("old-style signal with complex parameter (signal parameter part 2)")
+        << 674 << 32 << 674 << 37
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("old-style signal with complex parameter (signal parameter part 3)")
+        << 674 << 39 << 674 << 46
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("constructor parameter") << 681 << 64 << 681 << 88
+        << QList<int>{C_PARAMETER, C_DECLARATION} << 0;
+    QTest::newRow("non-const ref argument to constructor (2)") << 686 << 42 << 686 << 45
+        << QList<int>{C_LOCAL, C_OUTPUT_ARGUMENT} << 0;
+    QTest::newRow("local variable captured by lambda") << 442 << 24 << 442 << 27
+        << QList<int>{C_LOCAL} << 0;
+    QTest::newRow("static protected member") << 693 << 16 << 693 << 30
+        << QList<int>{C_FIELD, C_DECLARATION} << 0;
+    QTest::newRow("static private member") << 696 << 16 << 696 << 28
+        << QList<int>{C_FIELD, C_DECLARATION} << 0;
+    QTest::newRow("alias template declaration (opening angle bracket)") << 700 << 10 << 700 << 11
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("alias template declaration (closing angle bracket)") << 700 << 16 << 700 << 17
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("alias template declaration (new type)") << 700 << 24 << 700 << 28
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("alias template declaration (base type)") << 700 << 31 << 700 << 32
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("alias template declaration (base type opening angle bracket)")
+        << 700 << 32 << 700 << 33
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("alias template declaration (base type closing angle bracket)")
+        << 700 << 37 << 700 << 38
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("alias template instantiation (type)") << 701 << 1 << 701 << 5
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("alias template instantiation (opening angle bracket)") << 701 << 5 << 701 << 6
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("alias template instantiation (closing angle bracket)") << 701 << 7 << 701 << 8
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("function template specialization (opening angle bracket 1)")
+        << 802 << 9 << 802 << 10
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("function template specialization (closing angle bracket 1)")
+        << 802 << 10 << 802 << 11
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("function template specialization (function name)")
+        << 802 << 17 << 802 << 29
+        << QList<int>{C_FUNCTION, C_FUNCTION_DEFINITION, C_DECLARATION} << 0;
+    QTest::newRow("function template specialization (opening angle bracket 2)")
+        << 802 << 29 << 802 << 30
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("function template specialization (closing angle bracket 2)")
+        << 802 << 33 << 802 << 34
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("class template specialization (opening angle bracket 1)")
+        << 804 << 9 << 804 << 10
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("class template specialization (closing angle bracket 1)")
+        << 804 << 10 << 804 << 11
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("class template specialization (class name)")
+        << 804 << 18 << 804 << 21
+        << QList<int>{C_TYPE, C_DECLARATION} << 0;
+    QTest::newRow("class template specialization (opening angle bracket 2)")
+        << 804 << 21 << 804 << 22
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("class template specialization (closing angle bracket 2)")
+        << 804 << 25 << 804 << 26
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("structured binding (var 1)") << 737 << 17 << 737 << 18
+        << QList<int>{C_LOCAL, C_DECLARATION} << 0;
+    QTest::newRow("structured binding (var 2)") << 737 << 20 << 737 << 21
+        << QList<int>{C_LOCAL, C_DECLARATION} << 0;
+    QTest::newRow("local var via indirect macro") << 746 << 20 << 746 << 30
+        << QList<int>{C_LOCAL} << 0;
+    QTest::newRow("global variable in multi-dimensional array") << 752 << 13 << 752 << 23
+        << QList<int>{C_GLOBAL} << 0;
+    QTest::newRow("reference to global variable") << 764 << 5 << 764 << 14
+        << QList<int>{C_GLOBAL} << 0;
+    QTest::newRow("nested template instantiation (namespace 1)") << 773 << 8 << 773 << 11
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("nested template instantiation (type 1)") << 773 << 13 << 773 << 19
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("nested template instantiation (opening angle bracket 1)")
+        << 773 << 19 << 773 << 20
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("nested template instantiation (namespace 2)") << 773 << 20 << 773 << 23
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("nested template instantiation (type 2)") << 773 << 25 << 773 << 29
+        << QList<int>{C_TYPE} << 0;
+    QTest::newRow("nested template instantiation (opening angle bracket 2)")
+        << 773 << 29 << 773 << 30
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("nested template instantiation (closing angle bracket 1)")
+        << 773 << 38 << 773 << 39
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("nested template instantiation (closing angle bracket 2)")
+        << 773 << 39 << 773 << 40
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("nested template instantiation (variable)") << 773 << 41 << 773 << 43
+        << QList<int>{C_GLOBAL, C_DECLARATION} << 0;
+    QTest::newRow("doubly nested template instantiation (opening angle bracket 1)")
+        << 806 << 12 << 806 << 13
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("doubly nested template instantiation (opening angle bracket 2)")
+        << 806 << 24 << 806 << 25
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("doubly nested template instantiation (opening angle bracket 3)")
+        << 806 << 36 << 806 << 37
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("doubly nested template instantiation (closing angle bracket 1)")
+        << 806 << 40 << 806 << 41
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("doubly nested template instantiation (closing angle bracket 2)")
+        << 806 << 41 << 806 << 42
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("doubly nested template instantiation (closing angle bracket 3)")
+        << 806 << 42 << 806 << 43
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("triply nested template instantiation with spacing (opening angle bracket 1)")
+        << 808 << 13 << 808 << 14
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("triply nested template instantiation with spacing (opening angle bracket 2)")
+        << 808 << 27 << 808 << 28
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("triply nested template instantiation with spacing (opening angle bracket 3)")
+        << 808 << 39 << 808 << 40
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("triply nested template instantiation with spacing (opening angle bracket 4)")
+        << 809 << 12 << 809 << 13
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketOpen);
+    QTest::newRow("triply nested template instantiation with spacing (closing angle bracket 1)")
+        << 810 << 1 << 810 << 2
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("triply nested template instantiation with spacing (closing angle bracket 2)")
+        << 810 << 2 << 810 << 3
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("triply nested template instantiation with spacing (closing angle bracket 3)")
+        << 811 << 2 << 811 << 3
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("triply nested template instantiation with spacing (closing angle bracket 4)")
+        << 812 << 3 << 812 << 4
+        << QList<int>{C_PUNCTUATION} << int(CppTools::SemanticHighlighter::AngleBracketClose);
+    QTest::newRow("cyrillic string") << 792 << 24 << 792 << 27 << QList<int>{C_STRING} << 0;
+    QTest::newRow("macro in struct") << 795 << 9 << 795 << 14
+        << QList<int>{C_PREPROCESSOR, C_DECLARATION} << 0;
+    QTest::newRow("#ifdef'ed out code") << 800 << 1 << 800 << 17
+        << QList<int>{C_DISABLED_CODE} << 0;
+}
+
+void ClangdTestHighlighting::test()
+{
+    QFETCH(int, firstLine);
+    QFETCH(int, startColumn);
+    QFETCH(int, lastLine);
+    QFETCH(int, endColumn);
+    QFETCH(QList<int>, expectedStyles);
+    QFETCH(int, expectedKind);
+
+    const TextEditor::TextDocument * const doc = document("highlighting.cpp");
+    QVERIFY(doc);
+    const int startPos = Utils::Text::positionInText(doc->document(), firstLine, startColumn);
+    const int endPos = Utils::Text::positionInText(doc->document(), lastLine, endColumn);
+
+    const auto lessThan = [=](const TextEditor::HighlightingResult &r, int) {
+        return Utils::Text::positionInText(doc->document(), r.line, r.column) < startPos;
+    };
+    const auto findResults = [=] {
+        TextEditor::HighlightingResults results;
+        auto it = std::lower_bound(m_results.cbegin(), m_results.cend(), 0, lessThan);
+        if (it == m_results.cend())
+            return results;
+        while (it != m_results.cend()) {
+            const int resultEndPos = Utils::Text::positionInText(doc->document(), it->line,
+                                                                 it->column) + it->length;
+            if (resultEndPos > endPos)
+                break;
+            results << *it++;
+        }
+        return results;
+    };
+    const TextEditor::HighlightingResults results = findResults();
+
+    QEXPECT_FAIL("typedef as underlying type in enum declaration",
+                 "FIXME: clangd does not report this symbol",
+                 Abort);
+    QEXPECT_FAIL("Q_PROPERTY (property name)", "FIXME: How to do this?", Abort);
+    QEXPECT_FAIL("Q_PROPERTY (getter)", "FIXME: How to do this?", Abort);
+    QEXPECT_FAIL("Q_PROPERTY (notifier)", "FIXME: How to do this?", Abort);
+    QEXPECT_FAIL("Q_PROPERTY (type)", "FIXME: How to do this?", Abort);
+    QEXPECT_FAIL("multi-line Q_PROPERTY (property name)", "FIXME: How to do this?", Abort);
+    QEXPECT_FAIL("multi-line Q_PROPERTY (getter)", "FIXME: How to do this?", Abort);
+    QEXPECT_FAIL("multi-line Q_PROPERTY (notifier)", "FIXME: How to do this?", Abort);
+    QEXPECT_FAIL("old-style signal (signal)", "check if and how we want to support this", Abort);
+    QEXPECT_FAIL("old-style signal (signal parameter)",
+                 "check if and how we want to support this", Abort);
+    QEXPECT_FAIL("old-style slot (slot)", "check if and how we want to support this", Abort);
+    QEXPECT_FAIL("old-style slot (slot parameter)",
+                 "check if and how we want to support this", Abort);
+    QEXPECT_FAIL("old-style signal with complex parameter (signal)",
+                 "check if and how we want to support this", Abort);
+    QEXPECT_FAIL("old-style signal with complex parameter (signal parameter part 1)",
+                 "check if and how we want to support this", Abort);
+    QEXPECT_FAIL("old-style signal with complex parameter (signal parameter part 2)",
+                 "check if and how we want to support this", Abort);
+    QEXPECT_FAIL("old-style signal with complex parameter (signal parameter part 3)",
+                 "check if and how we want to support this", Abort);
+    QEXPECT_FAIL("alias template instantiation (type)", "FIXME: clangd doesn't report this", Abort);
+    QEXPECT_FAIL("alias template instantiation (opening angle bracket)",
+                 "FIXME: This construct does not appear in the AST", Abort);
+    QEXPECT_FAIL("alias template instantiation (closing angle bracket)",
+                 "FIXME: This construct does not appear in the AST", Abort);
+    QEXPECT_FAIL("function template specialization (opening angle bracket 1)",
+                 "specialization appears as a normal function in the AST", Abort);
+    QEXPECT_FAIL("function template specialization (closing angle bracket 1)",
+                 "specialization appears as a normal function in the AST", Abort);
+    QEXPECT_FAIL("function template specialization (opening angle bracket 2)",
+                 "specialization appears as a normal function in the AST", Abort);
+    QEXPECT_FAIL("function template specialization (closing angle bracket 2)",
+                 "specialization appears as a normal function in the AST", Abort);
+
+    QCOMPARE(results.length(), 1);
+
+    const TextEditor::HighlightingResult result = results.first();
+    QCOMPARE(result.line, firstLine);
+    QCOMPARE(result.column, startColumn);
+    QCOMPARE(result.length, endPos - startPos);
+    QList<int> actualStyles;
+    if (result.useTextSyles) {
+        actualStyles << result.textStyles.mainStyle;
+        for (const TextEditor::TextStyle s : result.textStyles.mixinStyles)
+            actualStyles << s;
+    }
+    QEXPECT_FAIL("virtual member function definition outside of class body",
+                 "FIXME: send virtual info in clangd", Continue);
+    QEXPECT_FAIL("virtual function call via pointer",
+                 "FIXME: send virtual info in clangd", Continue);
+    QEXPECT_FAIL("non-final virtual function call via pointer",
+                 "FIXME: send virtual info in clangd", Continue);
+    QEXPECT_FAIL("template non-type parameter",
+                 "FIXME: clangd reports non-type template parameters at \"typeParameter\"",
+                 Continue);
+    QEXPECT_FAIL("reference to non-type template parameter",
+                 "FIXME: clangd reports non-type template parameters at \"typeParameter\"",
+                 Continue);
+    QEXPECT_FAIL("non-const reference via member function call as output argument (function)",
+                 "Without punctuation and comment tokens from clangd, it's not possible "
+                 "to highlight entire expressions. But do we really want this? What about nested "
+                 "calls where the inner arguments are const?",
+                 Continue);
+
+    QCOMPARE(actualStyles, expectedStyles);
+    QCOMPARE(result.kind, expectedKind);
+}
+
 } // namespace Tests
 } // namespace Internal
 } // namespace ClangCodeModel

src/plugins/clangcodemodel/test/clangdtests.h

@@ -27,6 +27,7 @@
 
 #include <cpptools/cpptoolstestcase.h>
 #include <coreplugin/find/searchresultitem.h>
+#include <texteditor/semantichighlighter.h>
 #include <utils/fileutils.h>
 
 #include <QHash>
@@ -126,6 +127,21 @@ private slots:
     void test();
 };
 
+class ClangdTestHighlighting : public ClangdTest
+{
+    Q_OBJECT
+public:
+    ClangdTestHighlighting();
+
+private slots:
+    void initTestCase() override;
+    void test_data();
+    void test();
+
+private:
+    TextEditor::HighlightingResults m_results;
+};
+
 } // namespace Tests
 } // namespace Internal
 } // namespace ClangCodeModel

src/plugins/clangcodemodel/test/data/clangtestdata.qrc

@@ -41,5 +41,7 @@
         <file>local-references/references.cpp</file>
         <file>tooltips/tooltips.cpp</file>
         <file>tooltips/tooltips.pro</file>
+        <file>highlighting/highlighting.cpp</file>
+        <file>highlighting/highlighting.pro</file>
     </qresource>
 </RCC>

src/plugins/clangcodemodel/test/data/highlighting/highlighting.cpp

@@ -0,0 +1,813 @@
+auto *Variable       = "Variable";
+auto *u8Variable     = u8"Variable";
+auto *rawVariable    = R"(Vari
+)a"ble)";
+auto Character       = 'c';
+
+namespace std {
+template<typename T> class vector {};
+template<typename T, typename U> class pair {};
+}
+
+
+
+
+
+
+
+
+
+
+
+
+auto integer         = 1;
+auto numFloat        = 1.2f;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+int function(int x)
+{
+    return x;
+}
+
+struct Foo
+{
+    void memberFunction() {}
+};
+
+int functionDeclaration(int x);
+
+struct Foo2
+{
+    void memberFunction();
+};
+
+void f()
+{
+    function(1);
+}
+
+struct ConversionFunction {
+    operator Foo();
+    operator int();
+};
+
+void TypeReference()
+{
+    Foo foo;
+}
+
+void LocalVariableDeclaration()
+{
+    Foo foo;
+
+    foo.memberFunction();
+}
+
+void LocalVariableFunctionArgument(Foo &foo)
+{
+    foo.memberFunction();
+}
+
+struct Foo3 {
+    int ClassMember;
+
+    void ClassMemberReference()
+    {
+        ClassMember++;
+    }
+};
+
+struct Foo4
+{
+    void MemberFunctionReference();
+
+    void function()
+    {
+        MemberFunctionReference();
+    }
+};
+
+struct Foo5
+{
+    void StaticMethod();
+
+    void function()
+    {
+        Foo5::StaticMethod();
+    }
+};
+
+enum Enumeration
+{
+    Enumerator
+};
+
+void f2()
+{
+    Enumeration enumeration;
+
+    enumeration = Enumerator;
+}
+
+class ForwardReference;
+
+class Class
+{ public:
+    Class();
+    ~Class();
+};
+
+ForwardReference *f3()
+{
+    Class ConstructorReference;
+
+    return 0;
+}
+
+union Union
+{
+
+};
+
+Union UnionDeclarationReference;
+
+
+
+
+
+
+
+
+
+namespace NameSpace {
+struct StructInNameSpace {};
+}
+
+namespace NameSpaceAlias = NameSpace;
+using NameSpace::StructInNameSpace;
+NameSpace::StructInNameSpace foo6;
+
+class BaseClass {
+public:
+    virtual void VirtualFunction();
+    virtual void FinalVirtualFunction();
+};
+
+
+void f8()
+{
+    BaseClass NonVirtualFunctionCall;
+    NonVirtualFunctionCall.VirtualFunction();
+
+    BaseClass *NonVirtualFunctionCallPointer = new BaseClass();
+    NonVirtualFunctionCallPointer->VirtualFunction();
+}
+
+class DerivedClass : public BaseClass
+{public:
+    void VirtualFunction() override;
+    void FinalVirtualFunction() final;
+};
+
+void f8(BaseClass *VirtualFunctionCallPointer)
+{
+    VirtualFunctionCallPointer->VirtualFunction();
+}
+
+class FinalClass final : public DerivedClass
+{
+    void FinalClassThisCall();
+};
+
+void f8(DerivedClass *FinalVirtualFunctionCallPointer)
+{
+    FinalVirtualFunctionCallPointer->FinalVirtualFunction();
+}
+
+void f9(BaseClass *NonFinalVirtualFunctionCallPointer)
+{
+    NonFinalVirtualFunctionCallPointer->FinalVirtualFunction();
+}
+
+void f10(FinalClass *ClassFinalVirtualFunctionCallPointer)
+{
+    ClassFinalVirtualFunctionCallPointer->VirtualFunction();
+}
+
+class Operator {
+public:
+    Operator operator+=(const Operator &first);
+};
+
+Operator operator+(const Operator &first, const Operator &second);
+
+void f10()
+{
+    auto PlusOperator = Operator() + Operator();
+    Operator PlusAssignOperator;
+    PlusAssignOperator += Operator();
+}
+
+/* Comment */
+
+#define PreprocessorDefinition Class
+#define MacroDefinition(a,b) ((a)>(b)?(a):(b))
+
+void f11()
+{
+    MacroDefinition(2, 4);
+}
+
+#include "highlightingmarks.h"
+
+void f12() {
+GOTO_LABEL:
+
+    goto GOTO_LABEL;
+}
+
+template <class T>
+void TemplateFunction(T v)
+{
+    T XXXXX = v;
+}
+void TemplateReference()
+{
+    TemplateFunction(1);
+//    std::vector<int> TemplateIntance;
+}
+
+
+
+
+template <class T>
+class TemplateFoo {};
+
+
+template <class TemplateTypeParameter = Foo, int NonTypeTemplateParameter = 1, template <class> class TemplateTemplateParameter = TemplateFoo>
+void TemplateFunction(TemplateTypeParameter TemplateParameter)
+{
+    TemplateTypeParameter TemplateTypeParameterReference;
+    auto NonTypeTemplateParameterReference = NonTypeTemplateParameter;
+    TemplateTemplateParameter<TemplateTypeParameter> TemplateTemplateParameterReference;
+}
+
+
+
+void FinalClass::FinalClassThisCall()
+{
+    VirtualFunction();
+}
+
+
+void OutputArgument(int &one, const int &two, int *three=0);
+
+void f12b()
+{
+    int One;
+    OutputArgument(One, 2);
+}
+
+#include <highlightingmarks.h>
+
+#define FOREACH(variable, container) \
+    variable; \
+    auto x = container;
+
+#define foreach2 FOREACH
+
+
+
+void f13()
+{
+    auto container = 1;
+    foreach2(int index, container);
+}
+
+class SecondArgumentInMacroExpansionIsField {
+    int container = 1;
+
+    void f()
+    {
+        foreach2(int index, container);
+    }
+};
+
+typedef unsigned uint32;
+
+enum EnumerationType : uint32
+{
+    Other = 0,
+};
+
+
+struct TypeInCast {
+    void function();
+};
+
+void f14()
+{
+    static_cast<void (TypeInCast::*)()>(&TypeInCast::function);
+    reinterpret_cast<void (TypeInCast::*)()>(&TypeInCast::function);
+}
+
+using IntegerAlias = int;
+using SecondIntegerAlias = IntegerAlias;
+typedef int IntegerTypedef;
+using Function = void (*)();
+
+
+
+void f15()
+{
+    IntegerAlias integerAlias;
+    SecondIntegerAlias secondIntegerAlias;
+    IntegerTypedef integerTypedef;
+    Function();
+}
+
+class FriendFoo
+{
+public:
+    friend class FooFriend;
+    friend bool operator==(const FriendFoo &first, const FriendFoo &second);
+};
+
+class FieldInitialization
+{
+public:
+    FieldInitialization() :
+        member(0)
+    {}
+
+    int member;
+};
+
+template<class Type>
+void TemplateFunctionCall(Type type)
+{
+    type + type;
+}
+
+void f16()
+{
+    TemplateFunctionCall(1);
+}
+
+
+template <typename T>
+class TemplatedType
+{
+    T value = T();
+};
+
+void f17()
+{
+    TemplatedType<int> TemplatedTypeDeclaration;
+}
+
+void f18()
+{
+    auto value = 1 + 2;
+}
+
+class ScopeClass
+{
+public:
+    static void ScopeOperator();
+};
+
+void f19()
+{
+    ScopeClass::ScopeOperator();
+}
+
+namespace TemplateClassNamespace {
+template<class X>
+class TemplateClass
+{
+
+};
+}
+
+void f20()
+{
+   TemplateClassNamespace::TemplateClass<ScopeClass> TemplateClassDefinition;
+}
+
+void f21()
+{
+    typedef int TypeDefDeclaration;
+    TypeDefDeclaration TypeDefDeclarationUsage;
+}
+
+typedef int EnumerationTypeDef;
+
+enum Enumeration2 : EnumerationTypeDef {
+
+};
+
+struct Bar {
+    Bar &operator[](int &key);
+};
+
+void argumentToUserDefinedIndexOperator(Bar object, int index = 3)
+{
+    object[index];
+}
+
+struct LambdaTester
+{
+    int member = 0;
+    void func() {
+        const int var = 42, var2 = 84;
+        auto lambda = [var, this](int input) {
+            return var + input + member;
+        };
+        lambda(var2);
+    }
+};
+
+void NonConstReferenceArgument(int &argument);
+
+void f22()
+{
+    int x = 1;
+
+    NonConstReferenceArgument(x);
+}
+
+void ConstReferenceArgument(const int &argument);
+
+void f23()
+{
+    int x = 1;
+
+    ConstReferenceArgument(x);
+}
+
+void RValueReferenceArgument(int &&argument);
+
+void f24()
+{
+    int x = 1;
+
+    RValueReferenceArgument(static_cast<int&&>(x));
+}
+
+void NonConstPointerArgument(int *argument);
+
+void f25()
+{
+    int *x;
+
+    NonConstPointerArgument(x);
+}
+
+void PointerToConstArgument(const int *argument);
+void ConstPointerArgument(int *const argument);
+void f26()
+{
+    int *x;
+    PointerToConstArgument(x);
+    ConstPointerArgument(x);
+}
+
+void NonConstReferenceArgumentCallInsideCall(int x, int &argument);
+int GetArgument(int x);
+
+void f27()
+{
+    int x = 1;
+
+    NonConstReferenceArgumentCallInsideCall(GetArgument(x), x);
+}
+
+void f28(int &Reference)
+{
+    NonConstReferenceArgument(Reference);
+}
+
+void f29()
+{
+    int x;
+
+    NonConstPointerArgument(&x);
+}
+
+struct NonConstPointerArgumentAsMemberOfClass
+{
+    int member;
+};
+
+void f30()
+{
+    NonConstPointerArgumentAsMemberOfClass instance;
+
+    NonConstReferenceArgument(instance.member);
+}
+
+struct NonConstReferenceArgumentConstructor
+{
+    NonConstReferenceArgumentConstructor() = default;
+    NonConstReferenceArgumentConstructor(NonConstReferenceArgumentConstructor &other);
+
+    void NonConstReferenceArgumentMember(NonConstReferenceArgumentConstructor &other);
+};
+
+void f31()
+{
+    NonConstReferenceArgumentConstructor instance;
+
+    NonConstReferenceArgumentConstructor copy(instance);
+}
+
+struct NonConstReferenceMemberInitialization
+{
+    NonConstReferenceMemberInitialization(int &foo)
+        : foo(foo)
+    {}
+
+    int &foo;
+};
+
+template<class T> class Coo;
+template<class T> class Coo<T*>;
+
+namespace N { void goo(); }
+using N::goo;
+
+#if 1
+#endif
+
+#include <new>
+
+struct OtherOperator { void operator()(int); };
+void g(OtherOperator o, int var)
+{
+    o(var);
+}
+
+void NonConstPointerArgument(int &argument);
+
+struct PointerGetterClass
+{
+    int &getter();
+};
+
+void f32()
+{
+    PointerGetterClass x;
+
+    NonConstPointerArgument(x.getter());
+}
+
+namespace N { template <typename T> void SizeIs(); }
+using N::SizeIs;
+
+void BaseClass::VirtualFunction() {}
+
+class WithVirtualFunctionDefined {
+  virtual void VirtualFunctionDefinition() {};
+};
+
+namespace NFoo { namespace NBar { namespace NTest { class NamespaceTypeSpelling; } } }
+
+Undeclared u;
+#define Q_PROPERTY(arg) static_assert("Q_PROPERTY", #arg); // Keep these in sync with wrappedQtHeaders/QtCore/qobjectdefs.h
+#define SIGNAL(arg) #arg
+#define SLOT(arg) #arg
+class Property {
+    Q_PROPERTY(const volatile unsigned long long * prop READ getProp WRITE setProp NOTIFY propChanged)
+    Q_PROPERTY(const QString str READ getStr)
+};
+
+struct X {
+    void operator*(int) {}
+};
+
+void operator*(X, float) {}
+
+void CallSite() {
+    X x;
+    int y = 10;
+    float z = 10;
+    x * y;
+    x * z;
+}
+
+struct Dummy {
+    Dummy operator<<=(int key);
+    Dummy operator()(int a);
+    int& operator[ ] (unsigned index);
+    void* operator new(unsigned size);
+    void operator delete(void* ptr);
+    void* operator new[](unsigned size);
+    void operator delete[](void* ptr);
+};
+
+void TryOverloadedOperators(Dummy object)
+{
+    object <<= 3;
+
+    Dummy stacked;
+    stacked(4);
+    stacked[1];
+    int *i = new int;
+    Dummy* use_new = new Dummy();
+    delete use_new;
+    Dummy* many = new Dummy[10];
+    delete [] many;
+}
+
+enum {
+    Test = 0
+};
+
+namespace {
+class B {
+    struct {
+        int a;
+    };
+};
+}
+
+struct Dummy2 {
+    Dummy2 operator()();
+    int operator*();
+    Dummy2 operator=(int foo);
+};
+
+void TryOverloadedOperators2(Dummy object)
+{
+    Dummy2 dummy2;
+    dummy2();
+    *dummy2;
+    dummy2 = 3;
+}
+
+int OperatorTest() {
+    return 1 < 2 ? 20 : 30;
+}
+
+int signalSlotTest() {
+    SIGNAL(something(QString));
+    SLOT(something(QString));
+    SIGNAL(something(QString (*func1)(QString)));
+    1 == 2;
+}
+
+class NonConstParameterConstructor
+{
+    NonConstParameterConstructor() = default;
+    NonConstParameterConstructor(NonConstParameterConstructor &buildDependenciesStorage);
+
+    void Call()
+    {
+        NonConstParameterConstructor foo;
+        NonConstParameterConstructor bar(foo);
+    }
+};
+
+class StaticMembersAccess
+{
+protected:
+    static int protectedValue;
+
+private:
+    static int privateValue;
+};
+
+template <int i, int j> struct S { };
+template <int i> using spec = S<i, 1>;
+spec<2> s;
+
+class Property2 {
+    Q_PROPERTY(
+
+            const
+
+            volatile
+
+            unsigned
+
+            long
+
+            long
+
+            *
+
+            prop
+
+            READ
+
+            getProp
+
+            WRITE
+
+            setProp
+
+            NOTIFY
+
+            propChanged
+
+            )
+};
+
+void structuredBindingTest() {
+    const int a[] = {1, 2};
+    const auto [x, y] = a;
+}
+
+#define ASSIGN(decl, ptr) do { decl = *ptr; } while (false)
+#define ASSIGN2 ASSIGN
+void f4()
+{
+    int *thePointer = 0;
+    ASSIGN(int i, thePointer);
+    ASSIGN2(int i, thePointer);
+}
+
+const int MyConstant = 8;
+void f5()
+{
+    int arr[MyConstant][8];
+}
+
+static int GlobalVar = 0;
+
+namespace N { [[deprecated]] void f(); }
+
+template<typename T>
+void func(T v);
+
+void f6()
+{
+    GlobalVar = 5;
+    func(1);  // QTCREATORBUG-21856
+}
+
+template<typename T>
+void func(T v) {
+    GlobalVar = 5;
+}
+
+static std::vector<std::pair<int, int>> pv;
+
+template <class T, long S>
+struct vecn
+{
+    T v[S];
+};
+
+template <class T, long S>
+static inline constexpr vecn<T, S> operator<(vecn<T, S> a, vecn<T, S> b)
+{
+    vecn<T, S> x = vecn<T, S>{};
+    for(long i = 0; i < S; ++i)
+    {
+        x[i] = a[i] < b[i];
+    }
+    return x;
+}
+
+const char *cyrillic = "б";
+
+struct foo {
+#define blubb
+};
+
+template<typename T> void funcTemplate(T v);
+#if 0
+void whatever();
+#else
+template<> void funcTemplate<int>(int v) {}
+#endif
+template<> class Coo<int> {};
+
+std::vector<std::vector<std::vector<int>>> pv2;
+
+std::vector <std::vector  <std::vector<
+std::vector<int
+>>
+ >
+  >
+vp3;

src/plugins/clangcodemodel/test/data/highlighting/highlighting.pro

@@ -0,0 +1,3 @@
+TEMPLATE = app
+CONFIG -= qt
+SOURCES = highlighting.cpp

src/plugins/languageclient/client.cpp

@@ -426,11 +426,14 @@ void Client::closeDocument(TextEditor::TextDocument *document)
     deactivateDocument(document);
     const DocumentUri &uri = DocumentUri::fromFilePath(document->filePath());
     m_highlights[uri].clear();
-    if (m_openedDocument.remove(document) != 0 && m_state == Initialized) {
+    if (m_openedDocument.remove(document) != 0) {
+        handleDocumentClosed(document);
+        if (m_state == Initialized) {
             DidCloseTextDocumentParams params(TextDocumentIdentifier{uri});
             sendContent(DidCloseTextDocumentNotification(params));
         }
     }
+}
 
 void Client::updateCompletionProvider(TextEditor::TextDocument *document)
 {
@@ -1003,6 +1006,11 @@ void Client::setDiagnosticsHandlers(const TextMarkCreator &textMarkCreator,
     m_diagnosticManager.setDiagnosticsHandlers(textMarkCreator, hideHandler);
 }
 
+void Client::setSemanticTokensHandler(const SemanticTokensHandler &handler)
+{
+    m_tokentSupport.setTokensHandler(handler);
+}
+
 void Client::setSymbolStringifier(const LanguageServerProtocol::SymbolStringifier &stringifier)
 {
     m_symbolStringifier = stringifier;

src/plugins/languageclient/client.h

@@ -174,6 +174,7 @@ public:
                        const LanguageServerProtocol::Diagnostic &diag) const;
     void setDiagnosticsHandlers(const TextMarkCreator &textMarkCreator,
                                 const HideDiagnosticsHandler &hideHandler);
+    void setSemanticTokensHandler(const SemanticTokensHandler &handler);
     void setSymbolStringifier(const LanguageServerProtocol::SymbolStringifier &stringifier);
     LanguageServerProtocol::SymbolStringifier symbolStringifier() const;
 
@@ -226,6 +227,8 @@ private:
     void handleSemanticTokens(const LanguageServerProtocol::SemanticTokens &tokens);
     void rehighlight();
 
+    virtual void handleDocumentClosed(TextEditor::TextDocument *) {}
+
     using ContentHandler = std::function<void(const QByteArray &, QTextCodec *, QString &,
                                               LanguageServerProtocol::ResponseHandlers,
                                               LanguageServerProtocol::MethodHandler)>;

src/plugins/languageclient/semantichighlightsupport.cpp

@@ -273,6 +273,8 @@ void addModifiers(int key,
 
 void SemanticTokenSupport::setLegend(const LanguageServerProtocol::SemanticTokensLegend &legend)
 {
+    m_tokenTypeStrings = legend.tokenTypes();
+    m_tokenModifierStrings = legend.tokenModifiers();
     m_tokenTypes = Utils::transform(legend.tokenTypes(), [&](const QString &tokenTypeString){
         return m_tokenTypesMap.value(tokenTypeString, -1);
     });
@@ -424,6 +426,35 @@ void SemanticTokenSupport::highlight(const Utils::FilePath &filePath)
     SyntaxHighlighter *highlighter = doc->syntaxHighlighter();
     if (!highlighter)
         return;
+    const QList<SemanticToken> tokens = m_tokens.value(filePath).toTokens(m_tokenTypes,
+                                                                          m_tokenModifiers);
+    if (m_tokensHandler) {
+        int line = 1;
+        int column = 1;
+        QList<ExpandedSemanticToken> expandedTokens;
+        for (const SemanticToken &token : tokens) {
+            line += token.deltaLine;
+            if (token.deltaLine != 0) // reset the current column when we change the current line
+                column = 1;
+            column += token.deltaStart;
+            if (token.tokenIndex >= m_tokenTypeStrings.length())
+                continue;
+            ExpandedSemanticToken expandedToken;
+            expandedToken.type = m_tokenTypeStrings.at(token.tokenIndex);
+            int modifiers = token.rawTokenModifiers;
+            for (int bitPos = 0; modifiers && bitPos < m_tokenModifierStrings.length();
+                 ++bitPos, modifiers >>= 1) {
+                if (modifiers & 0x1)
+                    expandedToken.modifiers << m_tokenModifierStrings.at(bitPos);
+            }
+            expandedToken.line = line;
+            expandedToken.column = column;
+            expandedToken.length = token.length;
+            expandedTokens << expandedToken;
+        };
+        m_tokensHandler(doc, expandedTokens);
+        return;
+    }
     int line = 1;
     int column = 1;
     auto toResult = [&](const SemanticToken &token){
@@ -434,8 +465,6 @@ void SemanticTokenSupport::highlight(const Utils::FilePath &filePath)
         const int tokenKind = token.tokenType << tokenTypeBitOffset | token.tokenModifiers;
         return HighlightingResult(line, column, token.length, tokenKind);
     };
-    const QList<SemanticToken> tokens = m_tokens.value(filePath).toTokens(m_tokenTypes,
-                                                                          m_tokenModifiers);
     const HighlightingResults results = Utils::transform(tokens, toResult);
     SemanticHighlighter::setExtraAdditionalFormats(highlighter, results, m_formatHash);
 }

src/plugins/languageclient/semantichighlightsupport.h

@@ -34,9 +34,23 @@
 
 #include <QTextCharFormat>
 
+#include <functional>
+
 namespace LanguageClient {
 class Client;
 
+class LANGUAGECLIENT_EXPORT ExpandedSemanticToken
+{
+public:
+    int line = -1;
+    int column = -1;
+    int length = -1;
+    QString type;
+    QStringList modifiers;
+};
+using SemanticTokensHandler = std::function<void(TextEditor::TextDocument *doc,
+                                                 const QList<ExpandedSemanticToken> &)>;
+
 namespace SemanticHighligtingSupport {
 
 TextEditor::HighlightingResults generateResults(
@@ -66,6 +80,8 @@ public:
     // mixin capabilities need to be extended to be able to support more
 //    void setAdditionalTokenModifierStyles(const QHash<int, TextEditor::TextStyle> &modifierStyles);
 
+    void setTokensHandler(const SemanticTokensHandler &handler) { m_tokensHandler = handler; }
+
 private:
     LanguageServerProtocol::SemanticRequestTypes supportedSemanticRequests(
         TextEditor::TextDocument *document) const;
@@ -87,6 +103,9 @@ private:
 //    QHash<int, TextEditor::TextStyle> m_additionalModifierStyles;
     QMap<QString, int> m_tokenTypesMap;
     QMap<QString, int> m_tokenModifiersMap;
+    SemanticTokensHandler m_tokensHandler;
+    QStringList m_tokenTypeStrings;
+    QStringList m_tokenModifierStrings;
 };
 
 } // namespace LanguageClient

src/plugins/texteditor/semantichighlighter.h

@@ -77,6 +77,7 @@ public:
                 && length == other.length
                 && kind == other.kind;
     }
+    bool operator!=(const HighlightingResult& other) const { return !(*this == other); }
 };
 
 using HighlightingResults = QList<HighlightingResult>;