LSP: add semantic tokens

Change-Id: Ia6865ec6991ec62ae9f0dc2dfa692f1f27318ed1
Reviewed-by: Christian Stenger <christian.stenger@qt.io>

src/libs/languageserverprotocol/CMakeLists.txt

@@ -17,6 +17,7 @@ add_qtc_library(LanguageServerProtocol
     lsputils.cpp lsputils.h
     messages.cpp messages.h
     progresssupport.cpp progresssupport.h
+    semantictokens.cpp semantictokens.h
     servercapabilities.cpp servercapabilities.h
     shutdownmessages.cpp shutdownmessages.h
     textsynchronization.cpp textsynchronization.h

src/libs/languageserverprotocol/clientcapabilities.cpp

@@ -47,4 +47,58 @@ WorkspaceClientCapabilities::WorkspaceClientCapabilities()
     setWorkspaceFolders(true);
 }
 
+Utils::optional<Utils::variant<bool, QJsonObject>> SemanticTokensClientCapabilities::Requests::range()
+    const
+{
+    using RetType = Utils::variant<bool, QJsonObject>;
+    const QJsonValue &rangeOptions = value(rangeKey);
+    if (rangeOptions.isBool())
+        return RetType(rangeOptions.toBool());
+    if (rangeOptions.isObject())
+        return RetType(rangeOptions.toObject());
+    return Utils::nullopt;
+}
+
+void SemanticTokensClientCapabilities::Requests::setRange(
+    const Utils::variant<bool, QJsonObject> &range)
+{
+    insertVariant<bool, QJsonObject>(rangeKey, range);
+}
+
+Utils::optional<Utils::variant<bool, FullSemanticTokenOptions>>
+SemanticTokensClientCapabilities::Requests::full() const
+{
+    using RetType = Utils::variant<bool, FullSemanticTokenOptions>;
+    const QJsonValue &fullOptions = value(fullKey);
+    if (fullOptions.isBool())
+        return RetType(fullOptions.toBool());
+    if (fullOptions.isObject())
+        return RetType(FullSemanticTokenOptions(fullOptions.toObject()));
+    return Utils::nullopt;
+}
+
+void SemanticTokensClientCapabilities::Requests::setFull(
+    const Utils::variant<bool, FullSemanticTokenOptions> &full)
+{
+    insertVariant<bool, FullSemanticTokenOptions>(fullKey, full);
+}
+
+Utils::optional<SemanticTokensClientCapabilities> TextDocumentClientCapabilities::semanticTokens()
+    const
+{
+    return optionalValue<SemanticTokensClientCapabilities>(semanticTokensKey);
+}
+
+void TextDocumentClientCapabilities::setSemanticTokens(
+    const SemanticTokensClientCapabilities &semanticTokens)
+{
+    insert(semanticTokensKey, semanticTokens);
+}
+
+bool SemanticTokensClientCapabilities::isValid() const
+{
+    return contains(requestsKey) && contains(tokenTypesKey) && contains(tokenModifiersKey)
+           && contains(formatsKey);
+}
+
 } // namespace LanguageServerProtocol

src/libs/languageserverprotocol/clientcapabilities.h

@@ -27,6 +27,7 @@
 
 #include "jsonkeys.h"
 #include "lsptypes.h"
+#include "semantictokens.h"
 
 namespace LanguageServerProtocol {
 
@@ -40,6 +41,84 @@ public:
     void clearDynamicRegistration() { remove(dynamicRegistrationKey); }
 };
 
+class LANGUAGESERVERPROTOCOL_EXPORT FullSemanticTokenOptions : public JsonObject
+{
+public:
+    using JsonObject::JsonObject;
+
+    /**
+      * The client will send the `textDocument/semanticTokens/full/delta`
+      * request if the server provides a corresponding handler.
+      */
+    Utils::optional<bool> delta() const { return optionalValue<bool>(deltaKey); }
+    void setDelta(bool delta) { insert(deltaKey, delta); }
+    void clearDelta() { remove(deltaKey); }
+};
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokensClientCapabilities : public DynamicRegistrationCapabilities
+{
+public:
+    using DynamicRegistrationCapabilities::DynamicRegistrationCapabilities;
+    class LANGUAGESERVERPROTOCOL_EXPORT Requests : public JsonObject
+    {
+        /**
+         * Which requests the client supports and might send to the server
+         * depending on the server's capability. Please note that clients might not
+         * show semantic tokens or degrade some of the user experience if a range
+         * or full request is advertised by the client but not provided by the
+         * server. If for example the client capability `requests.full` and
+         * `request.range` are both set to true but the server only provides a
+         * range provider the client might not render a minimap correctly or might
+         * even decide to not show any semantic tokens at all.
+         */
+    public:
+        using JsonObject::JsonObject;
+
+        /**
+         * The client will send the `textDocument/semanticTokens/range` request
+         * if the server provides a corresponding handler.
+         */
+        Utils::optional<Utils::variant<bool, QJsonObject>> range() const;
+        void setRange(const Utils::variant<bool, QJsonObject> &range);
+        void clearRange() { remove(rangeKey); }
+
+        /**
+         * The client will send the `textDocument/semanticTokens/full` request
+         * if the server provides a corresponding handler.
+         */
+        Utils::optional<Utils::variant<bool, FullSemanticTokenOptions>> full() const;
+        void setFull(const Utils::variant<bool, FullSemanticTokenOptions> &full);
+        void clearFull() { remove(fullKey); }
+    };
+
+    Requests requests() const { return typedValue<Requests>(requestsKey); }
+    void setRequests(const Requests &requests) { insert(requestsKey, requests); }
+
+    /// The token types that the client supports.
+    QList<QString> tokenTypes() const { return array<QString>(tokenTypesKey); }
+    void setTokenTypes(const QList<QString> &value) { insertArray(tokenTypesKey, value); }
+
+    /// The token modifiers that the client supports.
+    QList<QString> tokenModifiers() const { return array<QString>(tokenModifiersKey); }
+    void setTokenModifiers(const QList<QString> &value) { insertArray(tokenModifiersKey, value); }
+
+    /// The formats the clients supports.
+    QList<QString> formats() const { return array<QString>(formatsKey); }
+    void setFormats(const QList<QString> &value) { insertArray(formatsKey, value); }
+
+    /// Whether the client supports tokens that can overlap each other.
+    Utils::optional<bool> overlappingTokenSupport() const { return optionalValue<bool>(overlappingTokenSupportKey); }
+    void setOverlappingTokenSupport(bool overlappingTokenSupport) { insert(overlappingTokenSupportKey, overlappingTokenSupport); }
+    void clearOverlappingTokenSupport() { remove(overlappingTokenSupportKey); }
+
+    /// Whether the client supports tokens that can span multiple lines.
+    Utils::optional<bool> multiLineTokenSupport() const { return optionalValue<bool>(multiLineTokenSupportKey); }
+    void setMultiLineTokenSupport(bool multiLineTokenSupport) { insert(multiLineTokenSupportKey, multiLineTokenSupport); }
+    void clearMultiLineTokenSupport() { remove(multiLineTokenSupportKey); }
+
+    bool isValid() const override;
+};
+
 class LANGUAGESERVERPROTOCOL_EXPORT SymbolCapabilities : public DynamicRegistrationCapabilities
 {
 public:
@@ -432,6 +511,10 @@ public:
     void setRename(const RenameClientCapabilities &rename)
     { insert(renameKey, rename); }
     void clearRename() { remove(renameKey); }
+
+    Utils::optional<SemanticTokensClientCapabilities> semanticTokens() const;
+    void setSemanticTokens(const SemanticTokensClientCapabilities &semanticTokens);
+    void clearSemanticTokens() { remove(semanticTokensKey); }
 };
 
 class LANGUAGESERVERPROTOCOL_EXPORT WorkspaceClientCapabilities : public JsonObject

src/libs/languageserverprotocol/jsonkeys.h

@@ -79,6 +79,8 @@ constexpr char dataKey[] = "data";
 constexpr char defaultCharset[] = "utf-8";
 constexpr char definitionKey[] = "definition";
 constexpr char definitionProviderKey[] = "definitionProvider";
+constexpr char deleteCountKey[] = "deleteCount";
+constexpr char deltaKey[] = "delta";
 constexpr char deprecatedKey[] = "deprecated";
 constexpr char detailKey[] = "detail";
 constexpr char diagnosticsKey[] = "diagnostics";
@@ -108,7 +110,9 @@ constexpr char executeCommandProviderKey[] = "executeCommandProvider";
 constexpr char experimentalKey[] = "experimental";
 constexpr char filterTextKey[] = "filterText";
 constexpr char firstTriggerCharacterKey[] = "firstTriggerCharacter";
+constexpr char formatsKey[] = "formats";
 constexpr char formattingKey[] = "formatting";
+constexpr char fullKey[] = "full";
 constexpr char greenKey[] = "green";
 constexpr char headerFieldSeparator[] = ": ";
 constexpr char headerSeparator[] = "\r\n";
@@ -131,12 +135,14 @@ constexpr char kindKey[] = "kind";
 constexpr char labelKey[] = "label";
 constexpr char languageIdKey[] = "languageId";
 constexpr char languageKey[] = "language";
+constexpr char legendKey[] = "legend";
 constexpr char lineKey[] = "line";
 constexpr char linesKey[] = "lines";
 constexpr char locationKey[] = "location";
 constexpr char messageKey[] = "message";
 constexpr char methodKey[] = "method";
 constexpr char moreTriggerCharacterKey[] = "moreTriggerCharacter";
+constexpr char multiLineTokenSupportKey[] = "multiLineTokenSupport";
 constexpr char nameKey[] = "name";
 constexpr char newNameKey[] = "newName";
 constexpr char newTextKey[] = "newText";
@@ -144,6 +150,7 @@ constexpr char onTypeFormattingKey[] = "onTypeFormatting";
 constexpr char onlyKey[] = "only";
 constexpr char openCloseKey[] = "openClose";
 constexpr char optionsKey[] = "options";
+constexpr char overlappingTokenSupportKey[] = "overlappingTokenSupport";
 constexpr char parametersKey[] = "params";
 constexpr char patternKey[] = "pattern";
 constexpr char percentageKey[] = "percentage";
@@ -151,6 +158,7 @@ constexpr char placeHolderKey[] = "placeHolder";
 constexpr char positionKey[] = "position";
 constexpr char prepareProviderKey[] = "prepareProvider";
 constexpr char prepareSupportKey[] = "prepareSupport";
+constexpr char previousResultIdKey[] = "previousResultId";
 constexpr char processIdKey[] = "processId";
 constexpr char queryKey[] = "query";
 constexpr char rangeFormattingKey[] = "rangeFormatting";
@@ -165,7 +173,9 @@ constexpr char registrationsKey[] = "registrations";
 constexpr char removedKey[] = "removed";
 constexpr char renameKey[] = "rename";
 constexpr char renameProviderKey[] = "renameProvider";
+constexpr char requestsKey[] = "requests";
 constexpr char resolveProviderKey[] = "resolveProvider";
+constexpr char resultIdKey[] = "resultId";
 constexpr char resultKey[] = "result";
 constexpr char retryKey[] = "retry";
 constexpr char rootPathKey[] = "rootPath";
@@ -178,6 +188,8 @@ constexpr char sectionKey[] = "section";
 constexpr char selectionRangeKey[] = "selectionRange";
 constexpr char semanticHighlightingCapabilitiesKey[] = "semanticHighlightingCapabilities";
 constexpr char semanticHighlightingKey[] = "semanticHighlighting";
+constexpr char semanticTokensKey[] = "semanticTokens";
+constexpr char semanticTokensProviderKey[] = "semanticTokensProvider";
 constexpr char settingsKey[] = "settings";
 constexpr char severityKey[] = "severity";
 constexpr char signatureHelpKey[] = "signatureHelp";
@@ -201,6 +213,8 @@ constexpr char textEditKey[] = "textEdit";
 constexpr char textKey[] = "text";
 constexpr char titleKey[] = "title";
 constexpr char tokenKey[] = "token";
+constexpr char tokenModifiersKey[] = "tokenModifiers";
+constexpr char tokenTypesKey[] = "tokenTypes";
 constexpr char tokensKey[] = "tokens";
 constexpr char traceKey[] = "trace";
 constexpr char triggerCharacterKey[] = "triggerCharacter";

src/libs/languageserverprotocol/languageserverprotocol.pro

@@ -18,6 +18,7 @@ HEADERS += \
     lsputils.h \
     messages.h \
     progresssupport.h \
+    semantictokens.h \
     servercapabilities.h \
     shutdownmessages.h \
     textsynchronization.h \
@@ -37,6 +38,7 @@ SOURCES += \
     lsputils.cpp \
     messages.cpp \
     progresssupport.cpp \
+    semantictokens.cpp \
     servercapabilities.cpp \
     shutdownmessages.cpp \
     textsynchronization.cpp \

src/libs/languageserverprotocol/languageserverprotocol.qbs

@@ -37,6 +37,8 @@ Project {
             "messages.h",
             "progresssupport.cpp",
             "progresssupport.h",
+            "semantictokens.cpp",
+            "semantictokens.h",
             "servercapabilities.cpp",
             "servercapabilities.h",
             "shutdownmessages.cpp",

src/libs/languageserverprotocol/semantictokens.cpp

@@ -0,0 +1,154 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "semantictokens.h"
+
+namespace LanguageServerProtocol {
+
+bool SemanticTokensLegend::isValid() const
+{
+    return contains(tokenTypesKey) && contains(tokenModifiersKey);
+}
+
+QMap<QString, int> SemanticTokens::defaultTokenTypesMap()
+{
+    QMap<QString, int> map;
+    map.insert("namespace", namespaceToken);
+    map.insert("type", typeToken);
+    map.insert("class", classToken);
+    map.insert("enum", enumToken);
+    map.insert("interface", interfaceToken);
+    map.insert("struct", structToken);
+    map.insert("typeParameter", typeParameterToken);
+    map.insert("parameter", parameterToken);
+    map.insert("variable", variableToken);
+    map.insert("property", propertyToken);
+    map.insert("enumMember", enumMemberToken);
+    map.insert("event", eventToken);
+    map.insert("function", functionToken);
+    map.insert("method", methodToken);
+    map.insert("macro", macroToken);
+    map.insert("keyword", keywordToken);
+    map.insert("modifier", modifierToken);
+    map.insert("comment", commentToken);
+    map.insert("string", stringToken);
+    map.insert("number", numberToken);
+    map.insert("regexp", regexpToken);
+    map.insert("operator", operatorToken);
+    return map;
+}
+
+QMap<QString, int> SemanticTokens::defaultTokenModifiersMap()
+{
+    QMap<QString, int> map;
+    map.insert("declaration", declarationModifier);
+    map.insert("definition", definitionModifier);
+    map.insert("readonly", readonlyModifier);
+    map.insert("static", staticModifier);
+    map.insert("deprecated", deprecatedModifier);
+    map.insert("abstract", abstractModifier);
+    map.insert("async", asyncModifier);
+    map.insert("modification", modificationModifier);
+    map.insert("documentation", documentationModifier);
+    map.insert("defaultLibrary", defaultLibraryModifier);
+    return map;
+}
+
+static int convertModifiers(int modifiersData, const QList<int> &tokenModifiers)
+{
+    int result = 0;
+    for (int i = 0; i < tokenModifiers.size() && modifiersData > 0; ++i) {
+        if (modifiersData & 0x1)
+            result |= tokenModifiers[i];
+        modifiersData = modifiersData >> 1;
+    }
+    return result;
+}
+
+QList<SemanticToken> SemanticTokens::toTokens(const QList<int> &tokenTypes,
+                                              const QList<int> &tokenModifiers) const
+{
+    const QList<int> &data = this->data();
+    if (data.size() % 5 != 0)
+        return {};
+    QList<SemanticToken> tokens;
+    tokens.reserve(int(data.size() / 5));
+    auto end = data.end();
+    for (auto it = data.begin(); it != end; it += 5) {
+        SemanticToken token;
+        token.deltaLine = *(it);
+        token.deltaStart = *(it + 1);
+        token.length = *(it + 2);
+        token.tokenType = tokenTypes.value(*(it + 3), -1);
+        token.tokenModifiers = convertModifiers(*(it + 4), tokenModifiers);
+        tokens << token;
+    }
+    return tokens;
+}
+
+bool SemanticTokensRangeParams::isValid() const
+{
+    return SemanticTokensParams::isValid() && contains(rangeKey);
+}
+
+SemanticTokensFullRequest::SemanticTokensFullRequest(const SemanticTokensParams &params)
+    : Request(methodName, params)
+{}
+
+SemanticTokensRangeRequest::SemanticTokensRangeRequest(const SemanticTokensRangeParams &params)
+    : Request(methodName, params)
+{}
+
+SemanticTokensResult::SemanticTokensResult(const QJsonValue &value)
+{
+    if (value.isObject())
+        emplace<SemanticTokens>(SemanticTokens(value.toObject()));
+    else
+        emplace<nullptr_t>(nullptr);
+}
+
+SemanticTokensFullDeltaRequest::SemanticTokensFullDeltaRequest(const SemanticTokensDeltaParams &params)
+    : Request(methodName, params)
+{}
+
+bool SemanticTokensDeltaParams::isValid() const
+{
+    return SemanticTokensParams::isValid() && contains(previousResultIdKey);
+}
+
+SemanticTokensDeltaResult::SemanticTokensDeltaResult(const QJsonValue &value)
+{
+    if (value.isObject()) {
+        QJsonObject object = value.toObject();
+        if (object.contains(editsKey))
+            emplace<SemanticTokensDelta>(object);
+        else
+            emplace<SemanticTokens>(object);
+    } else {
+        emplace<nullptr_t>(nullptr);
+    }
+}
+
+} // namespace LanguageServerProtocol

src/libs/languageserverprotocol/semantictokens.h

@@ -0,0 +1,241 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "jsonkeys.h"
+#include "jsonobject.h"
+#include "jsonrpcmessages.h"
+#include "languageserverprotocol_global.h"
+#include "lsptypes.h"
+
+namespace LanguageServerProtocol {
+
+struct LANGUAGESERVERPROTOCOL_EXPORT SemanticToken
+{
+    int deltaLine = 0;
+    int deltaStart = 0;
+    int length = 0;
+    int tokenType = 0;
+    int tokenModifiers = 0;
+};
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokensLegend : public JsonObject
+{
+public:
+    using JsonObject::JsonObject;
+
+    // The token types a server uses.
+    QList<QString> tokenTypes() const { return array<QString>(tokenTypesKey); }
+    void setTokenTypes(const QList<QString> &tokenTypes) { insertArray(tokenTypesKey, tokenTypes); }
+
+    // The token modifiers a server uses.
+    QList<QString> tokenModifiers() const { return array<QString>(tokenModifiersKey); }
+    void setTokenModifiers(const QList<QString> &value) { insertArray(tokenModifiersKey, value); }
+
+    bool isValid() const override;
+};
+
+enum SemanticTokenTypes {
+    namespaceToken,
+    typeToken,
+    classToken,
+    enumToken,
+    interfaceToken,
+    structToken,
+    typeParameterToken,
+    parameterToken,
+    variableToken,
+    propertyToken,
+    enumMemberToken,
+    eventToken,
+    functionToken,
+    methodToken,
+    macroToken,
+    keywordToken,
+    modifierToken,
+    commentToken,
+    stringToken,
+    numberToken,
+    regexpToken,
+    operatorToken
+};
+
+enum SemanticTokenModifiers {
+    declarationModifier = 0x1,
+    definitionModifier = 0x2,
+    readonlyModifier = 0x4,
+    staticModifier = 0x8,
+    deprecatedModifier = 0x10,
+    abstractModifier = 0x20,
+    asyncModifier = 0x40,
+    modificationModifier = 0x80,
+    documentationModifier = 0x100,
+    defaultLibraryModifier = 0x200
+};
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokensParams : public JsonObject
+{
+public:
+    using JsonObject::JsonObject;
+
+    TextDocumentIdentifier textDocument() const
+    { return typedValue<TextDocumentIdentifier>(textDocumentKey); }
+    void setTextDocument(const TextDocumentIdentifier &textDocument)
+    { insert(textDocumentKey, textDocument); }
+
+    bool isValid() const override { return contains(textDocumentKey); }
+};
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokensDeltaParams : public SemanticTokensParams
+{
+public:
+    using SemanticTokensParams::SemanticTokensParams;
+
+    QString previousResultId() const { return typedValue<QString>(previousResultIdKey); }
+    void setPreviousResultId(const QString &previousResultId)
+    {
+        insert(previousResultIdKey, previousResultId);
+    }
+
+    bool isValid() const override;
+};
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokensRangeParams : public SemanticTokensParams
+{
+public:
+    using SemanticTokensParams::SemanticTokensParams;
+
+    Range range() const { return typedValue<Range>(rangeKey); }
+    void setRange(const Range &range) { insert(rangeKey, range); }
+
+    bool isValid() const override;
+};
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokens : public JsonObject
+{
+public:
+    using JsonObject::JsonObject;
+
+    /**
+     * An optional result id. If provided and clients support delta updating
+     * the client will include the result id in the next semantic token request.
+     * A server can then instead of computing all semantic tokens again simply
+     * send a delta.
+     */
+    Utils::optional<QString> resultId() const { return optionalValue<QString>(resultIdKey); }
+    void setResultId(const QString &resultId) { insert(resultIdKey, resultId); }
+    void clearResultId() { remove(resultIdKey); }
+
+    /// The actual tokens.
+    QList<int> data() const { return array<int>(dataKey); }
+    void setData(const QList<int> &value) { insertArray(dataKey, value); }
+
+    bool isValid() const override { return contains(dataKey); }
+
+    QList<SemanticToken> toTokens(const QList<int> &tokenTypes,
+                                  const QList<int> &tokenModifiers) const;
+    static QMap<QString, int> defaultTokenTypesMap();
+    static QMap<QString, int> defaultTokenModifiersMap();
+};
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokensResult
+    : public Utils::variant<SemanticTokens, nullptr_t>
+{
+public:
+    using variant::variant;
+    explicit SemanticTokensResult(const QJsonValue &value);
+};
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokensFullRequest
+    : public Request<SemanticTokensResult, nullptr_t, SemanticTokensParams>
+{
+public:
+    explicit SemanticTokensFullRequest(const SemanticTokensParams &params);
+    using Request::Request;
+    constexpr static const char methodName[] = "textDocument/semanticTokens/full";
+};
+
+class SemanticTokensEdit : public JsonObject
+{
+public:
+    using JsonObject::JsonObject;
+
+    int start() const { return typedValue<int>(startKey); }
+    void setStart(int start) { insert(startKey, start); }
+
+    int deleteCount() const { return typedValue<int>(deleteCountKey); }
+    void setDeleteCount(int deleteCount) { insert(deleteCountKey, deleteCount); }
+
+    Utils::optional<QList<int>> data() const { return optionalArray<int>(dataKey); }
+    void setData(const QList<int> &value) { insertArray(dataKey, value); }
+    void clearData() { remove(dataKey); }
+
+    int dataSize() const { return contains(dataKey) ? value(dataKey).toArray().size() : 0; }
+
+    bool isValid() const override { return contains(dataKey) && contains(deleteCountKey); }
+};
+
+class SemanticTokensDelta : public JsonObject
+{
+public:
+    using JsonObject::JsonObject;
+
+    QString resultId() const { return typedValue<QString>(resultIdKey); }
+    void setResultId(const QString &resultId) { insert(resultIdKey, resultId); }
+
+    QList<SemanticTokensEdit> edits() const { return array<SemanticTokensEdit>(editsKey); }
+    void setEdits(const QList<SemanticTokensEdit> &edits) { insertArray(editsKey, edits); }
+
+    bool isValid() const override { return contains(resultIdKey) && contains(editsKey); }
+};
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokensDeltaResult
+    : public Utils::variant<SemanticTokens, SemanticTokensDelta, nullptr_t>
+{
+public:
+    using variant::variant;
+    explicit SemanticTokensDeltaResult(const QJsonValue &value);
+};
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokensFullDeltaRequest
+    : public Request<SemanticTokensDeltaResult, nullptr_t, SemanticTokensDeltaParams>
+{
+public:
+    explicit SemanticTokensFullDeltaRequest(const SemanticTokensDeltaParams &params);
+    using Request::Request;
+    constexpr static const char methodName[] = "textDocument/semanticTokens/full/delta";
+};
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokensRangeRequest
+    : public Request<SemanticTokensResult, nullptr_t, SemanticTokensRangeParams>
+{
+public:
+    explicit SemanticTokensRangeRequest(const SemanticTokensRangeParams &params);
+    using Request::Request;
+    constexpr static const char methodName[] = "textDocument/semanticTokens/range";
+};
+
+} // namespace LanguageServerProtocol

src/libs/languageserverprotocol/servercapabilities.cpp

@@ -165,6 +165,17 @@ void ServerCapabilities::setDocumentSymbolProvider(
                                                  documentSymbolProvider);
 }
 
+Utils::optional<SemanticTokensOptions> ServerCapabilities::semanticTokensProvider() const
+{
+    return optionalValue<SemanticTokensOptions>(semanticTokensProviderKey);
+}
+
+void ServerCapabilities::setSemanticTokensProvider(
+    const SemanticTokensOptions &semanticTokensProvider)
+{
+    insert(semanticTokensProviderKey, semanticTokensProvider);
+}
+
 Utils::optional<Utils::variant<bool, WorkDoneProgressOptions>>
 ServerCapabilities::workspaceSymbolProvider() const
 {
@@ -349,4 +360,56 @@ bool CodeActionOptions::isValid() const
     return WorkDoneProgressOptions::isValid() && contains(codeActionKindsKey);
 }
 
+Utils::optional<Utils::variant<bool, QJsonObject>> SemanticTokensOptions::range() const
+{
+    using RetType = Utils::variant<bool, QJsonObject>;
+    const QJsonValue &rangeOptions = value(rangeKey);
+    if (rangeOptions.isBool())
+        return RetType(rangeOptions.toBool());
+    if (rangeOptions.isObject())
+        return RetType(rangeOptions.toObject());
+    return Utils::nullopt;
+}
+
+void SemanticTokensOptions::setRange(const Utils::variant<bool, QJsonObject> &range)
+{
+    insertVariant<bool, QJsonObject>(rangeKey, range);
+}
+
+Utils::optional<Utils::variant<bool, SemanticTokensOptions::FullSemanticTokenOptions>>
+SemanticTokensOptions::full() const
+{
+    using RetType = Utils::variant<bool, SemanticTokensOptions::FullSemanticTokenOptions>;
+    const QJsonValue &fullOptions = value(fullKey);
+    if (fullOptions.isBool())
+        return RetType(fullOptions.toBool());
+    if (fullOptions.isObject())
+        return RetType(FullSemanticTokenOptions(fullOptions.toObject()));
+    return Utils::nullopt;
+}
+
+void SemanticTokensOptions::setFull(
+    const Utils::variant<bool, SemanticTokensOptions::FullSemanticTokenOptions> &full)
+{
+    insertVariant<bool, FullSemanticTokenOptions>(fullKey, full);
+}
+
+SemanticRequestTypes SemanticTokensOptions::supportedRequests() const
+{
+    SemanticRequestTypes result;
+    QJsonValue rangeValue = value(rangeKey);
+    if (rangeValue.isObject() || rangeValue.toBool())
+        result |= SemanticRequestType::Range;
+    QJsonValue fullValue = value(fullKey);
+    if (fullValue.isObject()) {
+        SemanticTokensOptions::FullSemanticTokenOptions options(fullValue.toObject());
+        if (options.delta().value_or(false))
+            result |= SemanticRequestType::FullDelta;
+        result |= SemanticRequestType::Full;
+    } else if (fullValue.toBool()) {
+        result |= SemanticRequestType::Full;
+    }
+    return result;
+}
+
 } // namespace LanguageServerProtocol

src/libs/languageserverprotocol/servercapabilities.h

@@ -26,6 +26,7 @@
 #pragma once
 
 #include "lsptypes.h"
+#include "semantictokens.h"
 
 namespace LanguageServerProtocol {
 
@@ -133,6 +134,49 @@ public:
     bool isValid() const override;
 };
 
+enum class SemanticRequestType {
+    None = 0x0,
+    Full = 0x1,
+    FullDelta = 0x2,
+    Range = 0x4
+};
+Q_DECLARE_FLAGS(SemanticRequestTypes, SemanticRequestType)
+
+class LANGUAGESERVERPROTOCOL_EXPORT SemanticTokensOptions : public WorkDoneProgressOptions
+{
+public:
+    using WorkDoneProgressOptions::WorkDoneProgressOptions;
+
+    /// The legend used by the server
+    SemanticTokensLegend legend() const { return typedValue<SemanticTokensLegend>(legendKey); }
+    void setLegend(const SemanticTokensLegend &legend) { insert(legendKey, legend); }
+
+    /// Server supports providing semantic tokens for a specific range of a document.
+    Utils::optional<Utils::variant<bool, QJsonObject>> range() const;
+    void setRange(const Utils::variant<bool, QJsonObject> &range);
+    void clearRange() { remove(rangeKey); }
+
+    class FullSemanticTokenOptions : public JsonObject
+    {
+    public:
+        using JsonObject::JsonObject;
+
+        /// The server supports deltas for full documents.
+        Utils::optional<bool> delta() const { return optionalValue<bool>(deltaKey); }
+        void setDelta(bool delta) { insert(deltaKey, delta); }
+        void clearDelta() { remove(deltaKey); }
+    };
+
+    /// Server supports providing semantic tokens for a full document.
+    Utils::optional<Utils::variant<bool, FullSemanticTokenOptions>> full() const;
+    void setFull(const Utils::variant<bool, FullSemanticTokenOptions> &full);
+    void clearFull() { remove(fullKey); }
+
+    bool isValid() const override { return contains(legendKey); }
+
+    SemanticRequestTypes supportedRequests() const;
+};
+
 class LANGUAGESERVERPROTOCOL_EXPORT ServerCapabilities : public JsonObject
 {
 public:
@@ -313,6 +357,10 @@ public:
     void setDocumentSymbolProvider(Utils::variant<bool, WorkDoneProgressOptions> documentSymbolProvider);
     void clearDocumentSymbolProvider() { remove(documentSymbolProviderKey); }
 
+    Utils::optional<SemanticTokensOptions> semanticTokensProvider() const;
+    void setSemanticTokensProvider(const SemanticTokensOptions &semanticTokensProvider);
+    void clearSemanticTokensProvider() { remove(semanticTokensProviderKey); }
+
     // The server provides workspace symbol support.
     Utils::optional<Utils::variant<bool, WorkDoneProgressOptions>> workspaceSymbolProvider() const;
     void setWorkspaceSymbolProvider(Utils::variant<bool, WorkDoneProgressOptions> workspaceSymbolProvider);

src/plugins/languageclient/client.cpp

@@ -81,6 +81,7 @@ Client::Client(BaseClientInterface *clientInterface)
     , m_documentSymbolCache(this)
     , m_hoverHandler(this)
     , m_symbolSupport(this)
+    , m_tokentSupport(this)
 {
     m_clientProviders.completionAssistProvider = new LanguageClientCompletionAssistProvider(this);
     m_clientProviders.functionHintProvider = new FunctionHintAssistProvider(this);
@@ -100,6 +101,9 @@ Client::Client(BaseClientInterface *clientInterface)
             &TextEditor::TextEditorSettings::fontSettingsChanged,
             this,
             &Client::rehighlight);
+
+    m_tokentSupport.setTokenTypesMap(SemanticTokens::defaultTokenTypesMap());
+    m_tokentSupport.setTokenModifiersMap(SemanticTokens::defaultTokenModifiersMap());
 }
 
 QString Client::name() const
@@ -160,6 +164,7 @@ static ClientCapabilities generateClientCapabilities()
     allowDynamicRegistration.setDynamicRegistration(true);
     workspaceCapabilities.setDidChangeConfiguration(allowDynamicRegistration);
     workspaceCapabilities.setExecuteCommand(allowDynamicRegistration);
+    workspaceCapabilities.setConfiguration(true);
     capabilities.setWorkspace(workspaceCapabilities);
 
     TextDocumentClientCapabilities documentCapabilities;
@@ -248,6 +253,29 @@ static ClientCapabilities generateClientCapabilities()
     documentCapabilities.setFormatting(allowDynamicRegistration);
     documentCapabilities.setRangeFormatting(allowDynamicRegistration);
     documentCapabilities.setOnTypeFormatting(allowDynamicRegistration);
+    SemanticTokensClientCapabilities tokens;
+    tokens.setDynamicRegistration(true);
+    FullSemanticTokenOptions tokenOptions;
+    tokenOptions.setDelta(true);
+    SemanticTokensClientCapabilities::Requests tokenRequests;
+    tokenRequests.setFull(tokenOptions);
+    tokens.setRequests(tokenRequests);
+    tokens.setTokenTypes({"type",
+                          "class",
+                          "enumMember",
+                          "typeParameter",
+                          "parameter",
+                          "variable",
+                          "function",
+                          "macro",
+                          "keyword",
+                          "comment",
+                          "string",
+                          "number",
+                          "operator"});
+    tokens.setTokenModifiers({"declaration", "definition"});
+    tokens.setFormats({"relative"});
+    documentCapabilities.setSemanticTokens(tokens);
     capabilities.setTextDocument(documentCapabilities);
 
     WindowClientClientCapabilities window;
@@ -312,6 +340,7 @@ void Client::openDocument(TextEditor::TextDocument *document)
         return;
 
     m_openedDocument[document] = document->plainText();
+
     if (m_state != Initialized)
         return;
 
@@ -506,6 +535,7 @@ void Client::activateDocument(TextEditor::TextDocument *document)
     auto uri = DocumentUri::fromFilePath(document->filePath());
     m_diagnosticManager.showDiagnostics(uri);
     SemanticHighligtingSupport::applyHighlight(document, m_highlights.value(uri), capabilities());
+    m_tokentSupport.updateSemanticTokens(document);
     // only replace the assist provider if the language server support it
     updateCompletionProvider(document);
     updateFunctionHintProvider(document);
@@ -678,6 +708,13 @@ void Client::registerCapabilities(const QList<Registration> &registrations)
             for (auto document : m_openedDocument.keys())
                 updateFunctionHintProvider(document);
         }
+        if (registration.method() == "textDocument/semanticTokens") {
+            SemanticTokensOptions options(registration.registerOptions());
+            if (options.isValid())
+                m_tokentSupport.setLegend(options.legend());
+            for (auto document : m_openedDocument.keys())
+                m_tokentSupport.updateSemanticTokens(document);
+        }
     }
     emit capabilitiesChanged(m_dynamicCapabilities);
 }
@@ -694,6 +731,10 @@ void Client::unregisterCapabilities(const QList<Unregistration> &unregistrations
             for (auto document : m_openedDocument.keys())
                 updateFunctionHintProvider(document);
         }
+        if (unregistration.method() == "textDocument/semanticTokens") {
+            for (auto document : m_openedDocument.keys())
+                m_tokentSupport.updateSemanticTokens(document);
+        }
     }
     emit capabilitiesChanged(m_dynamicCapabilities);
 }
@@ -1087,6 +1128,8 @@ void Client::sendPostponedDocumentUpdates()
 
         if (currentWidget && currentWidget->textDocument() == update.document)
             cursorPositionChanged(currentWidget);
+
+        m_tokentSupport.updateSemanticTokens(update.document);
     }
 }
 
@@ -1304,6 +1347,10 @@ void Client::initializeCallback(const InitializeRequest::Response &initResponse)
                 .value_or(ServerCapabilities::SignatureHelpOptions())
                 .triggerCharacters());
     }
+    auto tokenProvider = m_serverCapabilities.semanticTokensProvider().value_or(
+        SemanticTokensOptions());
+    if (tokenProvider.isValid())
+        m_tokentSupport.setLegend(tokenProvider.legend());
 
     qCDebug(LOGLSPCLIENT) << "language server " << m_displayName << " initialized";
     m_state = Initialized;

src/plugins/languageclient/client.h

@@ -37,6 +37,7 @@
 #include "languageclientsettings.h"
 #include "languageclientsymbolsupport.h"
 #include "progressmanager.h"
+#include "semantichighlightsupport.h"
 
 #include <coreplugin/messagemanager.h>
 
@@ -49,6 +50,7 @@
 #include <languageserverprotocol/languagefeatures.h>
 #include <languageserverprotocol/messages.h>
 #include <languageserverprotocol/progresssupport.h>
+#include <languageserverprotocol/semantictokens.h>
 #include <languageserverprotocol/shutdownmessages.h>
 #include <languageserverprotocol/textsynchronization.h>
 
@@ -201,6 +203,9 @@ private:
     void updateFunctionHintProvider(TextEditor::TextDocument *document);
 
     void requestDocumentHighlights(TextEditor::TextEditorWidget *widget);
+    LanguageServerProtocol::SemanticRequestTypes supportedSemanticRequests(TextEditor::TextDocument *document) const;
+    void requestSemanticTokens(TextEditor::TextEditorWidget *widget);
+    void handleSemanticTokens(const LanguageServerProtocol::SemanticTokens &tokens);
     void rehighlight();
 
     using ContentHandler = std::function<void(const QByteArray &, QTextCodec *, QString &,
@@ -244,6 +249,7 @@ private:
     SymbolSupport m_symbolSupport;
     ProgressManager m_progressManager;
     bool m_activateDocAutomatically = false;
+    SemanticTokenSupport m_tokentSupport;
 };
 
 } // namespace LanguageClient

src/plugins/languageclient/languageclientmanager.cpp

@@ -458,9 +458,9 @@ void LanguageClientManager::documentOpened(Core::IDocument *document)
     // check whether we have to start servers for this document
     const QList<BaseSettings *> settings = currentSettings();
     for (BaseSettings *setting : settings) {
-        QVector<Client *> clients = clientForSetting(setting);
         if (setting->isValid() && setting->m_enabled
             && setting->m_languageFilter.isSupported(document)) {
+            QVector<Client *> clients = clientForSetting(setting);
             if (setting->m_startBehavior == BaseSettings::RequiresProject) {
                 const Utils::FilePath &filePath = document->filePath();
                 for (ProjectExplorer::Project *project :
@@ -475,12 +475,14 @@ void LanguageClientManager::documentOpened(Core::IDocument *document)
                                                                         return client->project()
                                                                                == project;
                                                                     });
-                    if (!clientForProject) {
+                    if (!clientForProject)
                         clientForProject = startClient(setting, project);
-                        clients << clientForProject;
+
-                    }
                     QTC_ASSERT(clientForProject, continue);
                     openDocumentWithClient(textDocument, clientForProject);
+                    // Since we already opened the document in this client we remove the client
+                    // from the list of clients that receive the openDocument call
+                    clients.removeAll(clientForProject);
                 }
             } else if (setting->m_startBehavior == BaseSettings::RequiresFile && clients.isEmpty()) {
                 clients << startClient(setting);

src/plugins/languageclient/semantichighlightsupport.cpp

@@ -25,9 +25,18 @@
 
 #include "semantichighlightsupport.h"
 
+#include "client.h"
+#include "languageclientmanager.h"
+
+#include <texteditor/fontsettings.h>
+#include <texteditor/texteditor.h>
+#include <texteditor/texteditorsettings.h>
+#include <utils/mimetypes/mimedatabase.h>
+
 #include <QTextDocument>
 
 using namespace LanguageServerProtocol;
+using namespace TextEditor;
 
 namespace LanguageClient {
 namespace SemanticHighligtingSupport {
@@ -41,7 +50,7 @@ static const QList<QList<QString>> highlightScopes(const ServerCapabilities &cap
         .scopes().value_or(QList<QList<QString>>());
 }
 
-static Utils::optional<TextEditor::TextStyle> styleForScopes(const QList<QString> &scopes)
+static Utils::optional<TextStyle> styleForScopes(const QList<QString> &scopes)
 {
     // missing "Minimal Scope Coverage" scopes
 
@@ -63,24 +72,24 @@ static Utils::optional<TextEditor::TextStyle> styleForScopes(const QList<QString
     // invalid
     // invalid.deprecated
 
-    static const QMap<QString, TextEditor::TextStyle> styleForScopes = {
+    static const QMap<QString, TextStyle> styleForScopes = {
-        {"entity.name", TextEditor::C_TYPE},
+        {"entity.name", C_TYPE},
-        {"entity.name.function", TextEditor::C_FUNCTION},
+        {"entity.name.function", C_FUNCTION},
-        {"entity.name.function.method.static", TextEditor::C_GLOBAL},
+        {"entity.name.function.method.static", C_GLOBAL},
-        {"entity.name.function.preprocessor", TextEditor::C_PREPROCESSOR},
+        {"entity.name.function.preprocessor", C_PREPROCESSOR},
-        {"entity.name.label", TextEditor::C_LABEL},
+        {"entity.name.label", C_LABEL},
-        {"keyword", TextEditor::C_KEYWORD},
+        {"keyword", C_KEYWORD},
-        {"storage.type", TextEditor::C_KEYWORD},
+        {"storage.type", C_KEYWORD},
-        {"constant.numeric", TextEditor::C_NUMBER},
+        {"constant.numeric", C_NUMBER},
-        {"string", TextEditor::C_STRING},
+        {"string", C_STRING},
-        {"comment", TextEditor::C_COMMENT},
+        {"comment", C_COMMENT},
-        {"comment.block.documentation", TextEditor::C_DOXYGEN_COMMENT},
+        {"comment.block.documentation", C_DOXYGEN_COMMENT},
-        {"variable.function", TextEditor::C_FUNCTION},
+        {"variable.function", C_FUNCTION},
-        {"variable.other", TextEditor::C_LOCAL},
+        {"variable.other", C_LOCAL},
-        {"variable.other.member", TextEditor::C_FIELD},
+        {"variable.other.member", C_FIELD},
-        {"variable.other.field", TextEditor::C_FIELD},
+        {"variable.other.field", C_FIELD},
-        {"variable.other.field.static", TextEditor::C_GLOBAL},
+        {"variable.other.field.static", C_GLOBAL},
-        {"variable.parameter", TextEditor::C_PARAMETER},
+        {"variable.parameter", C_PARAMETER},
     };
 
     for (QString scope : scopes) {
@@ -98,28 +107,27 @@ static Utils::optional<TextEditor::TextStyle> styleForScopes(const QList<QString
 }
 
 static QHash<int, QTextCharFormat> scopesToFormatHash(QList<QList<QString>> scopes,
-                                                      const TextEditor::FontSettings &fontSettings)
+                                                      const FontSettings &fontSettings)
 {
     QHash<int, QTextCharFormat> scopesToFormat;
     for (int i = 0; i < scopes.size(); ++i) {
-        if (Utils::optional<TextEditor::TextStyle> style = styleForScopes(scopes[i]))
+        if (Utils::optional<TextStyle> style = styleForScopes(scopes[i]))
             scopesToFormat[i] = fontSettings.toTextCharFormat(style.value());
     }
     return scopesToFormat;
 }
 
-TextEditor::HighlightingResult tokenToHighlightingResult(int line,
+HighlightingResult tokenToHighlightingResult(int line, const SemanticHighlightToken &token)
-                                                         const SemanticHighlightToken &token)
 {
-    return TextEditor::HighlightingResult(unsigned(line) + 1,
+    return HighlightingResult(unsigned(line) + 1,
-                                          unsigned(token.character) + 1,
+                              unsigned(token.character) + 1,
-                                          token.length,
+                              token.length,
-                                          int(token.scope));
+                              int(token.scope));
 }
 
-TextEditor::HighlightingResults generateResults(const QList<SemanticHighlightingInformation> &lines)
+HighlightingResults generateResults(const QList<SemanticHighlightingInformation> &lines)
 {
-    TextEditor::HighlightingResults results;
+    HighlightingResults results;
 
     for (const SemanticHighlightingInformation &info : lines) {
         const int line = info.line();
@@ -132,8 +140,8 @@ TextEditor::HighlightingResults generateResults(const QList<SemanticHighlighting
     return results;
 }
 
-void applyHighlight(TextEditor::TextDocument *doc,
+void applyHighlight(TextDocument *doc,
-                    const TextEditor::HighlightingResults &results,
+                    const HighlightingResults &results,
                     const ServerCapabilities &capabilities)
 {
     if (!doc->syntaxHighlighter())
@@ -145,7 +153,7 @@ void applyHighlight(TextEditor::TextDocument *doc,
             auto b = doc->document()->findBlockByNumber(int(result.line - 1));
             const QString &text = b.text().mid(int(result.column - 1), int(result.length));
             auto resultScupes = scopes[result.kind];
-            auto style = styleForScopes(resultScupes).value_or(TextEditor::C_TEXT);
+            auto style = styleForScopes(resultScupes).value_or(C_TEXT);
             qCDebug(LOGLSPHIGHLIGHT) << result.line - 1 << '\t'
                                      << result.column - 1 << '\t'
                                      << result.length << '\t'
@@ -156,7 +164,7 @@ void applyHighlight(TextEditor::TextDocument *doc,
     }
 
     if (capabilities.semanticHighlighting().has_value()) {
-        TextEditor::SemanticHighlighter::setExtraAdditionalFormats(
+        SemanticHighlighter::setExtraAdditionalFormats(
             doc->syntaxHighlighter(),
             results,
             scopesToFormatHash(highlightScopes(capabilities), doc->fontSettings()));
@@ -164,4 +172,269 @@ void applyHighlight(TextEditor::TextDocument *doc,
 }
 
 } // namespace SemanticHighligtingSupport
+
+constexpr int tokenTypeBitOffset = 16;
+
+SemanticTokenSupport::SemanticTokenSupport(Client *client)
+    : m_client(client)
+{
+    QObject::connect(TextEditorSettings::instance(),
+                     &TextEditorSettings::fontSettingsChanged,
+                     client,
+                     [this]() { updateFormatHash(); });
+}
+
+void SemanticTokenSupport::reloadSemanticTokens(TextDocument *textDocument)
+{
+    const SemanticRequestTypes supportedRequests = supportedSemanticRequests(textDocument);
+    if (supportedRequests.testFlag(SemanticRequestType::None))
+        return;
+    const Utils::FilePath filePath = textDocument->filePath();
+    const TextDocumentIdentifier docId(DocumentUri::fromFilePath(filePath));
+    auto responseCallback = [this, filePath](const SemanticTokensFullRequest::Response &response){
+        handleSemanticTokens(filePath, response.result().value_or(nullptr));
+    };
+    /*if (supportedRequests.testFlag(SemanticRequestType::Range)) {
+        const int start = widget->firstVisibleBlockNumber();
+        const int end = widget->lastVisibleBlockNumber();
+        const int pageSize = end - start;
+        // request one extra page upfront and after the current visible range
+        Range range(Position(qMax(0, start - pageSize), 0),
+                    Position(qMin(widget->blockCount() - 1, end + pageSize), 0));
+        SemanticTokensRangeParams params;
+        params.setTextDocument(docId);
+        params.setRange(range);
+        SemanticTokensRangeRequest request(params);
+        request.setResponseCallback(responseCallback);
+        m_client->sendContent(request);
+    } else */
+    if (supportedRequests.testFlag(SemanticRequestType::Full)) {
+        SemanticTokensParams params;
+        params.setTextDocument(docId);
+        SemanticTokensFullRequest request(params);
+        request.setResponseCallback(responseCallback);
+        m_client->sendContent(request);
+    }
+}
+
+void SemanticTokenSupport::updateSemanticTokens(TextDocument *textDocument)
+{
+    const SemanticRequestTypes supportedRequests = supportedSemanticRequests(textDocument);
+    if (supportedRequests.testFlag(SemanticRequestType::FullDelta)) {
+        const Utils::FilePath filePath = textDocument->filePath();
+        const QString &previousResultId = m_tokens.value(filePath).resultId().value_or(QString());
+        if (!previousResultId.isEmpty()) {
+            SemanticTokensDeltaParams params;
+            params.setTextDocument(TextDocumentIdentifier(DocumentUri::fromFilePath(filePath)));
+            params.setPreviousResultId(previousResultId);
+            SemanticTokensFullDeltaRequest request(params);
+            request.setResponseCallback(
+                [this, filePath](const SemanticTokensFullDeltaRequest::Response &response) {
+                    handleSemanticTokensDelta(filePath, response.result().value_or(nullptr));
+                });
+            m_client->sendContent(request);
+            return;
+        }
+    }
+    reloadSemanticTokens(textDocument);
+}
+
+void SemanticTokenSupport::rehighlight()
+{
+    for (const Utils::FilePath &filePath : m_tokens.keys())
+        highlight(filePath);
+}
+
+void addModifiers(int key,
+                  QHash<int, QTextCharFormat> *formatHash,
+                  TextStyles styles,
+                  QList<int> tokenModifiers,
+                  const TextEditor::FontSettings &fs)
+{
+    if (tokenModifiers.isEmpty())
+        return;
+    int modifier = tokenModifiers.takeLast();
+    auto addModifier = [&](TextStyle style){
+        if (key & modifier) // already there don't add twice
+            return;
+        key = key | modifier;
+        styles.mixinStyles.push_back(style);
+        formatHash->insert(key, fs.toTextCharFormat(styles));
+    };
+    switch (modifier) {
+    case declarationModifier: addModifier(C_DECLARATION); break;
+    case definitionModifier: addModifier(C_FUNCTION_DEFINITION); break;
+    default: break;
+    }
+    addModifiers(key, formatHash, styles, tokenModifiers, fs);
+}
+
+void SemanticTokenSupport::setLegend(const LanguageServerProtocol::SemanticTokensLegend &legend)
+{
+    m_tokenTypes = Utils::transform(legend.tokenTypes(), [&](const QString &tokenTypeString){
+        return m_tokenTypesMap.value(tokenTypeString, -1);
+    });
+    m_tokenModifiers = Utils::transform(legend.tokenModifiers(), [&](const QString &tokenModifierString){
+        return m_tokenModifiersMap.value(tokenModifierString, -1);
+    });
+    updateFormatHash();
+}
+
+void SemanticTokenSupport::updateFormatHash()
+{
+    auto fontSettings = TextEditorSettings::fontSettings();
+    for (int tokenType : qAsConst(m_tokenTypes)) {
+        if (tokenType < 0)
+            continue;
+        TextStyle style;
+        switch (tokenType) {
+        case typeToken: style = C_TYPE; break;
+        case classToken: style = C_TYPE; break;
+        case enumMemberToken: style = C_ENUMERATION; break;
+        case typeParameterToken: style = C_FIELD; break;
+        case parameterToken: style = C_PARAMETER; break;
+        case variableToken: style = C_LOCAL; break;
+        case functionToken: style = C_FUNCTION; break;
+        case macroToken: style = C_PREPROCESSOR; break;
+        case keywordToken: style = C_KEYWORD; break;
+        case commentToken: style = C_COMMENT; break;
+        case stringToken: style = C_STRING; break;
+        case numberToken: style = C_NUMBER; break;
+        case operatorToken: style = C_OPERATOR; break;
+        default:
+            style = m_additionalTypeStyles.value(tokenType, C_TEXT);
+            break;
+        }
+        int mainHashPart = tokenType << tokenTypeBitOffset;
+        m_formatHash[mainHashPart] = fontSettings.toTextCharFormat(style);
+        TextStyles styles;
+        styles.mainStyle = style;
+        styles.mixinStyles.initializeElements();
+        addModifiers(mainHashPart, &m_formatHash, styles, m_tokenModifiers, fontSettings);
+    }
+    rehighlight();
+}
+
+void SemanticTokenSupport::setTokenTypesMap(const QMap<QString, int> &tokenTypesMap)
+{
+    m_tokenTypesMap = tokenTypesMap;
+}
+
+void SemanticTokenSupport::setTokenModifiersMap(const QMap<QString, int> &tokenModifiersMap)
+{
+    m_tokenModifiersMap = tokenModifiersMap;
+}
+
+void SemanticTokenSupport::setAdditionalTokenTypeStyles(
+    const QHash<int, TextStyle> &typeStyles)
+{
+    m_additionalTypeStyles = typeStyles;
+}
+
+//void SemanticTokenSupport::setAdditionalTokenModifierStyles(
+//    const QHash<int, TextStyle> &modifierStyles)
+//{
+//    m_additionalModifierStyles = modifierStyles;
+//}
+
+SemanticRequestTypes SemanticTokenSupport::supportedSemanticRequests(TextDocument *document) const
+{
+    auto supportedRequests = [&](const QJsonObject &options) -> SemanticRequestTypes {
+        TextDocumentRegistrationOptions docOptions(options);
+        if (docOptions.isValid()
+            && docOptions.filterApplies(document->filePath(),
+                                        Utils::mimeTypeForName(document->mimeType()))) {
+            return SemanticRequestType::None;
+        }
+        const SemanticTokensOptions semanticOptions(options);
+        return semanticOptions.supportedRequests();
+    };
+    const QString dynamicMethod = "textDocument/semanticTokens";
+    const DynamicCapabilities &dynamicCapabilities = m_client->dynamicCapabilities();
+    if (auto registered = dynamicCapabilities.isRegistered(dynamicMethod);
+        registered.has_value()) {
+        if (!registered.value())
+            return SemanticRequestType::None;
+        return supportedRequests(dynamicCapabilities.option(dynamicMethod).toObject());
+    }
+    if (m_client->capabilities().semanticTokensProvider().has_value())
+        return supportedRequests(m_client->capabilities().semanticTokensProvider().value());
+    return SemanticRequestType::None;
+}
+
+void SemanticTokenSupport::handleSemanticTokens(const Utils::FilePath &filePath,
+                                                const SemanticTokensResult &result)
+{
+    if (auto tokens = Utils::get_if<SemanticTokens>(&result))
+        m_tokens[filePath] = *tokens;
+    else
+        m_tokens.remove(filePath);
+    highlight(filePath);
+}
+
+void SemanticTokenSupport::handleSemanticTokensDelta(
+    const Utils::FilePath &filePath, const LanguageServerProtocol::SemanticTokensDeltaResult &result)
+{
+    if (auto tokens = Utils::get_if<SemanticTokens>(&result)) {
+        m_tokens[filePath] = *tokens;
+    } else if (auto tokensDelta = Utils::get_if<SemanticTokensDelta>(&result)) {
+        const QList<SemanticTokensEdit> &edits = tokensDelta->edits();
+        if (edits.isEmpty())
+            return;
+
+        SemanticTokens &tokens = m_tokens[filePath];
+        QList<int> data = tokens.data();
+
+        int newDataSize = data.size();
+        for (const SemanticTokensEdit &edit : edits)
+            newDataSize += edit.dataSize() - edit.deleteCount();
+        QList<int> newData;
+        newData.reserve(newDataSize);
+
+        auto it = data.begin();
+        int currentDelta = 0;
+        for (const SemanticTokensEdit &edit : edits) {
+            for (const auto start = it + edit.start() + currentDelta; it != start; ++it)
+                newData.append(*it);
+            const QList<int> insertData = edit.data().value_or(QList<int>());
+            newData.append(edit.data().value_or(QList<int>()));
+            const int deleteCount = edit.deleteCount();
+            currentDelta += insertData.size() - deleteCount;
+            it += edit.deleteCount();
+        }
+        for (const auto end = data.end(); it != end; ++it)
+            newData.append(*it);
+
+        tokens.setData(newData);
+        tokens.setResultId(tokensDelta->resultId());
+    } else {
+        m_tokens.remove(filePath);
+    }
+    highlight(filePath);
+}
+
+void SemanticTokenSupport::highlight(const Utils::FilePath &filePath)
+{
+    TextDocument *doc = TextDocument::textDocumentForFilePath(filePath);
+    if (!doc || LanguageClientManager::clientForDocument(doc) != m_client)
+        return;
+    SyntaxHighlighter *highlighter = doc->syntaxHighlighter();
+    if (!highlighter)
+        return;
+    int line = 1;
+    int column = 1;
+    auto toResult = [&](const SemanticToken &token){
+        line += token.deltaLine;
+        if (token.deltaLine != 0) // reset the current column when we change the current line
+            column = 1;
+        column += token.deltaStart;
+        const int tokenKind = token.tokenType << tokenTypeBitOffset | token.tokenModifiers;
+        return HighlightingResult(line, column, token.length, tokenKind);
+    };
+    const QList<SemanticToken> tokens = m_tokens.value(filePath).toTokens(m_tokenTypes,
+                                                                          m_tokenModifiers);
+    const HighlightingResults results = Utils::transform(tokens, toResult);
+    SemanticHighlighter::setExtraAdditionalFormats(highlighter, results, m_formatHash);
+}
+
 } // namespace LanguageClient

src/plugins/languageclient/semantichighlightsupport.h

@@ -32,7 +32,11 @@
 #include <texteditor/semantichighlighter.h>
 #include <texteditor/textdocument.h>
 
+#include <QTextCharFormat>
+
 namespace LanguageClient {
+class Client;
+
 namespace SemanticHighligtingSupport {
 
 TextEditor::HighlightingResults generateResults(
@@ -43,4 +47,46 @@ void applyHighlight(TextEditor::TextDocument *doc,
                     const LanguageServerProtocol::ServerCapabilities &capabilities);
 
 } // namespace SemanticHighligtingSupport
+
+class SemanticTokenSupport
+{
+public:
+    explicit SemanticTokenSupport(Client *client);
+
+    void reloadSemanticTokens(TextEditor::TextDocument *doc);
+    void updateSemanticTokens(TextEditor::TextDocument *doc);
+    void rehighlight();
+    void setLegend(const LanguageServerProtocol::SemanticTokensLegend &legend);
+
+    void setTokenTypesMap(const QMap<QString, int> &tokenTypesMap);
+    void setTokenModifiersMap(const QMap<QString, int> &tokenModifiersMap);
+
+    void setAdditionalTokenTypeStyles(const QHash<int, TextEditor::TextStyle> &typeStyles);
+    // TODO: currently only declaration and definition modifiers are supported. The TextStyles
+    // mixin capabilities need to be extended to be able to support more
+//    void setAdditionalTokenModifierStyles(const QHash<int, TextEditor::TextStyle> &modifierStyles);
+
+private:
+    LanguageServerProtocol::SemanticRequestTypes supportedSemanticRequests(
+        TextEditor::TextDocument *document) const;
+    void handleSemanticTokens(const Utils::FilePath &filePath,
+                              const LanguageServerProtocol::SemanticTokensResult &result);
+    void handleSemanticTokensDelta(const Utils::FilePath &filePath,
+                                   const LanguageServerProtocol::SemanticTokensDeltaResult &result);
+    void highlight(const Utils::FilePath &filePath);
+    void updateFormatHash();
+    void currentEditorChanged();
+
+    Client *m_client = nullptr;
+
+    QHash<Utils::FilePath, LanguageServerProtocol::SemanticTokens> m_tokens;
+    QList<int> m_tokenTypes;
+    QList<int> m_tokenModifiers;
+    QHash<int, QTextCharFormat> m_formatHash;
+    QHash<int, TextEditor::TextStyle> m_additionalTypeStyles;
+//    QHash<int, TextEditor::TextStyle> m_additionalModifierStyles;
+    QMap<QString, int> m_tokenTypesMap;
+    QMap<QString, int> m_tokenModifiersMap;
+};
+
 } // namespace LanguageClient