LSP: add semantic tokens

Change-Id: Ia6865ec6991ec62ae9f0dc2dfa692f1f27318ed1
Reviewed-by: Christian Stenger <christian.stenger@qt.io>

src/plugins/languageclient/client.cpp

@@ -81,6 +81,7 @@ Client::Client(BaseClientInterface *clientInterface)
     , m_documentSymbolCache(this)
     , m_hoverHandler(this)
     , m_symbolSupport(this)
+    , m_tokentSupport(this)
 {
     m_clientProviders.completionAssistProvider = new LanguageClientCompletionAssistProvider(this);
     m_clientProviders.functionHintProvider = new FunctionHintAssistProvider(this);
@@ -100,6 +101,9 @@ Client::Client(BaseClientInterface *clientInterface)
             &TextEditor::TextEditorSettings::fontSettingsChanged,
             this,
             &Client::rehighlight);
+
+    m_tokentSupport.setTokenTypesMap(SemanticTokens::defaultTokenTypesMap());
+    m_tokentSupport.setTokenModifiersMap(SemanticTokens::defaultTokenModifiersMap());
 }
 
 QString Client::name() const
@@ -160,6 +164,7 @@ static ClientCapabilities generateClientCapabilities()
     allowDynamicRegistration.setDynamicRegistration(true);
     workspaceCapabilities.setDidChangeConfiguration(allowDynamicRegistration);
     workspaceCapabilities.setExecuteCommand(allowDynamicRegistration);
+    workspaceCapabilities.setConfiguration(true);
     capabilities.setWorkspace(workspaceCapabilities);
 
     TextDocumentClientCapabilities documentCapabilities;
@@ -248,6 +253,29 @@ static ClientCapabilities generateClientCapabilities()
     documentCapabilities.setFormatting(allowDynamicRegistration);
     documentCapabilities.setRangeFormatting(allowDynamicRegistration);
     documentCapabilities.setOnTypeFormatting(allowDynamicRegistration);
+    SemanticTokensClientCapabilities tokens;
+    tokens.setDynamicRegistration(true);
+    FullSemanticTokenOptions tokenOptions;
+    tokenOptions.setDelta(true);
+    SemanticTokensClientCapabilities::Requests tokenRequests;
+    tokenRequests.setFull(tokenOptions);
+    tokens.setRequests(tokenRequests);
+    tokens.setTokenTypes({"type",
+                          "class",
+                          "enumMember",
+                          "typeParameter",
+                          "parameter",
+                          "variable",
+                          "function",
+                          "macro",
+                          "keyword",
+                          "comment",
+                          "string",
+                          "number",
+                          "operator"});
+    tokens.setTokenModifiers({"declaration", "definition"});
+    tokens.setFormats({"relative"});
+    documentCapabilities.setSemanticTokens(tokens);
     capabilities.setTextDocument(documentCapabilities);
 
     WindowClientClientCapabilities window;
@@ -312,6 +340,7 @@ void Client::openDocument(TextEditor::TextDocument *document)
         return;
 
     m_openedDocument[document] = document->plainText();
+
     if (m_state != Initialized)
         return;
 
@@ -506,6 +535,7 @@ void Client::activateDocument(TextEditor::TextDocument *document)
     auto uri = DocumentUri::fromFilePath(document->filePath());
     m_diagnosticManager.showDiagnostics(uri);
     SemanticHighligtingSupport::applyHighlight(document, m_highlights.value(uri), capabilities());
+    m_tokentSupport.updateSemanticTokens(document);
     // only replace the assist provider if the language server support it
     updateCompletionProvider(document);
     updateFunctionHintProvider(document);
@@ -678,6 +708,13 @@ void Client::registerCapabilities(const QList<Registration> &registrations)
             for (auto document : m_openedDocument.keys())
                 updateFunctionHintProvider(document);
         }
+        if (registration.method() == "textDocument/semanticTokens") {
+            SemanticTokensOptions options(registration.registerOptions());
+            if (options.isValid())
+                m_tokentSupport.setLegend(options.legend());
+            for (auto document : m_openedDocument.keys())
+                m_tokentSupport.updateSemanticTokens(document);
+        }
     }
     emit capabilitiesChanged(m_dynamicCapabilities);
 }
@@ -694,6 +731,10 @@ void Client::unregisterCapabilities(const QList<Unregistration> &unregistrations
             for (auto document : m_openedDocument.keys())
                 updateFunctionHintProvider(document);
         }
+        if (unregistration.method() == "textDocument/semanticTokens") {
+            for (auto document : m_openedDocument.keys())
+                m_tokentSupport.updateSemanticTokens(document);
+        }
     }
     emit capabilitiesChanged(m_dynamicCapabilities);
 }
@@ -1087,6 +1128,8 @@ void Client::sendPostponedDocumentUpdates()
 
         if (currentWidget && currentWidget->textDocument() == update.document)
             cursorPositionChanged(currentWidget);
+
+        m_tokentSupport.updateSemanticTokens(update.document);
     }
 }
 
@@ -1304,6 +1347,10 @@ void Client::initializeCallback(const InitializeRequest::Response &initResponse)
                 .value_or(ServerCapabilities::SignatureHelpOptions())
                 .triggerCharacters());
     }
+    auto tokenProvider = m_serverCapabilities.semanticTokensProvider().value_or(
+        SemanticTokensOptions());
+    if (tokenProvider.isValid())
+        m_tokentSupport.setLegend(tokenProvider.legend());
 
     qCDebug(LOGLSPCLIENT) << "language server " << m_displayName << " initialized";
     m_state = Initialized;

src/plugins/languageclient/client.h

@@ -37,6 +37,7 @@
 #include "languageclientsettings.h"
 #include "languageclientsymbolsupport.h"
 #include "progressmanager.h"
+#include "semantichighlightsupport.h"
 
 #include <coreplugin/messagemanager.h>
 
@@ -49,6 +50,7 @@
 #include <languageserverprotocol/languagefeatures.h>
 #include <languageserverprotocol/messages.h>
 #include <languageserverprotocol/progresssupport.h>
+#include <languageserverprotocol/semantictokens.h>
 #include <languageserverprotocol/shutdownmessages.h>
 #include <languageserverprotocol/textsynchronization.h>
 
@@ -201,6 +203,9 @@ private:
     void updateFunctionHintProvider(TextEditor::TextDocument *document);
 
     void requestDocumentHighlights(TextEditor::TextEditorWidget *widget);
+    LanguageServerProtocol::SemanticRequestTypes supportedSemanticRequests(TextEditor::TextDocument *document) const;
+    void requestSemanticTokens(TextEditor::TextEditorWidget *widget);
+    void handleSemanticTokens(const LanguageServerProtocol::SemanticTokens &tokens);
     void rehighlight();
 
     using ContentHandler = std::function<void(const QByteArray &, QTextCodec *, QString &,
@@ -244,6 +249,7 @@ private:
     SymbolSupport m_symbolSupport;
     ProgressManager m_progressManager;
     bool m_activateDocAutomatically = false;
+    SemanticTokenSupport m_tokentSupport;
 };
 
 } // namespace LanguageClient

src/plugins/languageclient/languageclientmanager.cpp

@@ -458,9 +458,9 @@ void LanguageClientManager::documentOpened(Core::IDocument *document)
     // check whether we have to start servers for this document
     const QList<BaseSettings *> settings = currentSettings();
     for (BaseSettings *setting : settings) {
-        QVector<Client *> clients = clientForSetting(setting);
         if (setting->isValid() && setting->m_enabled
             && setting->m_languageFilter.isSupported(document)) {
+            QVector<Client *> clients = clientForSetting(setting);
             if (setting->m_startBehavior == BaseSettings::RequiresProject) {
                 const Utils::FilePath &filePath = document->filePath();
                 for (ProjectExplorer::Project *project :
@@ -475,12 +475,14 @@ void LanguageClientManager::documentOpened(Core::IDocument *document)
                                                                         return client->project()
                                                                                == project;
                                                                     });
-                    if (!clientForProject) {
+                    if (!clientForProject)
                         clientForProject = startClient(setting, project);
-                        clients << clientForProject;
-                    }
+
                     QTC_ASSERT(clientForProject, continue);
                     openDocumentWithClient(textDocument, clientForProject);
+                    // Since we already opened the document in this client we remove the client
+                    // from the list of clients that receive the openDocument call
+                    clients.removeAll(clientForProject);
                 }
             } else if (setting->m_startBehavior == BaseSettings::RequiresFile && clients.isEmpty()) {
                 clients << startClient(setting);

src/plugins/languageclient/semantichighlightsupport.cpp

@@ -25,9 +25,18 @@
 
 #include "semantichighlightsupport.h"
 
+#include "client.h"
+#include "languageclientmanager.h"
+
+#include <texteditor/fontsettings.h>
+#include <texteditor/texteditor.h>
+#include <texteditor/texteditorsettings.h>
+#include <utils/mimetypes/mimedatabase.h>
+
 #include <QTextDocument>
 
 using namespace LanguageServerProtocol;
+using namespace TextEditor;
 
 namespace LanguageClient {
 namespace SemanticHighligtingSupport {
@@ -41,7 +50,7 @@ static const QList<QList<QString>> highlightScopes(const ServerCapabilities &cap
         .scopes().value_or(QList<QList<QString>>());
 }
 
-static Utils::optional<TextEditor::TextStyle> styleForScopes(const QList<QString> &scopes)
+static Utils::optional<TextStyle> styleForScopes(const QList<QString> &scopes)
 {
     // missing "Minimal Scope Coverage" scopes
 
@@ -63,24 +72,24 @@ static Utils::optional<TextEditor::TextStyle> styleForScopes(const QList<QString
     // invalid
     // invalid.deprecated
 
-    static const QMap<QString, TextEditor::TextStyle> styleForScopes = {
-        {"entity.name", TextEditor::C_TYPE},
-        {"entity.name.function", TextEditor::C_FUNCTION},
-        {"entity.name.function.method.static", TextEditor::C_GLOBAL},
-        {"entity.name.function.preprocessor", TextEditor::C_PREPROCESSOR},
-        {"entity.name.label", TextEditor::C_LABEL},
-        {"keyword", TextEditor::C_KEYWORD},
-        {"storage.type", TextEditor::C_KEYWORD},
-        {"constant.numeric", TextEditor::C_NUMBER},
-        {"string", TextEditor::C_STRING},
-        {"comment", TextEditor::C_COMMENT},
-        {"comment.block.documentation", TextEditor::C_DOXYGEN_COMMENT},
-        {"variable.function", TextEditor::C_FUNCTION},
-        {"variable.other", TextEditor::C_LOCAL},
-        {"variable.other.member", TextEditor::C_FIELD},
-        {"variable.other.field", TextEditor::C_FIELD},
-        {"variable.other.field.static", TextEditor::C_GLOBAL},
-        {"variable.parameter", TextEditor::C_PARAMETER},
+    static const QMap<QString, TextStyle> styleForScopes = {
+        {"entity.name", C_TYPE},
+        {"entity.name.function", C_FUNCTION},
+        {"entity.name.function.method.static", C_GLOBAL},
+        {"entity.name.function.preprocessor", C_PREPROCESSOR},
+        {"entity.name.label", C_LABEL},
+        {"keyword", C_KEYWORD},
+        {"storage.type", C_KEYWORD},
+        {"constant.numeric", C_NUMBER},
+        {"string", C_STRING},
+        {"comment", C_COMMENT},
+        {"comment.block.documentation", C_DOXYGEN_COMMENT},
+        {"variable.function", C_FUNCTION},
+        {"variable.other", C_LOCAL},
+        {"variable.other.member", C_FIELD},
+        {"variable.other.field", C_FIELD},
+        {"variable.other.field.static", C_GLOBAL},
+        {"variable.parameter", C_PARAMETER},
     };
 
     for (QString scope : scopes) {
@@ -98,28 +107,27 @@ static Utils::optional<TextEditor::TextStyle> styleForScopes(const QList<QString
 }
 
 static QHash<int, QTextCharFormat> scopesToFormatHash(QList<QList<QString>> scopes,
-                                                      const TextEditor::FontSettings &fontSettings)
+                                                      const FontSettings &fontSettings)
 {
     QHash<int, QTextCharFormat> scopesToFormat;
     for (int i = 0; i < scopes.size(); ++i) {
-        if (Utils::optional<TextEditor::TextStyle> style = styleForScopes(scopes[i]))
+        if (Utils::optional<TextStyle> style = styleForScopes(scopes[i]))
             scopesToFormat[i] = fontSettings.toTextCharFormat(style.value());
     }
     return scopesToFormat;
 }
 
-TextEditor::HighlightingResult tokenToHighlightingResult(int line,
-                                                         const SemanticHighlightToken &token)
+HighlightingResult tokenToHighlightingResult(int line, const SemanticHighlightToken &token)
 {
-    return TextEditor::HighlightingResult(unsigned(line) + 1,
-                                          unsigned(token.character) + 1,
-                                          token.length,
-                                          int(token.scope));
+    return HighlightingResult(unsigned(line) + 1,
+                              unsigned(token.character) + 1,
+                              token.length,
+                              int(token.scope));
 }
 
-TextEditor::HighlightingResults generateResults(const QList<SemanticHighlightingInformation> &lines)
+HighlightingResults generateResults(const QList<SemanticHighlightingInformation> &lines)
 {
-    TextEditor::HighlightingResults results;
+    HighlightingResults results;
 
     for (const SemanticHighlightingInformation &info : lines) {
         const int line = info.line();
@@ -132,8 +140,8 @@ TextEditor::HighlightingResults generateResults(const QList<SemanticHighlighting
     return results;
 }
 
-void applyHighlight(TextEditor::TextDocument *doc,
-                    const TextEditor::HighlightingResults &results,
+void applyHighlight(TextDocument *doc,
+                    const HighlightingResults &results,
                     const ServerCapabilities &capabilities)
 {
     if (!doc->syntaxHighlighter())
@@ -145,7 +153,7 @@ void applyHighlight(TextEditor::TextDocument *doc,
             auto b = doc->document()->findBlockByNumber(int(result.line - 1));
             const QString &text = b.text().mid(int(result.column - 1), int(result.length));
             auto resultScupes = scopes[result.kind];
-            auto style = styleForScopes(resultScupes).value_or(TextEditor::C_TEXT);
+            auto style = styleForScopes(resultScupes).value_or(C_TEXT);
             qCDebug(LOGLSPHIGHLIGHT) << result.line - 1 << '\t'
                                      << result.column - 1 << '\t'
                                      << result.length << '\t'
@@ -156,7 +164,7 @@ void applyHighlight(TextEditor::TextDocument *doc,
     }
 
     if (capabilities.semanticHighlighting().has_value()) {
-        TextEditor::SemanticHighlighter::setExtraAdditionalFormats(
+        SemanticHighlighter::setExtraAdditionalFormats(
             doc->syntaxHighlighter(),
             results,
             scopesToFormatHash(highlightScopes(capabilities), doc->fontSettings()));
@@ -164,4 +172,269 @@ void applyHighlight(TextEditor::TextDocument *doc,
 }
 
 } // namespace SemanticHighligtingSupport
+
+constexpr int tokenTypeBitOffset = 16;
+
+SemanticTokenSupport::SemanticTokenSupport(Client *client)
+    : m_client(client)
+{
+    QObject::connect(TextEditorSettings::instance(),
+                     &TextEditorSettings::fontSettingsChanged,
+                     client,
+                     [this]() { updateFormatHash(); });
+}
+
+void SemanticTokenSupport::reloadSemanticTokens(TextDocument *textDocument)
+{
+    const SemanticRequestTypes supportedRequests = supportedSemanticRequests(textDocument);
+    if (supportedRequests.testFlag(SemanticRequestType::None))
+        return;
+    const Utils::FilePath filePath = textDocument->filePath();
+    const TextDocumentIdentifier docId(DocumentUri::fromFilePath(filePath));
+    auto responseCallback = [this, filePath](const SemanticTokensFullRequest::Response &response){
+        handleSemanticTokens(filePath, response.result().value_or(nullptr));
+    };
+    /*if (supportedRequests.testFlag(SemanticRequestType::Range)) {
+        const int start = widget->firstVisibleBlockNumber();
+        const int end = widget->lastVisibleBlockNumber();
+        const int pageSize = end - start;
+        // request one extra page upfront and after the current visible range
+        Range range(Position(qMax(0, start - pageSize), 0),
+                    Position(qMin(widget->blockCount() - 1, end + pageSize), 0));
+        SemanticTokensRangeParams params;
+        params.setTextDocument(docId);
+        params.setRange(range);
+        SemanticTokensRangeRequest request(params);
+        request.setResponseCallback(responseCallback);
+        m_client->sendContent(request);
+    } else */
+    if (supportedRequests.testFlag(SemanticRequestType::Full)) {
+        SemanticTokensParams params;
+        params.setTextDocument(docId);
+        SemanticTokensFullRequest request(params);
+        request.setResponseCallback(responseCallback);
+        m_client->sendContent(request);
+    }
+}
+
+void SemanticTokenSupport::updateSemanticTokens(TextDocument *textDocument)
+{
+    const SemanticRequestTypes supportedRequests = supportedSemanticRequests(textDocument);
+    if (supportedRequests.testFlag(SemanticRequestType::FullDelta)) {
+        const Utils::FilePath filePath = textDocument->filePath();
+        const QString &previousResultId = m_tokens.value(filePath).resultId().value_or(QString());
+        if (!previousResultId.isEmpty()) {
+            SemanticTokensDeltaParams params;
+            params.setTextDocument(TextDocumentIdentifier(DocumentUri::fromFilePath(filePath)));
+            params.setPreviousResultId(previousResultId);
+            SemanticTokensFullDeltaRequest request(params);
+            request.setResponseCallback(
+                [this, filePath](const SemanticTokensFullDeltaRequest::Response &response) {
+                    handleSemanticTokensDelta(filePath, response.result().value_or(nullptr));
+                });
+            m_client->sendContent(request);
+            return;
+        }
+    }
+    reloadSemanticTokens(textDocument);
+}
+
+void SemanticTokenSupport::rehighlight()
+{
+    for (const Utils::FilePath &filePath : m_tokens.keys())
+        highlight(filePath);
+}
+
+void addModifiers(int key,
+                  QHash<int, QTextCharFormat> *formatHash,
+                  TextStyles styles,
+                  QList<int> tokenModifiers,
+                  const TextEditor::FontSettings &fs)
+{
+    if (tokenModifiers.isEmpty())
+        return;
+    int modifier = tokenModifiers.takeLast();
+    auto addModifier = [&](TextStyle style){
+        if (key & modifier) // already there don't add twice
+            return;
+        key = key | modifier;
+        styles.mixinStyles.push_back(style);
+        formatHash->insert(key, fs.toTextCharFormat(styles));
+    };
+    switch (modifier) {
+    case declarationModifier: addModifier(C_DECLARATION); break;
+    case definitionModifier: addModifier(C_FUNCTION_DEFINITION); break;
+    default: break;
+    }
+    addModifiers(key, formatHash, styles, tokenModifiers, fs);
+}
+
+void SemanticTokenSupport::setLegend(const LanguageServerProtocol::SemanticTokensLegend &legend)
+{
+    m_tokenTypes = Utils::transform(legend.tokenTypes(), [&](const QString &tokenTypeString){
+        return m_tokenTypesMap.value(tokenTypeString, -1);
+    });
+    m_tokenModifiers = Utils::transform(legend.tokenModifiers(), [&](const QString &tokenModifierString){
+        return m_tokenModifiersMap.value(tokenModifierString, -1);
+    });
+    updateFormatHash();
+}
+
+void SemanticTokenSupport::updateFormatHash()
+{
+    auto fontSettings = TextEditorSettings::fontSettings();
+    for (int tokenType : qAsConst(m_tokenTypes)) {
+        if (tokenType < 0)
+            continue;
+        TextStyle style;
+        switch (tokenType) {
+        case typeToken: style = C_TYPE; break;
+        case classToken: style = C_TYPE; break;
+        case enumMemberToken: style = C_ENUMERATION; break;
+        case typeParameterToken: style = C_FIELD; break;
+        case parameterToken: style = C_PARAMETER; break;
+        case variableToken: style = C_LOCAL; break;
+        case functionToken: style = C_FUNCTION; break;
+        case macroToken: style = C_PREPROCESSOR; break;
+        case keywordToken: style = C_KEYWORD; break;
+        case commentToken: style = C_COMMENT; break;
+        case stringToken: style = C_STRING; break;
+        case numberToken: style = C_NUMBER; break;
+        case operatorToken: style = C_OPERATOR; break;
+        default:
+            style = m_additionalTypeStyles.value(tokenType, C_TEXT);
+            break;
+        }
+        int mainHashPart = tokenType << tokenTypeBitOffset;
+        m_formatHash[mainHashPart] = fontSettings.toTextCharFormat(style);
+        TextStyles styles;
+        styles.mainStyle = style;
+        styles.mixinStyles.initializeElements();
+        addModifiers(mainHashPart, &m_formatHash, styles, m_tokenModifiers, fontSettings);
+    }
+    rehighlight();
+}
+
+void SemanticTokenSupport::setTokenTypesMap(const QMap<QString, int> &tokenTypesMap)
+{
+    m_tokenTypesMap = tokenTypesMap;
+}
+
+void SemanticTokenSupport::setTokenModifiersMap(const QMap<QString, int> &tokenModifiersMap)
+{
+    m_tokenModifiersMap = tokenModifiersMap;
+}
+
+void SemanticTokenSupport::setAdditionalTokenTypeStyles(
+    const QHash<int, TextStyle> &typeStyles)
+{
+    m_additionalTypeStyles = typeStyles;
+}
+
+//void SemanticTokenSupport::setAdditionalTokenModifierStyles(
+//    const QHash<int, TextStyle> &modifierStyles)
+//{
+//    m_additionalModifierStyles = modifierStyles;
+//}
+
+SemanticRequestTypes SemanticTokenSupport::supportedSemanticRequests(TextDocument *document) const
+{
+    auto supportedRequests = [&](const QJsonObject &options) -> SemanticRequestTypes {
+        TextDocumentRegistrationOptions docOptions(options);
+        if (docOptions.isValid()
+            && docOptions.filterApplies(document->filePath(),
+                                        Utils::mimeTypeForName(document->mimeType()))) {
+            return SemanticRequestType::None;
+        }
+        const SemanticTokensOptions semanticOptions(options);
+        return semanticOptions.supportedRequests();
+    };
+    const QString dynamicMethod = "textDocument/semanticTokens";
+    const DynamicCapabilities &dynamicCapabilities = m_client->dynamicCapabilities();
+    if (auto registered = dynamicCapabilities.isRegistered(dynamicMethod);
+        registered.has_value()) {
+        if (!registered.value())
+            return SemanticRequestType::None;
+        return supportedRequests(dynamicCapabilities.option(dynamicMethod).toObject());
+    }
+    if (m_client->capabilities().semanticTokensProvider().has_value())
+        return supportedRequests(m_client->capabilities().semanticTokensProvider().value());
+    return SemanticRequestType::None;
+}
+
+void SemanticTokenSupport::handleSemanticTokens(const Utils::FilePath &filePath,
+                                                const SemanticTokensResult &result)
+{
+    if (auto tokens = Utils::get_if<SemanticTokens>(&result))
+        m_tokens[filePath] = *tokens;
+    else
+        m_tokens.remove(filePath);
+    highlight(filePath);
+}
+
+void SemanticTokenSupport::handleSemanticTokensDelta(
+    const Utils::FilePath &filePath, const LanguageServerProtocol::SemanticTokensDeltaResult &result)
+{
+    if (auto tokens = Utils::get_if<SemanticTokens>(&result)) {
+        m_tokens[filePath] = *tokens;
+    } else if (auto tokensDelta = Utils::get_if<SemanticTokensDelta>(&result)) {
+        const QList<SemanticTokensEdit> &edits = tokensDelta->edits();
+        if (edits.isEmpty())
+            return;
+
+        SemanticTokens &tokens = m_tokens[filePath];
+        QList<int> data = tokens.data();
+
+        int newDataSize = data.size();
+        for (const SemanticTokensEdit &edit : edits)
+            newDataSize += edit.dataSize() - edit.deleteCount();
+        QList<int> newData;
+        newData.reserve(newDataSize);
+
+        auto it = data.begin();
+        int currentDelta = 0;
+        for (const SemanticTokensEdit &edit : edits) {
+            for (const auto start = it + edit.start() + currentDelta; it != start; ++it)
+                newData.append(*it);
+            const QList<int> insertData = edit.data().value_or(QList<int>());
+            newData.append(edit.data().value_or(QList<int>()));
+            const int deleteCount = edit.deleteCount();
+            currentDelta += insertData.size() - deleteCount;
+            it += edit.deleteCount();
+        }
+        for (const auto end = data.end(); it != end; ++it)
+            newData.append(*it);
+
+        tokens.setData(newData);
+        tokens.setResultId(tokensDelta->resultId());
+    } else {
+        m_tokens.remove(filePath);
+    }
+    highlight(filePath);
+}
+
+void SemanticTokenSupport::highlight(const Utils::FilePath &filePath)
+{
+    TextDocument *doc = TextDocument::textDocumentForFilePath(filePath);
+    if (!doc || LanguageClientManager::clientForDocument(doc) != m_client)
+        return;
+    SyntaxHighlighter *highlighter = doc->syntaxHighlighter();
+    if (!highlighter)
+        return;
+    int line = 1;
+    int column = 1;
+    auto toResult = [&](const SemanticToken &token){
+        line += token.deltaLine;
+        if (token.deltaLine != 0) // reset the current column when we change the current line
+            column = 1;
+        column += token.deltaStart;
+        const int tokenKind = token.tokenType << tokenTypeBitOffset | token.tokenModifiers;
+        return HighlightingResult(line, column, token.length, tokenKind);
+    };
+    const QList<SemanticToken> tokens = m_tokens.value(filePath).toTokens(m_tokenTypes,
+                                                                          m_tokenModifiers);
+    const HighlightingResults results = Utils::transform(tokens, toResult);
+    SemanticHighlighter::setExtraAdditionalFormats(highlighter, results, m_formatHash);
+}
+
 } // namespace LanguageClient

src/plugins/languageclient/semantichighlightsupport.h

@@ -32,7 +32,11 @@
 #include <texteditor/semantichighlighter.h>
 #include <texteditor/textdocument.h>
 
+#include <QTextCharFormat>
+
 namespace LanguageClient {
+class Client;
+
 namespace SemanticHighligtingSupport {
 
 TextEditor::HighlightingResults generateResults(
@@ -43,4 +47,46 @@ void applyHighlight(TextEditor::TextDocument *doc,
                     const LanguageServerProtocol::ServerCapabilities &capabilities);
 
 } // namespace SemanticHighligtingSupport
+
+class SemanticTokenSupport
+{
+public:
+    explicit SemanticTokenSupport(Client *client);
+
+    void reloadSemanticTokens(TextEditor::TextDocument *doc);
+    void updateSemanticTokens(TextEditor::TextDocument *doc);
+    void rehighlight();
+    void setLegend(const LanguageServerProtocol::SemanticTokensLegend &legend);
+
+    void setTokenTypesMap(const QMap<QString, int> &tokenTypesMap);
+    void setTokenModifiersMap(const QMap<QString, int> &tokenModifiersMap);
+
+    void setAdditionalTokenTypeStyles(const QHash<int, TextEditor::TextStyle> &typeStyles);
+    // TODO: currently only declaration and definition modifiers are supported. The TextStyles
+    // mixin capabilities need to be extended to be able to support more
+//    void setAdditionalTokenModifierStyles(const QHash<int, TextEditor::TextStyle> &modifierStyles);
+
+private:
+    LanguageServerProtocol::SemanticRequestTypes supportedSemanticRequests(
+        TextEditor::TextDocument *document) const;
+    void handleSemanticTokens(const Utils::FilePath &filePath,
+                              const LanguageServerProtocol::SemanticTokensResult &result);
+    void handleSemanticTokensDelta(const Utils::FilePath &filePath,
+                                   const LanguageServerProtocol::SemanticTokensDeltaResult &result);
+    void highlight(const Utils::FilePath &filePath);
+    void updateFormatHash();
+    void currentEditorChanged();
+
+    Client *m_client = nullptr;
+
+    QHash<Utils::FilePath, LanguageServerProtocol::SemanticTokens> m_tokens;
+    QList<int> m_tokenTypes;
+    QList<int> m_tokenModifiers;
+    QHash<int, QTextCharFormat> m_formatHash;
+    QHash<int, TextEditor::TextStyle> m_additionalTypeStyles;
+//    QHash<int, TextEditor::TextStyle> m_additionalModifierStyles;
+    QMap<QString, int> m_tokenTypesMap;
+    QMap<QString, int> m_tokenModifiersMap;
+};
+
 } // namespace LanguageClient