qmlls client: extend semantic tokens

Introduce the new tokens that is as fine grained as embedded
highlighter. Also, map semantic token types to the editor text style.
We need to make the textstyle mapping customizable to achieve this.
Introduce textstyle callback function that should be called while
handling the semantic tokens. This allows different clients to map
their custom token types into a particular text style.

Task-number: QTBUG-126550
Task-number: QTCREATORBUG-31102
Change-Id: I95551b4cbfb16e311478aa776eaceb2bf0002a00
Reviewed-by: David Schulz <david.schulz@qt.io>
Reviewed-by: Sami Shalayel <sami.shalayel@qt.io>

src/plugins/languageclient/semantichighlightsupport.cpp

@@ -36,6 +36,29 @@ SemanticTokenSupport::SemanticTokenSupport(Client *client)
                      &Core::EditorManager::currentEditorChanged,
                      this,
                      &SemanticTokenSupport::onCurrentEditorChanged);
+    m_textStyleForTokenType = [](int tokenType) -> std::optional<TextStyle> {
+        switch (tokenType) {
+        case namespaceToken: return C_NAMESPACE;
+        case typeToken: return C_TYPE;
+        case classToken: return C_TYPE;
+        case structToken: return C_TYPE;
+        case enumMemberToken: return C_ENUMERATION;
+        case typeParameterToken: return C_FIELD;
+        case parameterToken: return C_PARAMETER;
+        case variableToken: return C_LOCAL;
+        case functionToken: return C_FUNCTION;
+        case methodToken: return C_FUNCTION;
+        case macroToken: return C_MACRO;
+        case keywordToken: return C_KEYWORD;
+        case commentToken: return C_COMMENT;
+        case stringToken: return C_STRING;
+        case numberToken: return C_NUMBER;
+        case operatorToken: return C_OPERATOR;
+        default:
+            break;
+        }
+        return std::nullopt;
+    };
 }
 
 void SemanticTokenSupport::refresh()
@@ -239,31 +262,13 @@ void SemanticTokenSupport::updateFormatHash()
     for (int tokenType : std::as_const(m_tokenTypes)) {
         if (tokenType < 0)
             continue;
-        TextStyle style;
+        const std::optional<TextStyle> style = m_textStyleForTokenType(tokenType);
-        switch (tokenType) {
+        if (!style)
-        case namespaceToken: style = C_NAMESPACE; break;
-        case typeToken: style = C_TYPE; break;
-        case classToken: style = C_TYPE; break;
-        case structToken: style = C_TYPE; break;
-        case enumMemberToken: style = C_ENUMERATION; break;
-        case typeParameterToken: style = C_FIELD; break;
-        case parameterToken: style = C_PARAMETER; break;
-        case variableToken: style = C_LOCAL; break;
-        case functionToken: style = C_FUNCTION; break;
-        case methodToken: style = C_FUNCTION; break;
-        case macroToken: style = C_MACRO; break;
-        case keywordToken: style = C_KEYWORD; break;
-        case commentToken: style = C_COMMENT; break;
-        case stringToken: style = C_STRING; break;
-        case numberToken: style = C_NUMBER; break;
-        case operatorToken: style = C_OPERATOR; break;
-        default:
             continue;
-        }
         int mainHashPart = tokenType << tokenTypeBitOffset;
-        m_formatHash[mainHashPart] = fontSettings.toTextCharFormat(style);
+        m_formatHash[mainHashPart] = fontSettings.toTextCharFormat(*style);
         TextStyles styles;
-        styles.mainStyle = style;
+        styles.mainStyle = *style;
         styles.mixinStyles.initializeElements();
         addModifiers(mainHashPart, &m_formatHash, styles, m_tokenModifiers, fontSettings);
     }

src/plugins/languageclient/semantichighlightsupport.h

@@ -42,6 +42,7 @@ using SemanticTokensHandler = std::function<void(TextEditor::TextDocument *,
 class LANGUAGECLIENT_EXPORT SemanticTokenSupport : public QObject
 {
 public:
+    using TokenToTextStyle = std::optional<TextEditor::TextStyle> (*)(int);
     explicit SemanticTokenSupport(Client *client);
 
     void refresh();
@@ -61,6 +62,7 @@ public:
 //    void setAdditionalTokenModifierStyles(const QHash<int, TextEditor::TextStyle> &modifierStyles);
 
     void setTokensHandler(const SemanticTokensHandler &handler) { m_tokensHandler = handler; }
+    void setTextStyleForTokenType(TokenToTextStyle callback){ m_textStyleForTokenType = callback; }
 
 private:
     void reloadSemanticTokensImpl(TextEditor::TextDocument *doc, int remainingRerequests = 3);
@@ -99,6 +101,7 @@ private:
     QStringList m_tokenModifierStrings;
     QSet<TextEditor::TextDocument *> m_docReloadQueue;
     QHash<Utils::FilePath, LanguageServerProtocol::MessageId> m_runningRequests;
+    TokenToTextStyle m_textStyleForTokenType;
 };
 
 } // namespace LanguageClient

src/plugins/qmljseditor/qmllsclient.cpp

@@ -13,12 +13,15 @@
 
 #include <texteditor/textdocument.h>
 #include <texteditor/texteditor.h>
+#include <texteditor/texteditorconstants.h>
 
 #include <qmljs/qmljsmodelmanagerinterface.h>
 
 #include <utils/mimeconstants.h>
 
 #include <QLoggingCategory>
+#include <QMetaEnum>
+#include <optional>
 
 using namespace LanguageClient;
 using namespace Utils;
@@ -66,6 +69,19 @@ QmllsClient *QmllsClient::clientForQmlls(const FilePath &qmlls)
     return client;
 }
 
+QMap<QString, int> QmllsClient::semanticTokenTypesMap()
+{
+    QMap<QString, int> result;
+    QMetaEnum metaEnum = QMetaEnum::fromType<QmllsClient::QmlSemanticTokens>();
+    for (auto i = 0; i < metaEnum.keyCount(); ++i) {
+        auto &&enumName = QString::fromUtf8(metaEnum.key(i));
+        enumName.front() = enumName.front().toLower();
+        result.insert(std::move(enumName), metaEnum.value(i));
+    }
+
+    return result;
+}
+
 QmllsClient::QmllsClient(StdIOClientInterface *interface)
     : Client(interface)
 {
@@ -80,6 +96,63 @@ QmllsClient::QmllsClient(StdIOClientInterface *interface)
         {"qtCreatorHighlighting", true}
     };
     setInitializationOptions(initializationOptions);
+    semanticTokenSupport()->setTokenTypesMap(QmllsClient::semanticTokenTypesMap());
+    semanticTokenSupport()->setTextStyleForTokenType(
+        [](int tokenType) -> std::optional<TextEditor::TextStyle> {
+            using namespace TextEditor;
+            switch (tokenType) {
+            // customized lsp token types
+            case QmlSemanticTokens::Namespace:
+                return C_NAMESPACE;
+            case QmlSemanticTokens::Type:
+                return C_QML_TYPE_ID;
+            case QmlSemanticTokens::Enum:
+                return C_ENUMERATION;
+            case QmlSemanticTokens::Parameter:
+                return C_PARAMETER;
+            case QmlSemanticTokens::Variable:
+                return C_JS_SCOPE_VAR;
+            case QmlSemanticTokens::Property:
+                return C_BINDING;
+            case QmlSemanticTokens::EnumMember:
+                return C_FIELD;
+            case QmlSemanticTokens::Method:
+                return C_FUNCTION;
+            case QmlSemanticTokens::Keyword:
+                return C_KEYWORD;
+            case QmlSemanticTokens::Comment:
+                return C_COMMENT;
+            case QmlSemanticTokens::String:
+                return C_STRING;
+            case QmlSemanticTokens::Number:
+                return C_NUMBER;
+            case QmlSemanticTokens::Regexp:
+                return C_STRING;
+            case QmlSemanticTokens::Operator:
+                return C_OPERATOR;
+            case QmlSemanticTokens::QmlLocalId:
+                return C_QML_LOCAL_ID;
+            case QmlSemanticTokens::QmlExternalId:
+                return C_QML_EXTERNAL_ID;
+            case QmlSemanticTokens::QmlRootObjectProperty:
+                return C_QML_ROOT_OBJECT_PROPERTY;
+            case QmlSemanticTokens::QmlScopeObjectProperty:
+                return C_QML_SCOPE_OBJECT_PROPERTY;
+            case QmlSemanticTokens::QmlExternalObjectProperty:
+                return C_QML_EXTERNAL_OBJECT_PROPERTY;
+            case QmlSemanticTokens::JsScopeVar:
+                return C_JS_SCOPE_VAR;
+            case QmlSemanticTokens::JsImportVar:
+                return C_JS_IMPORT_VAR;
+            case QmlSemanticTokens::JsGlobalVar:
+                return C_JS_GLOBAL_VAR;
+            case QmlSemanticTokens::QmlStateName:
+                return C_QML_STATE_NAME;
+            default:
+                break;
+            }
+            return std::nullopt;
+        });
 }
 
 QmllsClient::~QmllsClient()

src/plugins/qmljseditor/qmllsclient.h

@@ -16,11 +16,47 @@ class QMLJSEDITOR_EXPORT QmllsClient : public LanguageClient::Client
 {
     Q_OBJECT
 public:
+    // Only token types that overlap with the token types registered in the language
+    // server are highlighted.
+    // Therefore, this should be matched with the server's token types
+    enum QmlSemanticTokens {
+        // Subset of the QLspSpefication::SemanticTokenTypes enum
+        // We register only the token types used in the qml semantic highlighting
+        Namespace,
+        Type,
+        Enum,
+        Parameter,
+        Variable,
+        Property,
+        EnumMember,
+        Method,
+        Keyword,
+        Comment,
+        String,
+        Number,
+        Regexp,
+        Operator,
+        Decorator,
+
+        // Additional token types for the extended semantic highlighting
+        QmlLocalId, // object id within the same file
+        QmlExternalId, // object id defined in another file
+        QmlRootObjectProperty, // qml property defined in the parent scopes
+        QmlScopeObjectProperty, // qml property defined in the current scope
+        QmlExternalObjectProperty, // qml property defined in the root object of another file
+        JsScopeVar, // js variable defined in the current file
+        JsImportVar, // js import name that is imported in the qml file
+        JsGlobalVar, // js global variables
+        QmlStateName // name of a qml state
+    };
+    Q_ENUM(QmlSemanticTokens);
     explicit QmllsClient(LanguageClient::StdIOClientInterface *interface);
     ~QmllsClient();
 
     void startImpl() override;
     static QmllsClient *clientForQmlls(const Utils::FilePath &qmlls);
+private:
+    static QMap<QString, int> semanticTokenTypesMap();
 };
 
 } // namespace QmlJSEditor