Clang: extra clangbackend job to collect full token info

Limit document annotations job to only highlighting data
collection and move more expensive calls into separate job
that runs after it.

Change-Id: Ie792a3f741ac45c81033dd5b3a20ed061604f927
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@qt.io>

src/libs/clangsupport/documentannotationschangedmessage.h

@@ -39,15 +39,23 @@ class CLANGSUPPORT_EXPORT DocumentAnnotationsChangedMessage
 {
 public:
     DocumentAnnotationsChangedMessage() = default;
+    // For pure token infos update
+    DocumentAnnotationsChangedMessage(const FileContainer &fileContainer,
+                                      const QVector<TokenInfoContainer> &tokenInfos)
+        : m_fileContainer(fileContainer),
+          m_tokenInfos(tokenInfos),
+          m_onlyTokenInfos(true)
+    {
+    }
     DocumentAnnotationsChangedMessage(const FileContainer &fileContainer,
                                       const QVector<DiagnosticContainer> &diagnostics,
                                       const DiagnosticContainer &firstHeaderErrorDiagnostic,
                                       const QVector<TokenInfoContainer> &tokenInfos,
                                       const QVector<SourceRangeContainer> &skippedPreprocessorRanges)
         : m_fileContainer(fileContainer),
+          m_tokenInfos(tokenInfos),
           m_diagnostics(diagnostics),
           m_firstHeaderErrorDiagnostic(firstHeaderErrorDiagnostic),
-          m_tokenInfos(tokenInfos),
           m_skippedPreprocessorRanges(skippedPreprocessorRanges)
     {
     }
@@ -72,6 +80,11 @@ public:
         return m_tokenInfos;
     }
 
+    bool onlyTokenInfos() const
+    {
+        return m_onlyTokenInfos;
+    }
+
     const QVector<SourceRangeContainer> &skippedPreprocessorRanges() const
     {
         return m_skippedPreprocessorRanges;
@@ -79,10 +92,13 @@ public:
 
     friend QDataStream &operator<<(QDataStream &out, const DocumentAnnotationsChangedMessage &message)
     {
+        out << message.m_onlyTokenInfos;
         out << message.m_fileContainer;
+        out << message.m_tokenInfos;
+        if (message.m_onlyTokenInfos)
+            return out;
         out << message.m_diagnostics;
         out << message.m_firstHeaderErrorDiagnostic;
-        out << message.m_tokenInfos;
         out << message.m_skippedPreprocessorRanges;
 
         return out;
@@ -90,10 +106,13 @@ public:
 
     friend QDataStream &operator>>(QDataStream &in, DocumentAnnotationsChangedMessage &message)
     {
+        in >> message.m_onlyTokenInfos;
         in >> message.m_fileContainer;
+        in >> message.m_tokenInfos;
+        if (message.m_onlyTokenInfos)
+            return in;
         in >> message.m_diagnostics;
         in >> message.m_firstHeaderErrorDiagnostic;
-        in >> message.m_tokenInfos;
         in >> message.m_skippedPreprocessorRanges;
 
         return in;
@@ -111,10 +130,11 @@ public:
 
 private:
     FileContainer m_fileContainer;
+    QVector<TokenInfoContainer> m_tokenInfos;
     QVector<DiagnosticContainer> m_diagnostics;
     DiagnosticContainer m_firstHeaderErrorDiagnostic;
-    QVector<TokenInfoContainer> m_tokenInfos;
     QVector<SourceRangeContainer> m_skippedPreprocessorRanges;
+    bool m_onlyTokenInfos = false;
 };
 
 CLANGSUPPORT_EXPORT QDebug operator<<(QDebug debug, const DocumentAnnotationsChangedMessage &message);

src/libs/clangsupport/tokeninfocontainer.cpp

@@ -64,6 +64,26 @@ static const char *highlightingTypeToCStringLiteral(HighlightingType type)
 }
 #undef RETURN_TEXT_FOR_CASE
 
+QDebug operator<<(QDebug debug, const ExtraInfo &extraInfo)
+{
+    debug.nospace() << "ExtraInfo("
+                    << extraInfo.token << ", "
+                    << extraInfo.typeSpelling << ", "
+                    << extraInfo.resultTypeSpelling << ", "
+                    << extraInfo.semanticParentTypeSpelling << ", "
+                    << static_cast<uint>(extraInfo.accessSpecifier) << ", "
+                    << static_cast<uint>(extraInfo.storageClass) << ", "
+                    << extraInfo.identifier << ", "
+                    << extraInfo.includeDirectivePath << ", "
+                    << extraInfo.declaration << ", "
+                    << extraInfo.definition << ", "
+                    << extraInfo.signal << ", "
+                    << extraInfo.slot << ", "
+                    << extraInfo.property
+                    << ")";
+    return debug;
+}
+
 QDebug operator<<(QDebug debug, const TokenInfoContainer &container)
 {
     debug.nospace() << "TokenInfosContainer("
@@ -71,16 +91,7 @@ QDebug operator<<(QDebug debug, const TokenInfoContainer &container)
                     << container.column() << ", "
                     << container.length() << ", "
                     << highlightingTypeToCStringLiteral(container.types().mainHighlightingType) << ", "
-                    << container.token() << ", "
+                    << container.extraInfo()
-                    << container.typeSpelling() << ", "
-                    << container.returnTypeSpelling() << ", "
-                    << container.semanticParentTypeSpelling() << ", "
-                    << static_cast<uint>(container.accessSpecifier()) << ", "
-                    << static_cast<uint>(container.storageClass()) << ", "
-                    << container.isIdentifier() << ", "
-                    << container.isIncludeDirectivePath() << ", "
-                    << container.isDeclaration() << ", "
-                    << container.isDefinition()
                     << ")";
 
     return debug;

src/libs/clangsupport/tokeninfocontainer.h

@@ -32,6 +32,7 @@
 #include <QDataStream>
 
 #include <bitset>
+
 namespace ClangBackEnd {
 
 inline QDataStream &operator<<(QDataStream &out, HighlightingType highlightingType);
@@ -46,51 +47,77 @@ using ByteSizeBitset = std::bitset<8>;
 inline QDataStream &operator<<(QDataStream &out, ByteSizeBitset bits);
 inline QDataStream &operator>>(QDataStream &in, ByteSizeBitset &bits);
 
+struct ExtraInfo
+{
+    ExtraInfo()
+        : identifier(false)
+        , includeDirectivePath(false)
+        , declaration(false)
+        , definition(false)
+        , signal(false)
+        , slot(false)
+        , property(false)
+    {
+    }
+    ExtraInfo(Utf8String token, Utf8String typeSpelling, Utf8String resultTypeSpelling,
+              Utf8String semanticParentTypeSpelling, AccessSpecifier accessSpecifier,
+              StorageClass storageClass, bool isIdentifier, bool isInclusion,
+              bool isDeclaration, bool isDefinition, bool isSignal, bool isSlot, bool isProperty)
+        : token(token)
+        , typeSpelling(typeSpelling)
+        , resultTypeSpelling(resultTypeSpelling)
+        , semanticParentTypeSpelling(semanticParentTypeSpelling)
+        , accessSpecifier(accessSpecifier)
+        , storageClass(storageClass)
+        , identifier(isIdentifier)
+        , includeDirectivePath(isInclusion)
+        , declaration(isDeclaration)
+        , definition(isDefinition)
+        , signal(isSignal)
+        , slot(isSlot)
+        , property(isProperty)
+    {
+    }
+    Utf8String token;
+    Utf8String typeSpelling;
+    Utf8String resultTypeSpelling;
+    Utf8String semanticParentTypeSpelling;
+    AccessSpecifier accessSpecifier = AccessSpecifier::Invalid;
+    StorageClass storageClass = StorageClass::Invalid;
+    bool identifier : 1;
+    bool includeDirectivePath : 1;
+    bool declaration : 1;
+    bool definition : 1;
+    bool signal : 1;
+    bool slot : 1;
+    bool property : 1;
+};
+
+inline QDataStream &operator<<(QDataStream &out, const ExtraInfo &extraInfo);
+inline QDataStream &operator>>(QDataStream &in, ExtraInfo &extraInfo);
+inline bool operator==(const ExtraInfo &first, const ExtraInfo &second);
+
 class TokenInfoContainer
 {
-    enum BitField
-    {
-        Identifier = 0,
-        IncludeDirectivePath = 1,
-        Declaration = 2,
-        Definition = 3,
-        Unused1 = 4,
-        Unused2 = 5,
-        Unused3 = 6,
-        Unused4 = 7,
-    };
 public:
     TokenInfoContainer() = default;
-    TokenInfoContainer(uint line, uint column, uint length, HighlightingTypes types,
+    TokenInfoContainer(uint line, uint column, uint length, HighlightingTypes types)
-                       const Utf8String &token,const Utf8String &typeSpelling,
+        : line_(line)
-                       const Utf8String &returnTypeSpelling,
+        , column_(column)
-                       const Utf8String &semanticParentTypeSpelling,
+        , length_(length)
-                       AccessSpecifier accessSpecifier, StorageClass storageClass,
+        , types_(types)
-                       bool isIdentifier = false, bool isIncludeDirectivePath = false,
-                       bool isDeclaration = false, bool isDefinition = false)
-        : line_(line),
-          column_(column),
-          length_(length),
-          types_(types),
-          token_(token),
-          typeSpelling_(typeSpelling),
-          returnTypeSpelling_(returnTypeSpelling),
-          semanticParentTypeSpelling_(semanticParentTypeSpelling),
-          accessSpecifier_(accessSpecifier),
-          storageClass_(storageClass)
     {
-        bitFields_.set(BitField::Identifier, isIdentifier);
-        bitFields_.set(BitField::IncludeDirectivePath, isIncludeDirectivePath);
-        bitFields_.set(BitField::Declaration, isDeclaration);
-        bitFields_.set(BitField::Definition, isDefinition);
     }
 
-    TokenInfoContainer(uint line, uint column, uint length, HighlightingType type)
+    TokenInfoContainer(uint line, uint column, uint length, HighlightingTypes types,
-        : line_(line),
+                       const ExtraInfo &extraInfo)
-          column_(column),
+        : line_(line)
-          length_(length)
+        , column_(column)
+        , length_(length)
+        , types_(types)
+        , extraInfo_(extraInfo)
+        , noExtraInfo_(false)
     {
-        types_.mainHighlightingType = type;
     }
 
     uint line() const
@@ -113,74 +140,28 @@ public:
         return types_;
     }
 
+    const ExtraInfo &extraInfo() const
+    {
+        return extraInfo_;
+    }
+
     bool isInvalid() const
     {
         return line_ == 0 && column_ == 0 && length_ == 0;
     }
 
-    bool isIdentifier() const
-    {
-        return bitFields_[BitField::Identifier];
-    }
-
-    bool isIncludeDirectivePath() const
-    {
-        return bitFields_[BitField::IncludeDirectivePath];
-    }
-
-    bool isDeclaration() const
-    {
-        return bitFields_[BitField::Declaration];
-    }
-
-    bool isDefinition() const
-    {
-        return bitFields_[BitField::Definition];
-    }
-
-    const Utf8String &token() const
-    {
-        return token_;
-    }
-
-    const Utf8String &typeSpelling() const
-    {
-        return typeSpelling_;
-    }
-
-    const Utf8String returnTypeSpelling() const
-    {
-        return returnTypeSpelling_;
-    }
-
-    const Utf8String semanticParentTypeSpelling() const
-    {
-        return semanticParentTypeSpelling_;
-    }
-
-    AccessSpecifier accessSpecifier() const
-    {
-        return accessSpecifier_;
-    }
-
-    StorageClass storageClass() const
-    {
-        return storageClass_;
-    }
-
     friend QDataStream &operator<<(QDataStream &out, const TokenInfoContainer &container)
     {
         out << container.line_;
         out << container.column_;
         out << container.length_;
         out << container.types_;
-        out << container.token_;
+        out << container.noExtraInfo_;
-        out << container.typeSpelling_;
+
-        out << container.returnTypeSpelling_;
+        if (container.noExtraInfo_)
-        out << container.semanticParentTypeSpelling_;
+            return out;
-        out << static_cast<uint>(container.accessSpecifier_);
+
-        out << static_cast<uint>(container.storageClass_);
+        out << container.extraInfo_;
-        out << container.bitFields_;
 
         return out;
     }
@@ -191,19 +172,12 @@ public:
         in >> container.column_;
         in >> container.length_;
         in >> container.types_;
-        in >> container.token_ ;
+        in >> container.noExtraInfo_;
-        in >> container.typeSpelling_;
-        in >> container.returnTypeSpelling_;
-        in >> container.semanticParentTypeSpelling_;
 
-        uint accessSpecifier;
+        if (container.noExtraInfo_)
-        uint storageClass;
+            return in;
-        in >> accessSpecifier;
-        in >> storageClass;
-        container.accessSpecifier_ = static_cast<AccessSpecifier>(accessSpecifier);
-        container.storageClass_ = static_cast<StorageClass>(storageClass);
 
-        in >> container.bitFields_;
+        in >> container.extraInfo_;
 
         return in;
     }
@@ -214,13 +188,8 @@ public:
             && first.column_ == second.column_
             && first.length_ == second.length_
             && first.types_ == second.types_
-            && first.token_ == second.token_
+            && first.noExtraInfo_ == second.noExtraInfo_
-            && first.typeSpelling_ == second.typeSpelling_
+            && first.extraInfo_ == second.extraInfo_;
-            && first.returnTypeSpelling_ == second.returnTypeSpelling_
-            && first.semanticParentTypeSpelling_ == second.semanticParentTypeSpelling_
-            && first.accessSpecifier_ == second.accessSpecifier_
-            && first.storageClass_ == second.storageClass_
-            && first.bitFields_ == second.bitFields_;
     }
 
 private:
@@ -228,15 +197,84 @@ private:
     uint column_ = 0;
     uint length_ = 0;
     HighlightingTypes types_;
-    Utf8String token_;
+    ExtraInfo extraInfo_;
-    Utf8String typeSpelling_;
+    bool noExtraInfo_ = true;
-    Utf8String returnTypeSpelling_;
-    Utf8String semanticParentTypeSpelling_;
-    AccessSpecifier accessSpecifier_;
-    StorageClass storageClass_;
-    ByteSizeBitset bitFields_;
 };
 
+inline QDataStream &operator<<(QDataStream &out, const ExtraInfo &extraInfo)
+{
+    out << extraInfo.token;
+    out << extraInfo.typeSpelling;
+    out << extraInfo.resultTypeSpelling;
+    out << extraInfo.semanticParentTypeSpelling;
+    out << static_cast<uint>(extraInfo.accessSpecifier);
+    out << static_cast<uint>(extraInfo.storageClass);
+    out << extraInfo.identifier;
+    out << extraInfo.includeDirectivePath;
+    out << extraInfo.declaration;
+    out << extraInfo.definition;
+    out << extraInfo.signal;
+    out << extraInfo.slot;
+    out << extraInfo.property;
+    return out;
+}
+
+inline QDataStream &operator>>(QDataStream &in, ExtraInfo &extraInfo)
+{
+    in >> extraInfo.token;
+    in >> extraInfo.typeSpelling;
+    in >> extraInfo.resultTypeSpelling;
+    in >> extraInfo.semanticParentTypeSpelling;
+
+    uint accessSpecifier;
+    uint storageClass;
+    bool isIdentifier;
+    bool isInclusion;
+    bool isDeclaration;
+    bool isDefinition;
+    bool isSignal;
+    bool isSlot;
+    bool isProperty;
+
+    in >> accessSpecifier;
+    in >> storageClass;
+    in >> isIdentifier;
+    in >> isInclusion;
+    in >> isDeclaration;
+    in >> isDefinition;
+    in >> isSignal;
+    in >> isSlot;
+    in >> isProperty;
+
+    extraInfo.accessSpecifier = static_cast<AccessSpecifier>(accessSpecifier);
+    extraInfo.storageClass = static_cast<StorageClass>(storageClass);
+    extraInfo.identifier = isIdentifier;
+    extraInfo.includeDirectivePath = isInclusion;
+    extraInfo.declaration = isDeclaration;
+    extraInfo.definition = isDefinition;
+    extraInfo.signal = isSignal;
+    extraInfo.slot = isSlot;
+    extraInfo.property = isProperty;
+    return in;
+}
+
+inline bool operator==(const ExtraInfo &first, const ExtraInfo &second)
+{
+    return first.token == second.token
+            && first.typeSpelling == second.typeSpelling
+            && first.resultTypeSpelling == second.resultTypeSpelling
+            && first.semanticParentTypeSpelling == second.semanticParentTypeSpelling
+            && first.accessSpecifier == second.accessSpecifier
+            && first.storageClass == second.storageClass
+            && first.identifier == second.identifier
+            && first.includeDirectivePath == second.includeDirectivePath
+            && first.declaration == second.declaration
+            && first.definition == second.definition
+            && first.signal == second.signal
+            && first.slot == second.slot
+            && first.property == second.property;
+}
+
 inline QDataStream &operator<<(QDataStream &out, HighlightingType highlightingType)
 {
     out << static_cast<const quint8>(highlightingType);

src/plugins/clangcodemodel/clangbackendreceiver.cpp

@@ -202,20 +202,26 @@ void BackendReceiver::documentAnnotationsChanged(const DocumentAnnotationsChange
 
     auto processor = ClangEditorDocumentProcessor::get(message.fileContainer().filePath());
 
-    if (processor) {
+    if (!processor)
+        return;
+
     const QString projectPartId = message.fileContainer().projectPartId();
     const QString filePath = message.fileContainer().filePath();
     const QString documentProjectPartId = CppTools::CppToolsBridge::projectPartIdForFile(filePath);
-        if (projectPartId == documentProjectPartId) {
+    if (projectPartId != documentProjectPartId)
+        return;
+
     const quint32 documentRevision = message.fileContainer().documentRevision();
+    if (message.onlyTokenInfos()) {
+        processor->updateTokenInfos(message.tokenInfos(), documentRevision);
+        return;
+    }
     processor->updateCodeWarnings(message.diagnostics(),
                                   message.firstHeaderErrorDiagnostic(),
                                   documentRevision);
     processor->updateHighlighting(message.tokenInfos(),
                                   message.skippedPreprocessorRanges(),
                                   documentRevision);
-        }
-    }
 }
 
 static

src/plugins/clangcodemodel/clangeditordocumentprocessor.cpp

@@ -260,6 +260,15 @@ void ClangEditorDocumentProcessor::updateHighlighting(
     }
 }
 
+void ClangEditorDocumentProcessor::updateTokenInfos(
+        const QVector<ClangBackEnd::TokenInfoContainer> &tokenInfos,
+        uint documentRevision)
+{
+    if (documentRevision != revision())
+        return;
+    m_tokenInfos = tokenInfos;
+}
+
 static int currentLine(const TextEditor::AssistInterface &assistInterface)
 {
     int line, column;

src/plugins/clangcodemodel/clangeditordocumentprocessor.h

@@ -76,6 +76,8 @@ public:
     void updateHighlighting(const QVector<ClangBackEnd::TokenInfoContainer> &tokenInfos,
                             const QVector<ClangBackEnd::SourceRangeContainer> &skippedPreprocessorRanges,
                             uint documentRevision);
+    void updateTokenInfos(const QVector<ClangBackEnd::TokenInfoContainer> &tokenInfos,
+                          uint documentRevision);
 
     TextEditor::QuickFixOperations
     extraRefactoringOperations(const TextEditor::AssistInterface &assistInterface) override;

src/plugins/clangcodemodel/clangfollowsymbol.cpp

@@ -83,12 +83,12 @@ static Utils::Link linkAtCursor(QTextCursor cursor, const QString &filePath, uin
     Link token(filePath, mark.line(), mark.column());
     token.linkTextStart = getMarkPos(cursor, mark);
     token.linkTextEnd = token.linkTextStart + mark.length();
-    if (mark.isIncludeDirectivePath()) {
+    if (mark.extraInfo().includeDirectivePath) {
         if (tokenStr != "include" && tokenStr != "#" && tokenStr != "<")
             return token;
         return Link();
     }
-    if (mark.isIdentifier() || tokenStr == "operator")
+    if (mark.extraInfo().identifier || tokenStr == "operator")
         return token;
     return Link();
 }

src/tools/clangbackend/source/clangbackendclangipc-source.pri

@@ -41,6 +41,7 @@ HEADERS += \
     $$PWD/clangtype.h \
     $$PWD/clangunsavedfilesshallowarguments.h \
     $$PWD/clangupdatedocumentannotationsjob.h \
+    $$PWD/clangupdateextradocumentannotationsjob.h \
     $$PWD/codecompleter.h \
     $$PWD/codecompletionchunkconverter.h \
     $$PWD/codecompletionsextractor.h \
@@ -50,6 +51,8 @@ HEADERS += \
     $$PWD/diagnosticset.h \
     $$PWD/diagnosticsetiterator.h \
     $$PWD/fixit.h \
+    $$PWD/fulltokeninfo.h \
+    $$PWD/fulltokeninfos.h \
     $$PWD/projectpart.h \
     $$PWD/projects.h \
     $$PWD/skippedsourceranges.h \
@@ -98,6 +101,7 @@ SOURCES += \
     $$PWD/clangtype.cpp \
     $$PWD/clangunsavedfilesshallowarguments.cpp \
     $$PWD/clangupdatedocumentannotationsjob.cpp \
+    $$PWD/clangupdateextradocumentannotationsjob.cpp \
     $$PWD/codecompleter.cpp \
     $$PWD/codecompletionchunkconverter.cpp \
     $$PWD/codecompletionsextractor.cpp \
@@ -106,6 +110,8 @@ SOURCES += \
     $$PWD/diagnostic.cpp \
     $$PWD/diagnosticset.cpp \
     $$PWD/fixit.cpp \
+    $$PWD/fulltokeninfo.cpp \
+    $$PWD/fulltokeninfos.cpp \
     $$PWD/projectpart.cpp \
     $$PWD/projects.cpp \
     $$PWD/skippedsourceranges.cpp \

src/tools/clangbackend/source/clangcodemodelserver.cpp

@@ -398,6 +398,8 @@ void ClangCodeModelServer::addAndRunUpdateJobs(std::vector<Document> documents)
         // Run the regular edit-reparse-job
         processor.addJob(JobRequest::Type::UpdateDocumentAnnotations,
                          PreferredTranslationUnit::PreviouslyParsed);
+        processor.addJob(JobRequest::Type::UpdateExtraDocumentAnnotations,
+                         PreferredTranslationUnit::PreviouslyParsed);
         processor.process();
 
         // If requested, run jobs to increase the responsiveness of the document
@@ -474,6 +476,7 @@ void ClangCodeModelServer::processInitialJobsForDocuments(const std::vector<Docu
     for (const auto &document : documents) {
         DocumentProcessor processor = documentProcessors().processor(document);
         processor.addJob(JobRequest::Type::UpdateDocumentAnnotations);
+        processor.addJob(JobRequest::Type::UpdateExtraDocumentAnnotations);
         processor.addJob(JobRequest::Type::CreateInitialDocumentPreamble);
         processor.process();
     }

src/tools/clangbackend/source/clangjobrequest.cpp

@@ -36,6 +36,7 @@
 #include "clangresumedocumentjob.h"
 #include "clangsuspenddocumentjob.h"
 #include "clangupdatedocumentannotationsjob.h"
+#include "clangupdateextradocumentannotationsjob.h"
 
 #include <clangsupport/clangcodemodelclientinterface.h>
 #include <clangsupport/cmbcodecompletedmessage.h>
@@ -57,6 +58,7 @@ static const char *JobRequestTypeToText(JobRequest::Type type)
     switch (type) {
         RETURN_TEXT_FOR_CASE(Invalid);
         RETURN_TEXT_FOR_CASE(UpdateDocumentAnnotations);
+        RETURN_TEXT_FOR_CASE(UpdateExtraDocumentAnnotations);
         RETURN_TEXT_FOR_CASE(ParseSupportiveTranslationUnit);
         RETURN_TEXT_FOR_CASE(ReparseSupportiveTranslationUnit);
         RETURN_TEXT_FOR_CASE(CreateInitialDocumentPreamble);
@@ -126,6 +128,7 @@ static JobRequest::ExpirationConditions expirationConditionsForType(JobRequest::
 
     switch (type) {
     case Type::UpdateDocumentAnnotations:
+    case Type::UpdateExtraDocumentAnnotations:
         return Conditions(Condition::AnythingChanged);
     case Type::RequestReferences:
     case Type::RequestDocumentAnnotations:
@@ -192,6 +195,7 @@ bool JobRequest::isTakeOverable() const
     // Discard these as they are initial jobs that will be recreated on demand
     // anyway.
     case Type::UpdateDocumentAnnotations:
+    case Type::UpdateExtraDocumentAnnotations:
     case Type::CreateInitialDocumentPreamble:
 
     // Discard these as they only make sense in a row. Avoid splitting them up.
@@ -223,6 +227,8 @@ IAsyncJob *JobRequest::createJob() const
         break;
     case JobRequest::Type::UpdateDocumentAnnotations:
         return new UpdateDocumentAnnotationsJob();
+    case JobRequest::Type::UpdateExtraDocumentAnnotations:
+        return new UpdateExtraDocumentAnnotationsJob();
     case JobRequest::Type::ParseSupportiveTranslationUnit:
         return new ParseSupportiveTranslationUnitJob();
     case JobRequest::Type::ReparseSupportiveTranslationUnit:
@@ -256,6 +262,7 @@ void JobRequest::cancelJob(ClangCodeModelClientInterface &client) const
     switch (type) {
     case JobRequest::Type::Invalid:
     case JobRequest::Type::UpdateDocumentAnnotations:
+    case JobRequest::Type::UpdateExtraDocumentAnnotations:
     case JobRequest::Type::ParseSupportiveTranslationUnit:
     case JobRequest::Type::ReparseSupportiveTranslationUnit:
     case JobRequest::Type::CreateInitialDocumentPreamble:

src/tools/clangbackend/source/clangjobrequest.h

@@ -50,6 +50,7 @@ public:
         Invalid,
 
         UpdateDocumentAnnotations,
+        UpdateExtraDocumentAnnotations,
         CreateInitialDocumentPreamble,
 
         ParseSupportiveTranslationUnit,

src/tools/clangbackend/source/clangtranslationunit.cpp

@@ -36,8 +36,8 @@
 #include <cursor.h>
 #include <diagnosticcontainer.h>
 #include <diagnosticset.h>
-#include <tokeninfo.h>
 #include <tokeninfos.h>
+#include <fulltokeninfos.h>
 #include <skippedsourceranges.h>
 #include <sourcelocation.h>
 #include <sourcerange.h>
@@ -140,7 +140,6 @@ void TranslationUnit::extractDocumentAnnotations(
     skippedSourceRanges = this->skippedSourceRanges().toSourceRangeContainers();
 }
 
-
 ToolTipInfo TranslationUnit::tooltip(UnsavedFiles &unsavedFiles,
                                      const Utf8String &textCodecName,
                                      uint line,
@@ -217,6 +216,21 @@ TokenInfos TranslationUnit::tokenInfosInRange(const SourceRange &range) const
     return TokenInfos(m_cxTranslationUnit, cxTokens, cxTokensCount);
 }
 
+FullTokenInfos TranslationUnit::fullTokenInfos() const
+{
+    return fullTokenInfosInRange(cursor().sourceRange());
+}
+
+FullTokenInfos TranslationUnit::fullTokenInfosInRange(const SourceRange &range) const
+{
+    CXToken *cxTokens = 0;
+    uint cxTokensCount = 0;
+
+    clang_tokenize(m_cxTranslationUnit, range, &cxTokens, &cxTokensCount);
+
+    return FullTokenInfos(m_cxTranslationUnit, cxTokens, cxTokensCount);
+}
+
 SkippedSourceRanges TranslationUnit::skippedSourceRanges() const
 {
     return SkippedSourceRanges(m_cxTranslationUnit, m_filePath.constData());

src/tools/clangbackend/source/clangtranslationunit.h

@@ -36,6 +36,7 @@ class DiagnosticContainer;
 class DiagnosticSet;
 class TokenInfoContainer;
 class TokenInfos;
+class FullTokenInfos;
 class ReferencesResult;
 class SkippedSourceRanges;
 class SourceLocation;
@@ -85,7 +86,6 @@ public:
                                     QVector<TokenInfoContainer> &tokenInfos,
                                     QVector<SourceRangeContainer> &skippedSourceRanges) const;
 
-
     ReferencesResult references(uint line, uint column, bool localReferences = false) const;
     ToolTipInfo tooltip(UnsavedFiles &unsavedFiles,
                         const Utf8String &textCodecName,
@@ -104,6 +104,9 @@ public:
     TokenInfos tokenInfos() const;
     TokenInfos tokenInfosInRange(const SourceRange &range) const;
 
+    FullTokenInfos fullTokenInfos() const;
+    FullTokenInfos fullTokenInfosInRange(const SourceRange &range) const;
+
     SkippedSourceRanges skippedSourceRanges() const;
     SourceRangeContainer followSymbol(uint line, uint column) const;
 

src/tools/clangbackend/source/clangupdateextradocumentannotationsjob.cpp

@@ -0,0 +1,61 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "clangupdateextradocumentannotationsjob.h"
+#include "fulltokeninfos.h"
+
+#include <clangsupport/clangsupportdebugutils.h>
+#include <clangsupport/clangcodemodelclientinterface.h>
+#include <clangsupport/documentannotationschangedmessage.h>
+
+#include <utils/qtcassert.h>
+
+namespace ClangBackEnd {
+
+IAsyncJob::AsyncPrepareResult UpdateExtraDocumentAnnotationsJob::prepareAsyncRun()
+{
+    const JobRequest jobRequest = context().jobRequest;
+    QTC_ASSERT(acquireDocument(), return AsyncPrepareResult());
+
+    const TranslationUnit translationUnit = *m_translationUnit;
+    setRunner([translationUnit]() {
+        TIME_SCOPE_DURATION("UpdateExtraDocumentAnnotationsJobRunner");
+        return translationUnit.fullTokenInfos().toTokenInfoContainers();
+    });
+
+    return AsyncPrepareResult{translationUnit.id()};
+}
+
+void UpdateExtraDocumentAnnotationsJob::finalizeAsyncRun()
+{
+    if (context().isOutdated())
+        return;
+
+    context().client->documentAnnotationsChanged(
+                DocumentAnnotationsChangedMessage(m_pinnedFileContainer, asyncResult()));
+}
+
+} // namespace ClangBackEnd
+

src/tools/clangbackend/source/clangupdateextradocumentannotationsjob.h

@@ -0,0 +1,43 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "clangdocumentjob.h"
+
+#include <clangsupport/tokeninfocontainer.h>
+
+namespace ClangBackEnd {
+
+using UpdateExtraDocumentAnnotationsJobResult = QVector<TokenInfoContainer>;
+
+class UpdateExtraDocumentAnnotationsJob : public DocumentJob<UpdateExtraDocumentAnnotationsJobResult>
+{
+public:
+    AsyncPrepareResult prepareAsyncRun() override;
+    void finalizeAsyncRun() override;
+};
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/fulltokeninfo.cpp

@@ -0,0 +1,153 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "clangstring.h"
+#include "cursor.h"
+#include "fulltokeninfo.h"
+
+namespace ClangBackEnd {
+
+FullTokenInfo::FullTokenInfo(const CXCursor &cxCursor,
+                             CXToken *cxToken,
+                             CXTranslationUnit cxTranslationUnit,
+                             std::vector<CXSourceRange> &currentOutputArgumentRanges)
+    : TokenInfo(cxCursor, cxToken, cxTranslationUnit, currentOutputArgumentRanges)
+{
+}
+
+FullTokenInfo::operator TokenInfoContainer() const
+{
+    return TokenInfoContainer(line(), column(), length(), types(), m_extraInfo);
+}
+
+static Utf8String fullyQualifiedType(const Cursor &cursor)
+{
+    Utf8String typeSpelling = cursor.type().canonical().utf8Spelling();
+    if (typeSpelling.isEmpty()) {
+        // Only if it's the namespaces level.
+        typeSpelling = cursor.unifiedSymbolResolution();
+        typeSpelling.replace(Utf8StringLiteral("c:@N@"), Utf8StringLiteral(""));
+        typeSpelling.replace(Utf8StringLiteral("@N@"), Utf8StringLiteral("::"));
+    }
+    return typeSpelling;
+}
+
+void FullTokenInfo::updateTypeSpelling(const Cursor &cursor, bool functionLike)
+{
+    m_extraInfo.typeSpelling = fullyQualifiedType(cursor);
+    m_extraInfo.semanticParentTypeSpelling = fullyQualifiedType(cursor.semanticParent());
+    if (!functionLike)
+        return;
+    Type type = cursor.type().canonical();
+    m_extraInfo.resultTypeSpelling = type.resultType().utf8Spelling();
+    bool hasSpaceAfterReturnType = false;
+    if (m_extraInfo.resultTypeSpelling.byteSize() < m_extraInfo.typeSpelling.byteSize()) {
+        const char *data = m_extraInfo.typeSpelling.constData();
+        hasSpaceAfterReturnType = (data[m_extraInfo.resultTypeSpelling.byteSize()] == ' ');
+    }
+    m_extraInfo.typeSpelling
+            = m_extraInfo.typeSpelling.mid(m_extraInfo.resultTypeSpelling.byteSize()
+                                         + (hasSpaceAfterReturnType ? 1 : 0));
+}
+
+void FullTokenInfo::identifierKind(const Cursor &cursor, Recursion recursion)
+{
+    updateTypeSpelling(cursor);
+
+    TokenInfo::identifierKind(cursor, recursion);
+
+    m_extraInfo.identifier = (cursor.kind() != CXCursor_PreprocessingDirective);
+}
+
+void FullTokenInfo::referencedTypeKind(const Cursor &cursor)
+{
+    updateTypeSpelling(cursor.referenced());
+
+    TokenInfo::referencedTypeKind(cursor);
+}
+
+void FullTokenInfo::functionKind(const Cursor &cursor, Recursion recursion)
+{
+    updateTypeSpelling(cursor, true);
+
+    TokenInfo::functionKind(cursor, recursion);
+
+    m_extraInfo.accessSpecifier = cursor.accessSpecifier();
+    m_extraInfo.storageClass = cursor.storageClass();
+
+    bool isSignal = false;
+    bool isSlot = false;
+    m_originalCursor.visit([&isSignal, &isSlot](CXCursor cursor, CXCursor) {
+        Cursor cur(cursor);
+        ClangString spelling = cur.spelling();
+        if (spelling == "qt_signal")
+            isSignal = true;
+        else if (spelling == "qt_slot")
+            isSlot = true;
+        return CXChildVisit_Break;
+    });
+    m_extraInfo.signal = isSignal;
+    m_extraInfo.slot = isSlot;
+}
+
+void FullTokenInfo::variableKind(const Cursor &cursor)
+{
+    TokenInfo::variableKind(cursor);
+
+    m_extraInfo.storageClass = cursor.storageClass();
+}
+
+void FullTokenInfo::fieldKind(const Cursor &cursor)
+{
+    TokenInfo::fieldKind(cursor);
+
+    m_extraInfo.accessSpecifier = cursor.accessSpecifier();
+    m_extraInfo.storageClass = cursor.storageClass();
+}
+
+void FullTokenInfo::memberReferenceKind(const Cursor &cursor)
+{
+    TokenInfo::memberReferenceKind(cursor);
+    if (cursor.isDynamicCall()) {
+        m_extraInfo.storageClass = cursor.storageClass();
+        m_extraInfo.accessSpecifier = cursor.accessSpecifier();
+    }
+}
+
+void FullTokenInfo::evaluate()
+{
+    TokenInfo::evaluate();
+
+    m_extraInfo.token = ClangString(clang_getTokenSpelling(m_cxTranslationUnit, *m_cxToken));
+
+    auto cxTokenKind = clang_getTokenKind(*m_cxToken);
+    if (cxTokenKind == CXToken_Identifier) {
+        m_extraInfo.declaration = m_originalCursor.isDeclaration();
+        m_extraInfo.definition = m_originalCursor.isDefinition();
+    }
+    m_extraInfo.includeDirectivePath = (m_originalCursor.kind() == CXCursor_InclusionDirective);
+}
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/fulltokeninfo.h

@@ -0,0 +1,55 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "tokeninfo.h"
+
+namespace ClangBackEnd {
+
+class FullTokenInfo : public TokenInfo
+{
+public:
+    FullTokenInfo(const CXCursor &cxCursor,
+                  CXToken *cxToken,
+                  CXTranslationUnit cxTranslationUnit,
+                  std::vector<CXSourceRange> &m_currentOutputArgumentRanges);
+    void evaluate() override;
+
+    operator TokenInfoContainer() const override;
+protected:
+    void identifierKind(const Cursor &cursor, Recursion recursion) override;
+    void referencedTypeKind(const Cursor &cursor) override;
+    void functionKind(const Cursor &cursor, Recursion recursion) override;
+    void variableKind(const Cursor &cursor) override;
+    void fieldKind(const Cursor &cursor) override;
+    void memberReferenceKind(const Cursor &cursor) override;
+private:
+    void updateTypeSpelling(const Cursor &cursor, bool functionLike = false);
+
+    ExtraInfo m_extraInfo;
+};
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/fulltokeninfos.cpp

@@ -0,0 +1,93 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "fulltokeninfos.h"
+
+#include <clangsupport/tokeninfocontainer.h>
+
+#include <QVector>
+
+namespace ClangBackEnd {
+
+FullTokenInfos::FullTokenInfos(CXTranslationUnit cxTranslationUnit, CXToken *tokens, uint tokensCount)
+    : cxTranslationUnit(cxTranslationUnit),
+      cxTokens(tokens),
+      cxTokenCount(tokensCount)
+{
+    cxCursors.resize(tokensCount);
+    clang_annotateTokens(cxTranslationUnit, cxTokens, cxTokenCount, cxCursors.data());
+}
+
+FullTokenInfos::~FullTokenInfos()
+{
+    clang_disposeTokens(cxTranslationUnit, cxTokens, cxTokenCount);
+}
+
+QVector<TokenInfoContainer> FullTokenInfos::toTokenInfoContainers() const
+{
+    QVector<TokenInfoContainer> containers;
+    containers.reserve(static_cast<int>(size()));
+
+    const auto isValidTokenInfo = [] (const TokenInfo &tokenInfo) {
+        // Do not exclude StringLiteral because it can be a filename for an #include
+        return !tokenInfo.hasInvalidMainType()
+                && !tokenInfo.hasMainType(HighlightingType::NumberLiteral)
+                && !tokenInfo.hasMainType(HighlightingType::Comment);
+    };
+    for (size_t index = 0; index < cxCursors.size(); ++index) {
+        FullTokenInfo fullTokenInfo = (*this)[index];
+        if (isValidTokenInfo(fullTokenInfo))
+            containers.push_back(fullTokenInfo);
+    }
+
+    return containers;
+}
+
+bool FullTokenInfos::isEmpty() const
+{
+    return cxTokenCount == 0;
+}
+
+bool FullTokenInfos::isNull() const
+{
+    return cxTokens == nullptr;
+}
+
+size_t FullTokenInfos::size() const
+{
+    return cxTokenCount;
+}
+
+FullTokenInfo FullTokenInfos::operator[](size_t index) const
+{
+    FullTokenInfo tokenInfo(cxCursors[index],
+                            cxTokens + index,
+                            cxTranslationUnit,
+                            currentOutputArgumentRanges);
+    tokenInfo.evaluate();
+    return tokenInfo;
+}
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/fulltokeninfos.h

@@ -0,0 +1,59 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "fulltokeninfo.h"
+
+#include <clang-c/Index.h>
+
+#include <vector>
+
+namespace ClangBackEnd {
+
+class FullTokenInfos
+{
+public:
+    FullTokenInfos() = default;
+    FullTokenInfos(CXTranslationUnit cxTranslationUnit, CXToken *tokens, uint tokensCount);
+    ~FullTokenInfos();
+
+    bool isEmpty() const;
+    bool isNull() const;
+    size_t size() const;
+
+    FullTokenInfo operator[](size_t index) const;
+    QVector<TokenInfoContainer> toTokenInfoContainers() const;
+
+private:
+    mutable std::vector<CXSourceRange> currentOutputArgumentRanges;
+    CXTranslationUnit cxTranslationUnit = nullptr;
+    CXToken *const cxTokens = nullptr;
+    const uint cxTokenCount = 0;
+
+    std::vector<CXCursor> cxCursors;
+};
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/tokeninfo.cpp

@@ -23,8 +23,6 @@
 **
 ****************************************************************************/
 
-#include <tokeninfocontainer.h>
-
 #include "clangstring.h"
 #include "cursor.h"
 #include "tokeninfo.h"
@@ -42,8 +40,10 @@ TokenInfo::TokenInfo(const CXCursor &cxCursor,
                      CXToken *cxToken,
                      CXTranslationUnit cxTranslationUnit,
                      std::vector<CXSourceRange> &currentOutputArgumentRanges)
-    : m_currentOutputArgumentRanges(&currentOutputArgumentRanges),
+    : m_originalCursor(cxCursor),
-      m_originalCursor(cxCursor)
+      m_cxToken(cxToken),
+      m_cxTranslationUnit(cxTranslationUnit),
+      m_currentOutputArgumentRanges(&currentOutputArgumentRanges)
 {
     const SourceRange sourceRange {cxTranslationUnit,
                                    clang_getTokenExtent(cxTranslationUnit, *cxToken)};
@@ -54,24 +54,6 @@ TokenInfo::TokenInfo(const CXCursor &cxCursor,
     m_column = start.column();
     m_offset = start.offset();
     m_length = end.offset() - start.offset();
-    collectKinds(cxTranslationUnit, cxToken, m_originalCursor);
-}
-
-TokenInfo::TokenInfo(uint line, uint column, uint length, HighlightingTypes types)
-    : m_line(line),
-      m_column(column),
-      m_length(length),
-      m_types(types)
-{
-}
-
-TokenInfo::TokenInfo(uint line, uint column, uint length, HighlightingType type)
-    : m_line(line),
-      m_column(column),
-      m_length(length),
-      m_types(HighlightingTypes())
-{
-    m_types.mainHighlightingType = type;
 }
 
 bool TokenInfo::hasInvalidMainType() const
@@ -115,11 +97,7 @@ bool TokenInfo::hasFunctionArguments() const
 
 TokenInfo::operator TokenInfoContainer() const
 {
-    return TokenInfoContainer(m_line, m_column, m_length, m_types, m_token, m_typeSpelling,
+    return TokenInfoContainer(m_line, m_column, m_length, m_types);
-                              m_resultTypeSpelling, m_semanticParentTypeSpelling,
-                              m_accessSpecifier, m_storageClass,
-                              m_isIdentifier, m_isInclusion,
-                              m_isDeclaration, m_isDefinition);
 }
 
 namespace {
@@ -146,8 +124,6 @@ bool isFunctionInFinalClass(const Cursor &cursor)
 void TokenInfo::memberReferenceKind(const Cursor &cursor)
 {
     if (cursor.isDynamicCall()) {
-        m_storageClass = cursor.storageClass();
-        m_accessSpecifier = cursor.accessSpecifier();
         if (isFinalFunction(cursor) || isFunctionInFinalClass(cursor))
             m_types.mainHighlightingType = HighlightingType::Function;
         else
@@ -173,7 +149,6 @@ void TokenInfo::overloadedDeclRefKind(const Cursor &cursor)
 
 void TokenInfo::variableKind(const Cursor &cursor)
 {
-    m_storageClass = cursor.storageClass();
     if (cursor.isLocalVariable())
         m_types.mainHighlightingType = HighlightingType::LocalVariable;
     else
@@ -183,10 +158,8 @@ void TokenInfo::variableKind(const Cursor &cursor)
         m_types.mixinHighlightingTypes.push_back(HighlightingType::OutputArgument);
 }
 
-void TokenInfo::fieldKind(const Cursor &cursor)
+void TokenInfo::fieldKind(const Cursor &)
 {
-    m_accessSpecifier = cursor.accessSpecifier();
-    m_storageClass = cursor.storageClass();
     m_types.mainHighlightingType = HighlightingType::Field;
 
     if (isOutputArgument())
@@ -283,11 +256,6 @@ void TokenInfo::filterOutPreviousOutputArguments()
 
 void TokenInfo::functionKind(const Cursor &cursor, Recursion recursion)
 {
-    updateTypeSpelling(cursor, true);
-
-    m_accessSpecifier = cursor.accessSpecifier();
-    m_storageClass = cursor.storageClass();
-
     if (isRealDynamicCall(cursor) || isVirtualMethodDeclarationOrDefinition(cursor))
         m_types.mainHighlightingType = HighlightingType::VirtualFunction;
     else
@@ -302,37 +270,9 @@ void TokenInfo::functionKind(const Cursor &cursor, Recursion recursion)
         addExtraTypeIfFirstPass(HighlightingType::FunctionDefinition, recursion);
 }
 
-Utf8String fullyQualifiedType(const Cursor &cursor)
-{
-    Utf8String typeSpelling = cursor.type().canonical().utf8Spelling();
-    if (typeSpelling.isEmpty()) {
-        // Only if it's the namespaces level.
-        typeSpelling = cursor.unifiedSymbolResolution();
-        typeSpelling.replace(Utf8StringLiteral("c:@N@"), Utf8StringLiteral(""));
-        typeSpelling.replace(Utf8StringLiteral("@N@"), Utf8StringLiteral("::"));
-    }
-    return typeSpelling;
-}
-
-void TokenInfo::updateTypeSpelling(const Cursor &cursor, bool functionLike)
-{
-    m_typeSpelling = fullyQualifiedType(cursor);
-    m_semanticParentTypeSpelling = fullyQualifiedType(cursor.semanticParent());
-    if (!functionLike)
-        return;
-    Type type = cursor.type().canonical();
-    m_resultTypeSpelling = type.resultType().utf8Spelling();
-    const bool hasSpaceAfterReturnVal
-            = m_typeSpelling.constData()[m_resultTypeSpelling.byteSize()] == ' ';
-    m_typeSpelling = m_typeSpelling.mid(m_resultTypeSpelling.byteSize()
-                                        + (hasSpaceAfterReturnVal ? 1 : 0));
-}
-
 void TokenInfo::referencedTypeKind(const Cursor &cursor)
 {
-    const Cursor ref = cursor.referenced();
+    typeKind(cursor.referenced());
-    updateTypeSpelling(ref);
-    typeKind(ref);
 }
 
 void TokenInfo::typeKind(const Cursor &cursor)
@@ -390,12 +330,7 @@ void TokenInfo::typeKind(const Cursor &cursor)
 
 void TokenInfo::identifierKind(const Cursor &cursor, Recursion recursion)
 {
-    // For now save type spelling only for declarations.
-    if (m_isDeclaration)
-        updateTypeSpelling(cursor);
-
     const CXCursorKind kind = cursor.kind();
-    m_isIdentifier = (kind != CXCursor_PreprocessingDirective);
 
     switch (kind) {
         case CXCursor_Destructor:
@@ -403,30 +338,47 @@ void TokenInfo::identifierKind(const Cursor &cursor, Recursion recursion)
         case CXCursor_FunctionDecl:
         case CXCursor_FunctionTemplate:
         case CXCursor_CallExpr:
-        case CXCursor_CXXMethod:                 functionKind(cursor, recursion); break;
+        case CXCursor_CXXMethod:
-        case CXCursor_NonTypeTemplateParameter:  m_types.mainHighlightingType = HighlightingType::LocalVariable; break;
+            functionKind(cursor, recursion);
+            break;
+        case CXCursor_NonTypeTemplateParameter:
+            m_types.mainHighlightingType = HighlightingType::LocalVariable;
+            break;
         case CXCursor_ParmDecl:
-        case CXCursor_VarDecl:                   variableKind(cursor); break;
+        case CXCursor_VarDecl:
-        case CXCursor_DeclRefExpr:               identifierKind(cursor.referenced(), Recursion::RecursivePass); break;
+            variableKind(cursor);
-        case CXCursor_MemberRefExpr:             memberReferenceKind(cursor); break;
+            break;
+        case CXCursor_DeclRefExpr:
+            identifierKind(cursor.referenced(), Recursion::RecursivePass);
+            break;
+        case CXCursor_MemberRefExpr:
+            memberReferenceKind(cursor);
+            break;
         case CXCursor_FieldDecl:
-        case CXCursor_MemberRef:                 fieldKind(cursor); break;
+        case CXCursor_MemberRef:
+            fieldKind(cursor);
+            break;
         case CXCursor_ObjCIvarDecl:
         case CXCursor_ObjCPropertyDecl:
         case CXCursor_ObjCClassMethodDecl:
         case CXCursor_ObjCInstanceMethodDecl:
         case CXCursor_ObjCSynthesizeDecl:
-        case CXCursor_ObjCDynamicDecl:           m_types.mainHighlightingType = HighlightingType::Field; break;
+        case CXCursor_ObjCDynamicDecl:
+            m_types.mainHighlightingType = HighlightingType::Field;
+            break;
         case CXCursor_TemplateRef:
         case CXCursor_NamespaceRef:
-        case CXCursor_TypeRef:                   referencedTypeKind(cursor); break;
+        case CXCursor_TypeRef:
+            referencedTypeKind(cursor);
+            break;
         case CXCursor_ClassDecl:
         case CXCursor_ClassTemplate:
         case CXCursor_ClassTemplatePartialSpecialization:
         case CXCursor_UnionDecl:
         case CXCursor_StructDecl:
         case CXCursor_EnumDecl:
-        case CXCursor_Namespace:                 m_types.mixinHighlightingTypes.push_back(HighlightingType::Declaration);
+        case CXCursor_Namespace:
+            m_types.mixinHighlightingTypes.push_back(HighlightingType::Declaration);
             Q_FALLTHROUGH();
         case CXCursor_TemplateTypeParameter:
         case CXCursor_TemplateTemplateParameter:
@@ -442,16 +394,33 @@ void TokenInfo::identifierKind(const Cursor &cursor, Recursion recursion)
         case CXCursor_ObjCProtocolDecl:
         case CXCursor_ObjCProtocolRef:
         case CXCursor_ObjCClassRef:
-        case CXCursor_ObjCSuperClassRef:         typeKind(cursor); break;
+        case CXCursor_ObjCSuperClassRef:
-        case CXCursor_OverloadedDeclRef:         overloadedDeclRefKind(cursor); break;
+            typeKind(cursor);
-        case CXCursor_EnumConstantDecl:          m_types.mainHighlightingType = HighlightingType::Enumeration; break;
+            break;
-        case CXCursor_PreprocessingDirective:    m_types.mainHighlightingType = HighlightingType::Preprocessor; break;
+        case CXCursor_OverloadedDeclRef:
-        case CXCursor_MacroExpansion:            m_types.mainHighlightingType = HighlightingType::PreprocessorExpansion; break;
+            overloadedDeclRefKind(cursor);
-        case CXCursor_MacroDefinition:           m_types.mainHighlightingType = HighlightingType::PreprocessorDefinition; break;
+            break;
-        case CXCursor_InclusionDirective:        m_types.mainHighlightingType = HighlightingType::StringLiteral; break;
+        case CXCursor_EnumConstantDecl:
+            m_types.mainHighlightingType = HighlightingType::Enumeration;
+            break;
+        case CXCursor_PreprocessingDirective:
+            m_types.mainHighlightingType = HighlightingType::Preprocessor;
+            break;
+        case CXCursor_MacroExpansion:
+            m_types.mainHighlightingType = HighlightingType::PreprocessorExpansion;
+            break;
+        case CXCursor_MacroDefinition:
+            m_types.mainHighlightingType = HighlightingType::PreprocessorDefinition;
+            break;
+        case CXCursor_InclusionDirective:
+            m_types.mainHighlightingType = HighlightingType::StringLiteral;
+            break;
         case CXCursor_LabelRef:
-        case CXCursor_LabelStmt:                 m_types.mainHighlightingType = HighlightingType::Label; break;
+        case CXCursor_LabelStmt:
-        default:                                 break;
+            m_types.mainHighlightingType = HighlightingType::Label;
+            break;
+        default:
+            break;
     }
 }
 
@@ -462,11 +431,14 @@ HighlightingType literalKind(const Cursor &cursor)
         case CXCursor_CharacterLiteral:
         case CXCursor_StringLiteral:
         case CXCursor_InclusionDirective:
-        case CXCursor_ObjCStringLiteral: return HighlightingType::StringLiteral;
+        case CXCursor_ObjCStringLiteral:
+            return HighlightingType::StringLiteral;
         case CXCursor_IntegerLiteral:
         case CXCursor_ImaginaryLiteral:
-        case CXCursor_FloatingLiteral:   return HighlightingType::NumberLiteral;
+        case CXCursor_FloatingLiteral:
-        default:                         return HighlightingType::Invalid;
+            return HighlightingType::NumberLiteral;
+        default:
+            return HighlightingType::Invalid;
     }
 
     Q_UNREACHABLE();
@@ -534,28 +506,31 @@ static HighlightingType highlightingTypeForKeyword(CXTranslationUnit cxTranslati
     return HighlightingType::Keyword;
 }
 
-void TokenInfo::collectKinds(CXTranslationUnit cxTranslationUnit,
+void TokenInfo::evaluate()
-                                    CXToken *cxToken, const Cursor &cursor)
 {
-    auto cxTokenKind = clang_getTokenKind(*cxToken);
+    auto cxTokenKind = clang_getTokenKind(*m_cxToken);
 
     m_types = HighlightingTypes();
-    m_token = ClangString(clang_getTokenSpelling(cxTranslationUnit, *cxToken));
-
-    if (cxTokenKind == CXToken_Identifier) {
-        m_isDeclaration = cursor.isDeclaration();
-        m_isDefinition = cursor.isDefinition();
-    }
 
     switch (cxTokenKind) {
-        case CXToken_Keyword:     m_types.mainHighlightingType = highlightingTypeForKeyword(cxTranslationUnit, cxToken, m_originalCursor); break;
+        case CXToken_Keyword:
-        case CXToken_Punctuation: m_types.mainHighlightingType = punctuationKind(cursor); break;
+            m_types.mainHighlightingType = highlightingTypeForKeyword(m_cxTranslationUnit,
-        case CXToken_Identifier:  identifierKind(cursor, Recursion::FirstPass); break;
+                                                                      m_cxToken,
-        case CXToken_Comment:     m_types.mainHighlightingType = HighlightingType::Comment; break;
+                                                                      m_originalCursor);
-        case CXToken_Literal:     m_types.mainHighlightingType = literalKind(cursor); break;
+            break;
+        case CXToken_Punctuation:
+            m_types.mainHighlightingType = punctuationKind(m_originalCursor);
+            break;
+        case CXToken_Identifier:
+            identifierKind(m_originalCursor, Recursion::FirstPass);
+            break;
+        case CXToken_Comment:
+            m_types.mainHighlightingType = HighlightingType::Comment;
+            break;
+        case CXToken_Literal:
+            m_types.mainHighlightingType = literalKind(m_originalCursor);
+            break;
     }
-
-    m_isInclusion = (cursor.kind() == CXCursor_InclusionDirective);
 }
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/tokeninfo.h

@@ -27,7 +27,8 @@
 
 #include "clangsupport_global.h"
 #include "cursor.h"
-#include "tokeninfocontainer.h"
+
+#include <clangsupport/tokeninfocontainer.h>
 
 #include <sqlite/utf8string.h>
 
@@ -39,18 +40,14 @@ class TokenInfo
 {
     friend bool operator==(const TokenInfo &first, const TokenInfo &second);
 
-    enum class Recursion {
-        FirstPass,
-        RecursivePass
-    };
-
 public:
     TokenInfo(const CXCursor &cxCursor,
               CXToken *cxToken,
               CXTranslationUnit cxTranslationUnit,
               std::vector<CXSourceRange> &m_currentOutputArgumentRanges);
-    TokenInfo(uint m_line, uint m_column, uint m_length, HighlightingTypes m_types);
+    virtual ~TokenInfo() = default;
-    TokenInfo(uint m_line, uint m_column, uint m_length, HighlightingType type);
+
+    virtual void evaluate();
 
     bool hasInvalidMainType() const;
     bool hasMainType(HighlightingType type) const;
@@ -60,7 +57,7 @@ public:
     bool hasOnlyType(HighlightingType type) const;
     bool hasFunctionArguments() const;
 
-    operator TokenInfoContainer() const;
+    virtual operator TokenInfoContainer() const;
 
     const HighlightingTypes &types() const
     {
@@ -82,45 +79,42 @@ public:
         return m_length;
     }
 
+protected:
+    enum class Recursion {
+        FirstPass,
+        RecursivePass
+    };
+
+    virtual void identifierKind(const Cursor &cursor, Recursion recursion);
+    virtual void referencedTypeKind(const Cursor &cursor);
+    virtual void overloadedDeclRefKind(const Cursor &cursor);
+    virtual void variableKind(const Cursor &cursor);
+    virtual void fieldKind(const Cursor &cursor);
+    virtual void functionKind(const Cursor &cursor, Recursion recursion);
+    virtual void memberReferenceKind(const Cursor &cursor);
+    virtual HighlightingType punctuationKind(const Cursor &cursor);
+    virtual void typeKind(const Cursor &cursor);
+
+    Cursor m_originalCursor;
+    CXToken *m_cxToken;
+    CXTranslationUnit m_cxTranslationUnit;
+
 private:
-    void identifierKind(const Cursor &cursor, Recursion recursion);
-    void referencedTypeKind(const Cursor &cursor);
-    void overloadedDeclRefKind(const Cursor &cursor);
-    void variableKind(const Cursor &cursor);
-    void fieldKind(const Cursor &cursor);
     bool isVirtualMethodDeclarationOrDefinition(const Cursor &cursor) const;
     bool isDefinition() const;
-    void functionKind(const Cursor &cursor, Recursion recursion);
-    void memberReferenceKind(const Cursor &cursor);
-    HighlightingType punctuationKind(const Cursor &cursor);
-    void typeKind(const Cursor &cursor);
-    void collectKinds(CXTranslationUnit cxTranslationUnit, CXToken *cxToken, const Cursor &cursor);
     bool isRealDynamicCall(const Cursor &cursor) const;
     void addExtraTypeIfFirstPass(HighlightingType type, Recursion recursion);
     bool isOutputArgument() const;
     void collectOutputArguments(const Cursor &cursor);
     void filterOutPreviousOutputArguments();
     bool isArgumentInCurrentOutputArgumentLocations() const;
-    void updateTypeSpelling(const Cursor &cursor, bool functionLike = false);
 
-private:
     std::vector<CXSourceRange> *m_currentOutputArgumentRanges = nullptr;
-    Cursor m_originalCursor;
     uint m_line;
     uint m_column;
     uint m_length;
     uint m_offset = 0;
     HighlightingTypes m_types;
-    Utf8String m_token;
-    Utf8String m_typeSpelling;
-    Utf8String m_resultTypeSpelling;
-    Utf8String m_semanticParentTypeSpelling;
-    AccessSpecifier m_accessSpecifier = AccessSpecifier::Invalid;
-    StorageClass m_storageClass = StorageClass::Invalid;
-    bool m_isIdentifier = false;
-    bool m_isInclusion = false;
-    bool m_isDeclaration = false;
-    bool m_isDefinition = false;
 };
 
 

src/tools/clangbackend/source/tokeninfos.cpp

@@ -33,30 +33,30 @@ namespace ClangBackEnd {
 
 TokenInfos::TokenInfos(CXTranslationUnit cxTranslationUnit, CXToken *tokens, uint tokensCount)
     : cxTranslationUnit(cxTranslationUnit),
-      cxToken(tokens),
+      cxTokens(tokens),
       cxTokenCount(tokensCount)
 {
-    cxCursor.resize(tokensCount);
+    cxCursors.resize(tokensCount);
-    clang_annotateTokens(cxTranslationUnit, cxToken, cxTokenCount, cxCursor.data());
+    clang_annotateTokens(cxTranslationUnit, cxTokens, cxTokenCount, cxCursors.data());
 }
 
 TokenInfos::~TokenInfos()
 {
-    clang_disposeTokens(cxTranslationUnit, cxToken, cxTokenCount);
+    clang_disposeTokens(cxTranslationUnit, cxTokens, cxTokenCount);
 }
 
 TokenInfos::const_iterator TokenInfos::begin() const
 {
-    return const_iterator(cxCursor.cbegin(),
+    return const_iterator(cxCursors.cbegin(),
-                          cxToken,
+                          cxTokens,
                           cxTranslationUnit,
                           currentOutputArgumentRanges);
 }
 
 TokenInfos::const_iterator TokenInfos::end() const
 {
-    return const_iterator(cxCursor.cend(),
+    return const_iterator(cxCursors.cend(),
-                          cxToken + cxTokenCount,
+                          cxTokens + cxTokenCount,
                           cxTranslationUnit,
                           currentOutputArgumentRanges);
 }
@@ -90,7 +90,7 @@ bool TokenInfos::isEmpty() const
 
 bool ClangBackEnd::TokenInfos::isNull() const
 {
-    return cxToken == nullptr;
+    return cxTokens == nullptr;
 }
 
 uint TokenInfos::size() const
@@ -100,10 +100,12 @@ uint TokenInfos::size() const
 
 TokenInfo TokenInfos::operator[](size_t index) const
 {
-    return TokenInfo(cxCursor[index],
+    TokenInfo tokenInfo(cxCursors[index],
-                            cxToken + index,
+                        cxTokens + index,
                         cxTranslationUnit,
                         currentOutputArgumentRanges);
+    tokenInfo.evaluate();
+    return tokenInfo;
 }
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/tokeninfos.h

@@ -63,10 +63,10 @@ public:
 private:
     mutable std::vector<CXSourceRange> currentOutputArgumentRanges;
     CXTranslationUnit cxTranslationUnit = nullptr;
-    CXToken *const cxToken = nullptr;
+    CXToken *const cxTokens = nullptr;
     const uint cxTokenCount = 0;
 
-    std::vector<CXCursor> cxCursor;
+    std::vector<CXCursor> cxCursors;
 };
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/tokeninfositerator.h

@@ -80,10 +80,12 @@ public:
 
     TokenInfo operator*()
     {
-        return TokenInfo(*cxCursorIterator,
+        TokenInfo tokenInfo(*cxCursorIterator,
                             cxToken,
                             cxTranslationUnit,
                             currentOutputArgumentRanges);
+        tokenInfo.evaluate();
+        return tokenInfo;
     }
 
 private:

tests/unit/unittest/clangcodemodelserver-test.cpp

@@ -92,6 +92,8 @@ MATCHER_P5(HasDirtyDocument,
     }
 }
 
+static constexpr int AnnotationJobsMultiplier = 2;
+
 class ClangCodeModelServer : public ::testing::Test
 {
 protected:
@@ -106,14 +108,14 @@ protected:
     void changeProjectPartArguments();
 
     void registerProjectAndFile(const Utf8String &filePath,
-                                int expectedDocumentAnnotationsChangedMessages = 1);
+                                int expectedDocumentAnnotationsChangedMessages = AnnotationJobsMultiplier);
     void registerProjectAndFileAndWaitForFinished(const Utf8String &filePath,
-                                                  int expectedDocumentAnnotationsChangedMessages = 1);
+                                                  int expectedDocumentAnnotationsChangedMessages = AnnotationJobsMultiplier);
-    void registerProjectAndFilesAndWaitForFinished(int expectedDocumentAnnotationsChangedMessages = 2);
+    void registerProjectAndFilesAndWaitForFinished(int expectedDocumentAnnotationsChangedMessages = 2 * AnnotationJobsMultiplier);
     void registerFile(const Utf8String &filePath,
-                      int expectedDocumentAnnotationsChangedMessages = 1);
+                      int expectedDocumentAnnotationsChangedMessages = AnnotationJobsMultiplier);
     void registerFile(const Utf8String &filePath, const Utf8String &projectPartId,
-                      int expectedDocumentAnnotationsChangedMessages = 1);
+                      int expectedDocumentAnnotationsChangedMessages = AnnotationJobsMultiplier);
     void registerFiles(int expectedDocumentAnnotationsChangedMessages);
     void registerFileWithUnsavedContent(const Utf8String &filePath, const Utf8String &content);
 
@@ -245,7 +247,7 @@ TEST_F(ClangCodeModelServerSlowTest, NoInitialDocumentAnnotationsForClosedDocume
 
 TEST_F(ClangCodeModelServerSlowTest, NoDocumentAnnotationsForClosedDocument)
 {
-    const int expectedDocumentAnnotationsChangedCount = 1; // Only for registration.
+    const int expectedDocumentAnnotationsChangedCount = AnnotationJobsMultiplier; // Only for registration.
     registerProjectAndFileAndWaitForFinished(filePathA, expectedDocumentAnnotationsChangedCount);
     updateUnsavedContent(filePathA, Utf8String(), 1);
 
@@ -254,7 +256,7 @@ TEST_F(ClangCodeModelServerSlowTest, NoDocumentAnnotationsForClosedDocument)
 
 TEST_F(ClangCodeModelServerSlowTest, NoInitialDocumentAnnotationsForOutdatedDocumentRevision)
 {
-    const int expectedDocumentAnnotationsChangedCount = 1; // Only for registration.
+    const int expectedDocumentAnnotationsChangedCount = AnnotationJobsMultiplier; // Only for registration.
     registerProjectAndFile(filePathA, expectedDocumentAnnotationsChangedCount);
 
     updateUnsavedContent(filePathA, Utf8String(), 1);
@@ -262,7 +264,7 @@ TEST_F(ClangCodeModelServerSlowTest, NoInitialDocumentAnnotationsForOutdatedDocu
 
 TEST_F(ClangCodeModelServerSlowTest, NoCompletionsForClosedDocument)
 {
-    const int expectedDocumentAnnotationsChangedCount = 1; // Only for registration.
+    const int expectedDocumentAnnotationsChangedCount = AnnotationJobsMultiplier; // Only for registration.
     registerProjectAndFileAndWaitForFinished(filePathA, expectedDocumentAnnotationsChangedCount);
     completeCodeInFileA();
 
@@ -271,7 +273,7 @@ TEST_F(ClangCodeModelServerSlowTest, NoCompletionsForClosedDocument)
 
 TEST_F(ClangCodeModelServerSlowTest, CodeCompletionDependingOnProject)
 {
-    const int expectedDocumentAnnotationsChangedCount = 2; // For registration and due to project change.
+    const int expectedDocumentAnnotationsChangedCount = 2 * AnnotationJobsMultiplier; // For registration and due to project change.
     registerProjectAndFileAndWaitForFinished(filePathB, expectedDocumentAnnotationsChangedCount);
 
     expectCompletionFromFileBEnabledByMacro();
@@ -282,7 +284,7 @@ TEST_F(ClangCodeModelServerSlowTest, CodeCompletionDependingOnProject)
 TEST_F(ClangCodeModelServerSlowTest, GetCodeCompletionForUnsavedFile)
 {
     registerProjectPart();
-    expectDocumentAnnotationsChanged(1);
+    expectDocumentAnnotationsChanged(AnnotationJobsMultiplier);
     registerFileWithUnsavedContent(filePathA, unsavedContent(filePathAUnsavedVersion1));
     expectCompletionFromFileAUnsavedMethodVersion1();
 
@@ -291,7 +293,7 @@ TEST_F(ClangCodeModelServerSlowTest, GetCodeCompletionForUnsavedFile)
 
 TEST_F(ClangCodeModelServerSlowTest, GetNoCodeCompletionAfterRemovingUnsavedFile)
 {
-    const int expectedDocumentAnnotationsChangedCount = 2; // For registration and update/removal.
+    const int expectedDocumentAnnotationsChangedCount = 2 * AnnotationJobsMultiplier; // For registration and update/removal.
     registerProjectAndFileAndWaitForFinished(filePathA, expectedDocumentAnnotationsChangedCount);
     removeUnsavedFile(filePathA);
 
@@ -301,7 +303,7 @@ TEST_F(ClangCodeModelServerSlowTest, GetNoCodeCompletionAfterRemovingUnsavedFile
 
 TEST_F(ClangCodeModelServerSlowTest, GetNewCodeCompletionAfterUpdatingUnsavedFile)
 {
-    const int expectedDocumentAnnotationsChangedCount = 2; // For registration and update/removal.
+    const int expectedDocumentAnnotationsChangedCount = 2 * AnnotationJobsMultiplier; // For registration and update/removal.
     registerProjectAndFileAndWaitForFinished(filePathA, expectedDocumentAnnotationsChangedCount);
     updateUnsavedContent(filePathA, unsavedContent(filePathAUnsavedVersion2), 1);
 
@@ -311,7 +313,7 @@ TEST_F(ClangCodeModelServerSlowTest, GetNewCodeCompletionAfterUpdatingUnsavedFil
 
 TEST_F(ClangCodeModelServerSlowTest, TranslationUnitAfterCreationIsNotDirty)
 {
-    registerProjectAndFile(filePathA, 1);
+    registerProjectAndFile(filePathA, AnnotationJobsMultiplier);
 
     ASSERT_THAT(clangServer, HasDirtyDocument(filePathA, projectPartId, 0U, false, false));
 }
@@ -331,7 +333,7 @@ TEST_F(ClangCodeModelServerSlowTest, SetCurrentAndVisibleEditor)
 
 TEST_F(ClangCodeModelServerSlowTest, StartCompletionJobFirstOnEditThatTriggersCompletion)
 {
-    registerProjectAndFile(filePathA, 2);
+    registerProjectAndFile(filePathA, 2 * AnnotationJobsMultiplier);
     ASSERT_TRUE(waitUntilAllJobsFinished());
     expectCompletionFromFileA();
 
@@ -345,7 +347,7 @@ TEST_F(ClangCodeModelServerSlowTest, StartCompletionJobFirstOnEditThatTriggersCo
 
 TEST_F(ClangCodeModelServerSlowTest, SupportiveTranslationUnitNotInitializedAfterRegister)
 {
-    registerProjectAndFile(filePathA, 1);
+    registerProjectAndFile(filePathA, AnnotationJobsMultiplier);
 
     ASSERT_TRUE(waitUntilAllJobsFinished());
     ASSERT_FALSE(isSupportiveTranslationUnitInitialized(filePathA));
@@ -353,7 +355,7 @@ TEST_F(ClangCodeModelServerSlowTest, SupportiveTranslationUnitNotInitializedAfte
 
 TEST_F(ClangCodeModelServerSlowTest, SupportiveTranslationUnitIsSetupAfterFirstEdit)
 {
-    registerProjectAndFile(filePathA, 2);
+    registerProjectAndFile(filePathA, 2 * AnnotationJobsMultiplier);
     ASSERT_TRUE(waitUntilAllJobsFinished());
 
     updateUnsavedContent(filePathA, unsavedContent(filePathAUnsavedVersion2), 1);
@@ -364,7 +366,7 @@ TEST_F(ClangCodeModelServerSlowTest, SupportiveTranslationUnitIsSetupAfterFirstE
 
 TEST_F(ClangCodeModelServerSlowTest, DoNotRunDuplicateJobs)
 {
-    registerProjectAndFile(filePathA, 3);
+    registerProjectAndFile(filePathA, 3 * AnnotationJobsMultiplier);
     ASSERT_TRUE(waitUntilAllJobsFinished());
     updateUnsavedContent(filePathA, unsavedContent(filePathAUnsavedVersion2), 1);
     ASSERT_TRUE(waitUntilAllJobsFinished());
@@ -378,7 +380,7 @@ TEST_F(ClangCodeModelServerSlowTest, DoNotRunDuplicateJobs)
 
 TEST_F(ClangCodeModelServerSlowTest, OpenDocumentAndEdit)
 {
-    registerProjectAndFile(filePathA, 4);
+    registerProjectAndFile(filePathA, 4 * AnnotationJobsMultiplier);
     ASSERT_TRUE(waitUntilAllJobsFinished());
 
     for (unsigned revision = 1; revision <= 3; ++revision) {
@@ -404,7 +406,7 @@ TEST_F(ClangCodeModelServerSlowTest, IsNotCurrentCurrentAndVisibleEditorAnymore)
 
 TEST_F(ClangCodeModelServerSlowTest, TranslationUnitAfterUpdateNeedsReparse)
 {
-    registerProjectAndFileAndWaitForFinished(filePathA, 2);
+    registerProjectAndFileAndWaitForFinished(filePathA, 2 * AnnotationJobsMultiplier);
 
     updateUnsavedContent(filePathA, unsavedContent(filePathAUnsavedVersion1), 1U);
     ASSERT_THAT(clangServer, HasDirtyDocument(filePathA, projectPartId, 1U, true, true));
@@ -412,7 +414,7 @@ TEST_F(ClangCodeModelServerSlowTest, TranslationUnitAfterUpdateNeedsReparse)
 
 TEST_F(ClangCodeModelServerSlowTest, TakeOverJobsOnProjectPartChange)
 {
-    registerProjectAndFileAndWaitForFinished(filePathC, 2);
+    registerProjectAndFileAndWaitForFinished(filePathC, 2 * AnnotationJobsMultiplier);
     updateVisibilty(filePathB, filePathB); // Disable processing jobs
     requestReferences();
 
@@ -669,16 +671,21 @@ void ClangCodeModelServer::expectDocumentAnnotationsChangedForFileBWithSpecificH
     types.mainHighlightingType = ClangBackEnd::HighlightingType::Function;
     types.mixinHighlightingTypes.push_back(ClangBackEnd::HighlightingType::Declaration);
     types.mixinHighlightingTypes.push_back(ClangBackEnd::HighlightingType::FunctionDefinition);
-    const TokenInfoContainer tokenInfo(1, 6, 8, types, Utf8String("function", 8),
+    const TokenInfoContainer tokenInfo(1, 6, 8, types);
-                                       Utf8String("(int)", 5), Utf8String("void", 4),
+    const TokenInfoContainer fullTokenInfo(1, 6, 8, types,
-                                       Utf8String(), AccessSpecifier::Invalid,
+                                           ClangBackEnd::ExtraInfo {Utf8String("function", 8),
-                                       StorageClass::None, true, false, true, true);
+                                                                    Utf8String("(int)", 5),
+                                                                    Utf8String("void", 4),
+                                                                    Utf8String(),
+                                                                    AccessSpecifier::Invalid,
+                                                                    StorageClass::None,
+                                                                    true, false, true, true,
+                                                                    false, false, false});
 
     EXPECT_CALL(mockClangCodeModelClient,
                 documentAnnotationsChanged(
                     Property(&DocumentAnnotationsChangedMessage::tokenInfos,
-                             Contains(tokenInfo))))
+                             Contains(AnyOf(tokenInfo, fullTokenInfo)))));
-        .Times(1);
 }
 
 void ClangCodeModelServer::updateUnsavedContent(const Utf8String &filePath,

tests/unit/unittest/clientserverinprocess-test.cpp

@@ -227,7 +227,7 @@ TEST_F(ClientServerInProcess, UpdateVisibleTranslationUnitsMessage)
 
 TEST_F(ClientServerInProcess, SendDocumentAnnotationsChangedMessage)
 {
-    ClangBackEnd::TokenInfoContainer tokenInfo(1, 1, 1, ClangBackEnd::HighlightingType::Keyword);
+    ClangBackEnd::TokenInfoContainer tokenInfo(1, 1, 1, {ClangBackEnd::HighlightingType::Keyword, {}});
     ClangBackEnd::DiagnosticContainer diagnostic(Utf8StringLiteral("don't do that"),
                                                 Utf8StringLiteral("warning"),
                                                 {Utf8StringLiteral("-Wpadded"), Utf8StringLiteral("-Wno-padded")},

tests/unit/unittest/gtest-creator-printing.cpp

@@ -521,6 +521,26 @@ std::ostream &operator<<(std::ostream &os, HighlightingTypes types)
     return os;
 }
 
+std::ostream &operator<<(std::ostream &os, const ExtraInfo &extraInfo)
+{
+    os << "("
+       << extraInfo.token << ", "
+       << extraInfo.typeSpelling << ", "
+       << extraInfo.resultTypeSpelling << ", "
+       << extraInfo.semanticParentTypeSpelling << ", "
+       << static_cast<uint>(extraInfo.accessSpecifier) << ", "
+       << static_cast<uint>(extraInfo.storageClass) << ", "
+       << extraInfo.identifier << ", "
+       << extraInfo.includeDirectivePath << ", "
+       << extraInfo.declaration << ", "
+       << extraInfo.definition << ", "
+       << extraInfo.signal << ", "
+       << extraInfo.slot << ", "
+       << extraInfo.property
+       << ")";
+    return os;
+}
+
 std::ostream &operator<<(std::ostream &os, const TokenInfoContainer &container)
 {
     os << "("
@@ -528,8 +548,7 @@ std::ostream &operator<<(std::ostream &os, const TokenInfoContainer &container)
        << container.column() << ", "
        << container.length() << ", "
        << container.types() << ", "
-       << container.isIdentifier() << ", "
+       << container.extraInfo() << ", "
-       << container.isIncludeDirectivePath()
        << ")";
 
     return os;

tests/unit/unittest/readandwritemessageblock-test.cpp

@@ -174,7 +174,7 @@ TEST_F(ReadAndWriteMessageBlock, CompareDocumentAnnotationsChangedMessage)
                                                 {},
                                                 {});
 
-    ClangBackEnd::TokenInfoContainer tokenInfo(1, 1, 1, ClangBackEnd::HighlightingType::Keyword);
+    ClangBackEnd::TokenInfoContainer tokenInfo(1, 1, 1, {ClangBackEnd::HighlightingType::Keyword, {}});
 
     CompareMessage(ClangBackEnd::DocumentAnnotationsChangedMessage(fileContainer,
                                                                    {diagnostic},

tests/unit/unittest/tokeninfos-test.cpp

@@ -38,6 +38,8 @@
 #include <sourcerange.h>
 #include <tokeninfo.h>
 #include <tokeninfos.h>
+#include <fulltokeninfo.h>
+#include <fulltokeninfos.h>
 #include <unsavedfiles.h>
 
 #include <clang-c/Index.h>
@@ -72,7 +74,10 @@ namespace {
 
 MATCHER_P4(IsHighlightingMark, line, column, length, type,
            std::string(negation ? "isn't " : "is ")
-           + PrintToString(TokenInfo(line, column, length, type))
+           + PrintToString(line) + ", "
+           + PrintToString(column) + ", "
+           + PrintToString(length) + ", "
+           + PrintToString(type) + ", "
            )
 {
     return arg.line() == line && arg.column() == column && arg.length() == length
@@ -1211,45 +1216,44 @@ TEST_F(TokenInfos, UsingTemplateFunction)
 
 TEST_F(TokenInfos, HeaderNameIsInclusion)
 {
-    const auto infos = translationUnit.tokenInfosInRange(sourceRange(239, 31));
+    const auto infos = translationUnit.fullTokenInfosInRange(sourceRange(239, 31));
     ClangBackEnd::TokenInfoContainer container(infos[2]);
-    ASSERT_THAT(container.isIncludeDirectivePath(), true);
+    ASSERT_THAT(container.extraInfo().includeDirectivePath, true);
 }
 
 TEST_F(TokenInfos, HeaderNameIsInclusionWithAngleBrackets)
 {
-    const auto infos = translationUnit.tokenInfosInRange(sourceRange(289, 31));
+    const auto infos = translationUnit.fullTokenInfosInRange(sourceRange(289, 31));
     ClangBackEnd::TokenInfoContainer container(infos[2]);
-    ASSERT_THAT(container.isIncludeDirectivePath(), true);
+    ASSERT_THAT(container.extraInfo().includeDirectivePath, true);
 }
 
-
 TEST_F(TokenInfos, NotInclusion)
 {
-    const auto infos = translationUnit.tokenInfosInRange(sourceRange(241, 13));
+    const auto infos = translationUnit.fullTokenInfosInRange(sourceRange(241, 13));
     ClangBackEnd::TokenInfoContainer container(infos[1]);
-    ASSERT_THAT(container.isIncludeDirectivePath(), false);
+    ASSERT_THAT(container.extraInfo().includeDirectivePath, false);
 }
 
 TEST_F(TokenInfos, MacroIsIdentifier)
 {
-    const auto infos = translationUnit.tokenInfosInRange(sourceRange(232, 30));
+    const auto infos = translationUnit.fullTokenInfosInRange(sourceRange(232, 30));
     ClangBackEnd::TokenInfoContainer container(infos[2]);
-    ASSERT_THAT(container.isIdentifier(), true);
+    ASSERT_THAT(container.extraInfo().identifier, true);
 }
 
 TEST_F(TokenInfos, DefineIsNotIdentifier)
 {
-    const auto infos = translationUnit.tokenInfosInRange(sourceRange(232, 30));
+    const auto infos = translationUnit.fullTokenInfosInRange(sourceRange(232, 30));
     ClangBackEnd::TokenInfoContainer container(infos[1]);
-    ASSERT_THAT(container.isIncludeDirectivePath(), false);
+    ASSERT_THAT(container.extraInfo().includeDirectivePath, false);
 }
 
 TEST_F(TokenInfos, NamespaceTypeSpelling)
 {
-    const auto infos = translationUnit.tokenInfosInRange(sourceRange(592, 59));
+    const auto infos = translationUnit.fullTokenInfosInRange(sourceRange(592, 59));
     ClangBackEnd::TokenInfoContainer container(infos[10]);
-    ASSERT_THAT(container.semanticParentTypeSpelling(), Utf8StringLiteral("NFoo::NBar::NTest"));
+    ASSERT_THAT(container.extraInfo().semanticParentTypeSpelling, Utf8StringLiteral("NFoo::NBar::NTest"));
 }
 
 Data *TokenInfos::d;

tests/unit/unittest/tokeninfosreporter-test.cpp

@@ -82,7 +82,7 @@ QVector<TokenInfoContainer> generateTokenInfos(uint count)
 
     for (uint i = 0; i < count; ++i) {
         const uint line = i + 1;
-        container.append(TokenInfoContainer(line, 1, 1, HighlightingType::Type));
+        container.append(TokenInfoContainer(line, 1, 1, {HighlightingType::Type, {}}));
     }
 
     return container;
@@ -156,9 +156,9 @@ TEST_F(TokenInfosReporter, ReportSingleChunkAndRest)
 TEST_F(TokenInfosReporter, ReportCompleteLines)
 {
     QVector<TokenInfoContainer> tokenInfos {
-        TokenInfoContainer(1, 1, 1, HighlightingType::Type),
+        TokenInfoContainer(1, 1, 1, {HighlightingType::Type, {}}),
-        TokenInfoContainer(1, 2, 1, HighlightingType::Type),
+        TokenInfoContainer(1, 2, 1, {HighlightingType::Type, {}}),
-        TokenInfoContainer(2, 1, 1, HighlightingType::Type),
+        TokenInfoContainer(2, 1, 1, {HighlightingType::Type, {}}),
     };
     auto reporter = new ClangCodeModel::TokenInfosReporter(tokenInfos);
     reporter->setChunkSize(1);