Clang: extra clangbackend job to collect full token info

Limit document annotations job to only highlighting data
collection and move more expensive calls into separate job
that runs after it.

Change-Id: Ie792a3f741ac45c81033dd5b3a20ed061604f927
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@qt.io>

src/tools/clangbackend/source/clangbackendclangipc-source.pri

@@ -41,6 +41,7 @@ HEADERS += \
     $$PWD/clangtype.h \
     $$PWD/clangunsavedfilesshallowarguments.h \
     $$PWD/clangupdatedocumentannotationsjob.h \
+    $$PWD/clangupdateextradocumentannotationsjob.h \
     $$PWD/codecompleter.h \
     $$PWD/codecompletionchunkconverter.h \
     $$PWD/codecompletionsextractor.h \
@@ -50,6 +51,8 @@ HEADERS += \
     $$PWD/diagnosticset.h \
     $$PWD/diagnosticsetiterator.h \
     $$PWD/fixit.h \
+    $$PWD/fulltokeninfo.h \
+    $$PWD/fulltokeninfos.h \
     $$PWD/projectpart.h \
     $$PWD/projects.h \
     $$PWD/skippedsourceranges.h \
@@ -98,6 +101,7 @@ SOURCES += \
     $$PWD/clangtype.cpp \
     $$PWD/clangunsavedfilesshallowarguments.cpp \
     $$PWD/clangupdatedocumentannotationsjob.cpp \
+    $$PWD/clangupdateextradocumentannotationsjob.cpp \
     $$PWD/codecompleter.cpp \
     $$PWD/codecompletionchunkconverter.cpp \
     $$PWD/codecompletionsextractor.cpp \
@@ -106,6 +110,8 @@ SOURCES += \
     $$PWD/diagnostic.cpp \
     $$PWD/diagnosticset.cpp \
     $$PWD/fixit.cpp \
+    $$PWD/fulltokeninfo.cpp \
+    $$PWD/fulltokeninfos.cpp \
     $$PWD/projectpart.cpp \
     $$PWD/projects.cpp \
     $$PWD/skippedsourceranges.cpp \

src/tools/clangbackend/source/clangcodemodelserver.cpp

@@ -398,6 +398,8 @@ void ClangCodeModelServer::addAndRunUpdateJobs(std::vector<Document> documents)
         // Run the regular edit-reparse-job
         processor.addJob(JobRequest::Type::UpdateDocumentAnnotations,
                          PreferredTranslationUnit::PreviouslyParsed);
+        processor.addJob(JobRequest::Type::UpdateExtraDocumentAnnotations,
+                         PreferredTranslationUnit::PreviouslyParsed);
         processor.process();
 
         // If requested, run jobs to increase the responsiveness of the document
@@ -474,6 +476,7 @@ void ClangCodeModelServer::processInitialJobsForDocuments(const std::vector<Docu
     for (const auto &document : documents) {
         DocumentProcessor processor = documentProcessors().processor(document);
         processor.addJob(JobRequest::Type::UpdateDocumentAnnotations);
+        processor.addJob(JobRequest::Type::UpdateExtraDocumentAnnotations);
         processor.addJob(JobRequest::Type::CreateInitialDocumentPreamble);
         processor.process();
     }

src/tools/clangbackend/source/clangjobrequest.cpp

@@ -36,6 +36,7 @@
 #include "clangresumedocumentjob.h"
 #include "clangsuspenddocumentjob.h"
 #include "clangupdatedocumentannotationsjob.h"
+#include "clangupdateextradocumentannotationsjob.h"
 
 #include <clangsupport/clangcodemodelclientinterface.h>
 #include <clangsupport/cmbcodecompletedmessage.h>
@@ -57,6 +58,7 @@ static const char *JobRequestTypeToText(JobRequest::Type type)
     switch (type) {
         RETURN_TEXT_FOR_CASE(Invalid);
         RETURN_TEXT_FOR_CASE(UpdateDocumentAnnotations);
+        RETURN_TEXT_FOR_CASE(UpdateExtraDocumentAnnotations);
         RETURN_TEXT_FOR_CASE(ParseSupportiveTranslationUnit);
         RETURN_TEXT_FOR_CASE(ReparseSupportiveTranslationUnit);
         RETURN_TEXT_FOR_CASE(CreateInitialDocumentPreamble);
@@ -126,6 +128,7 @@ static JobRequest::ExpirationConditions expirationConditionsForType(JobRequest::
 
     switch (type) {
     case Type::UpdateDocumentAnnotations:
+    case Type::UpdateExtraDocumentAnnotations:
         return Conditions(Condition::AnythingChanged);
     case Type::RequestReferences:
     case Type::RequestDocumentAnnotations:
@@ -192,6 +195,7 @@ bool JobRequest::isTakeOverable() const
     // Discard these as they are initial jobs that will be recreated on demand
     // anyway.
     case Type::UpdateDocumentAnnotations:
+    case Type::UpdateExtraDocumentAnnotations:
     case Type::CreateInitialDocumentPreamble:
 
     // Discard these as they only make sense in a row. Avoid splitting them up.
@@ -223,6 +227,8 @@ IAsyncJob *JobRequest::createJob() const
         break;
     case JobRequest::Type::UpdateDocumentAnnotations:
         return new UpdateDocumentAnnotationsJob();
+    case JobRequest::Type::UpdateExtraDocumentAnnotations:
+        return new UpdateExtraDocumentAnnotationsJob();
     case JobRequest::Type::ParseSupportiveTranslationUnit:
         return new ParseSupportiveTranslationUnitJob();
     case JobRequest::Type::ReparseSupportiveTranslationUnit:
@@ -256,6 +262,7 @@ void JobRequest::cancelJob(ClangCodeModelClientInterface &client) const
     switch (type) {
     case JobRequest::Type::Invalid:
     case JobRequest::Type::UpdateDocumentAnnotations:
+    case JobRequest::Type::UpdateExtraDocumentAnnotations:
     case JobRequest::Type::ParseSupportiveTranslationUnit:
     case JobRequest::Type::ReparseSupportiveTranslationUnit:
     case JobRequest::Type::CreateInitialDocumentPreamble:

src/tools/clangbackend/source/clangjobrequest.h

@@ -50,6 +50,7 @@ public:
         Invalid,
 
         UpdateDocumentAnnotations,
+        UpdateExtraDocumentAnnotations,
         CreateInitialDocumentPreamble,
 
         ParseSupportiveTranslationUnit,

src/tools/clangbackend/source/clangtranslationunit.cpp

@@ -36,8 +36,8 @@
 #include <cursor.h>
 #include <diagnosticcontainer.h>
 #include <diagnosticset.h>
-#include <tokeninfo.h>
 #include <tokeninfos.h>
+#include <fulltokeninfos.h>
 #include <skippedsourceranges.h>
 #include <sourcelocation.h>
 #include <sourcerange.h>
@@ -140,7 +140,6 @@ void TranslationUnit::extractDocumentAnnotations(
     skippedSourceRanges = this->skippedSourceRanges().toSourceRangeContainers();
 }
 
-
 ToolTipInfo TranslationUnit::tooltip(UnsavedFiles &unsavedFiles,
                                      const Utf8String &textCodecName,
                                      uint line,
@@ -217,6 +216,21 @@ TokenInfos TranslationUnit::tokenInfosInRange(const SourceRange &range) const
     return TokenInfos(m_cxTranslationUnit, cxTokens, cxTokensCount);
 }
 
+FullTokenInfos TranslationUnit::fullTokenInfos() const
+{
+    return fullTokenInfosInRange(cursor().sourceRange());
+}
+
+FullTokenInfos TranslationUnit::fullTokenInfosInRange(const SourceRange &range) const
+{
+    CXToken *cxTokens = 0;
+    uint cxTokensCount = 0;
+
+    clang_tokenize(m_cxTranslationUnit, range, &cxTokens, &cxTokensCount);
+
+    return FullTokenInfos(m_cxTranslationUnit, cxTokens, cxTokensCount);
+}
+
 SkippedSourceRanges TranslationUnit::skippedSourceRanges() const
 {
     return SkippedSourceRanges(m_cxTranslationUnit, m_filePath.constData());

src/tools/clangbackend/source/clangtranslationunit.h

@@ -36,6 +36,7 @@ class DiagnosticContainer;
 class DiagnosticSet;
 class TokenInfoContainer;
 class TokenInfos;
+class FullTokenInfos;
 class ReferencesResult;
 class SkippedSourceRanges;
 class SourceLocation;
@@ -85,7 +86,6 @@ public:
                                     QVector<TokenInfoContainer> &tokenInfos,
                                     QVector<SourceRangeContainer> &skippedSourceRanges) const;
 
-
     ReferencesResult references(uint line, uint column, bool localReferences = false) const;
     ToolTipInfo tooltip(UnsavedFiles &unsavedFiles,
                         const Utf8String &textCodecName,
@@ -104,6 +104,9 @@ public:
     TokenInfos tokenInfos() const;
     TokenInfos tokenInfosInRange(const SourceRange &range) const;
 
+    FullTokenInfos fullTokenInfos() const;
+    FullTokenInfos fullTokenInfosInRange(const SourceRange &range) const;
+
     SkippedSourceRanges skippedSourceRanges() const;
     SourceRangeContainer followSymbol(uint line, uint column) const;
 

src/tools/clangbackend/source/clangupdateextradocumentannotationsjob.cpp

@@ -0,0 +1,61 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "clangupdateextradocumentannotationsjob.h"
+#include "fulltokeninfos.h"
+
+#include <clangsupport/clangsupportdebugutils.h>
+#include <clangsupport/clangcodemodelclientinterface.h>
+#include <clangsupport/documentannotationschangedmessage.h>
+
+#include <utils/qtcassert.h>
+
+namespace ClangBackEnd {
+
+IAsyncJob::AsyncPrepareResult UpdateExtraDocumentAnnotationsJob::prepareAsyncRun()
+{
+    const JobRequest jobRequest = context().jobRequest;
+    QTC_ASSERT(acquireDocument(), return AsyncPrepareResult());
+
+    const TranslationUnit translationUnit = *m_translationUnit;
+    setRunner([translationUnit]() {
+        TIME_SCOPE_DURATION("UpdateExtraDocumentAnnotationsJobRunner");
+        return translationUnit.fullTokenInfos().toTokenInfoContainers();
+    });
+
+    return AsyncPrepareResult{translationUnit.id()};
+}
+
+void UpdateExtraDocumentAnnotationsJob::finalizeAsyncRun()
+{
+    if (context().isOutdated())
+        return;
+
+    context().client->documentAnnotationsChanged(
+                DocumentAnnotationsChangedMessage(m_pinnedFileContainer, asyncResult()));
+}
+
+} // namespace ClangBackEnd
+

src/tools/clangbackend/source/clangupdateextradocumentannotationsjob.h

@@ -0,0 +1,43 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "clangdocumentjob.h"
+
+#include <clangsupport/tokeninfocontainer.h>
+
+namespace ClangBackEnd {
+
+using UpdateExtraDocumentAnnotationsJobResult = QVector<TokenInfoContainer>;
+
+class UpdateExtraDocumentAnnotationsJob : public DocumentJob<UpdateExtraDocumentAnnotationsJobResult>
+{
+public:
+    AsyncPrepareResult prepareAsyncRun() override;
+    void finalizeAsyncRun() override;
+};
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/fulltokeninfo.cpp

@@ -0,0 +1,153 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "clangstring.h"
+#include "cursor.h"
+#include "fulltokeninfo.h"
+
+namespace ClangBackEnd {
+
+FullTokenInfo::FullTokenInfo(const CXCursor &cxCursor,
+                             CXToken *cxToken,
+                             CXTranslationUnit cxTranslationUnit,
+                             std::vector<CXSourceRange> &currentOutputArgumentRanges)
+    : TokenInfo(cxCursor, cxToken, cxTranslationUnit, currentOutputArgumentRanges)
+{
+}
+
+FullTokenInfo::operator TokenInfoContainer() const
+{
+    return TokenInfoContainer(line(), column(), length(), types(), m_extraInfo);
+}
+
+static Utf8String fullyQualifiedType(const Cursor &cursor)
+{
+    Utf8String typeSpelling = cursor.type().canonical().utf8Spelling();
+    if (typeSpelling.isEmpty()) {
+        // Only if it's the namespaces level.
+        typeSpelling = cursor.unifiedSymbolResolution();
+        typeSpelling.replace(Utf8StringLiteral("c:@N@"), Utf8StringLiteral(""));
+        typeSpelling.replace(Utf8StringLiteral("@N@"), Utf8StringLiteral("::"));
+    }
+    return typeSpelling;
+}
+
+void FullTokenInfo::updateTypeSpelling(const Cursor &cursor, bool functionLike)
+{
+    m_extraInfo.typeSpelling = fullyQualifiedType(cursor);
+    m_extraInfo.semanticParentTypeSpelling = fullyQualifiedType(cursor.semanticParent());
+    if (!functionLike)
+        return;
+    Type type = cursor.type().canonical();
+    m_extraInfo.resultTypeSpelling = type.resultType().utf8Spelling();
+    bool hasSpaceAfterReturnType = false;
+    if (m_extraInfo.resultTypeSpelling.byteSize() < m_extraInfo.typeSpelling.byteSize()) {
+        const char *data = m_extraInfo.typeSpelling.constData();
+        hasSpaceAfterReturnType = (data[m_extraInfo.resultTypeSpelling.byteSize()] == ' ');
+    }
+    m_extraInfo.typeSpelling
+            = m_extraInfo.typeSpelling.mid(m_extraInfo.resultTypeSpelling.byteSize()
+                                         + (hasSpaceAfterReturnType ? 1 : 0));
+}
+
+void FullTokenInfo::identifierKind(const Cursor &cursor, Recursion recursion)
+{
+    updateTypeSpelling(cursor);
+
+    TokenInfo::identifierKind(cursor, recursion);
+
+    m_extraInfo.identifier = (cursor.kind() != CXCursor_PreprocessingDirective);
+}
+
+void FullTokenInfo::referencedTypeKind(const Cursor &cursor)
+{
+    updateTypeSpelling(cursor.referenced());
+
+    TokenInfo::referencedTypeKind(cursor);
+}
+
+void FullTokenInfo::functionKind(const Cursor &cursor, Recursion recursion)
+{
+    updateTypeSpelling(cursor, true);
+
+    TokenInfo::functionKind(cursor, recursion);
+
+    m_extraInfo.accessSpecifier = cursor.accessSpecifier();
+    m_extraInfo.storageClass = cursor.storageClass();
+
+    bool isSignal = false;
+    bool isSlot = false;
+    m_originalCursor.visit([&isSignal, &isSlot](CXCursor cursor, CXCursor) {
+        Cursor cur(cursor);
+        ClangString spelling = cur.spelling();
+        if (spelling == "qt_signal")
+            isSignal = true;
+        else if (spelling == "qt_slot")
+            isSlot = true;
+        return CXChildVisit_Break;
+    });
+    m_extraInfo.signal = isSignal;
+    m_extraInfo.slot = isSlot;
+}
+
+void FullTokenInfo::variableKind(const Cursor &cursor)
+{
+    TokenInfo::variableKind(cursor);
+
+    m_extraInfo.storageClass = cursor.storageClass();
+}
+
+void FullTokenInfo::fieldKind(const Cursor &cursor)
+{
+    TokenInfo::fieldKind(cursor);
+
+    m_extraInfo.accessSpecifier = cursor.accessSpecifier();
+    m_extraInfo.storageClass = cursor.storageClass();
+}
+
+void FullTokenInfo::memberReferenceKind(const Cursor &cursor)
+{
+    TokenInfo::memberReferenceKind(cursor);
+    if (cursor.isDynamicCall()) {
+        m_extraInfo.storageClass = cursor.storageClass();
+        m_extraInfo.accessSpecifier = cursor.accessSpecifier();
+    }
+}
+
+void FullTokenInfo::evaluate()
+{
+    TokenInfo::evaluate();
+
+    m_extraInfo.token = ClangString(clang_getTokenSpelling(m_cxTranslationUnit, *m_cxToken));
+
+    auto cxTokenKind = clang_getTokenKind(*m_cxToken);
+    if (cxTokenKind == CXToken_Identifier) {
+        m_extraInfo.declaration = m_originalCursor.isDeclaration();
+        m_extraInfo.definition = m_originalCursor.isDefinition();
+    }
+    m_extraInfo.includeDirectivePath = (m_originalCursor.kind() == CXCursor_InclusionDirective);
+}
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/fulltokeninfo.h

@@ -0,0 +1,55 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "tokeninfo.h"
+
+namespace ClangBackEnd {
+
+class FullTokenInfo : public TokenInfo
+{
+public:
+    FullTokenInfo(const CXCursor &cxCursor,
+                  CXToken *cxToken,
+                  CXTranslationUnit cxTranslationUnit,
+                  std::vector<CXSourceRange> &m_currentOutputArgumentRanges);
+    void evaluate() override;
+
+    operator TokenInfoContainer() const override;
+protected:
+    void identifierKind(const Cursor &cursor, Recursion recursion) override;
+    void referencedTypeKind(const Cursor &cursor) override;
+    void functionKind(const Cursor &cursor, Recursion recursion) override;
+    void variableKind(const Cursor &cursor) override;
+    void fieldKind(const Cursor &cursor) override;
+    void memberReferenceKind(const Cursor &cursor) override;
+private:
+    void updateTypeSpelling(const Cursor &cursor, bool functionLike = false);
+
+    ExtraInfo m_extraInfo;
+};
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/fulltokeninfos.cpp

@@ -0,0 +1,93 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "fulltokeninfos.h"
+
+#include <clangsupport/tokeninfocontainer.h>
+
+#include <QVector>
+
+namespace ClangBackEnd {
+
+FullTokenInfos::FullTokenInfos(CXTranslationUnit cxTranslationUnit, CXToken *tokens, uint tokensCount)
+    : cxTranslationUnit(cxTranslationUnit),
+      cxTokens(tokens),
+      cxTokenCount(tokensCount)
+{
+    cxCursors.resize(tokensCount);
+    clang_annotateTokens(cxTranslationUnit, cxTokens, cxTokenCount, cxCursors.data());
+}
+
+FullTokenInfos::~FullTokenInfos()
+{
+    clang_disposeTokens(cxTranslationUnit, cxTokens, cxTokenCount);
+}
+
+QVector<TokenInfoContainer> FullTokenInfos::toTokenInfoContainers() const
+{
+    QVector<TokenInfoContainer> containers;
+    containers.reserve(static_cast<int>(size()));
+
+    const auto isValidTokenInfo = [] (const TokenInfo &tokenInfo) {
+        // Do not exclude StringLiteral because it can be a filename for an #include
+        return !tokenInfo.hasInvalidMainType()
+                && !tokenInfo.hasMainType(HighlightingType::NumberLiteral)
+                && !tokenInfo.hasMainType(HighlightingType::Comment);
+    };
+    for (size_t index = 0; index < cxCursors.size(); ++index) {
+        FullTokenInfo fullTokenInfo = (*this)[index];
+        if (isValidTokenInfo(fullTokenInfo))
+            containers.push_back(fullTokenInfo);
+    }
+
+    return containers;
+}
+
+bool FullTokenInfos::isEmpty() const
+{
+    return cxTokenCount == 0;
+}
+
+bool FullTokenInfos::isNull() const
+{
+    return cxTokens == nullptr;
+}
+
+size_t FullTokenInfos::size() const
+{
+    return cxTokenCount;
+}
+
+FullTokenInfo FullTokenInfos::operator[](size_t index) const
+{
+    FullTokenInfo tokenInfo(cxCursors[index],
+                            cxTokens + index,
+                            cxTranslationUnit,
+                            currentOutputArgumentRanges);
+    tokenInfo.evaluate();
+    return tokenInfo;
+}
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/fulltokeninfos.h

@@ -0,0 +1,59 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "fulltokeninfo.h"
+
+#include <clang-c/Index.h>
+
+#include <vector>
+
+namespace ClangBackEnd {
+
+class FullTokenInfos
+{
+public:
+    FullTokenInfos() = default;
+    FullTokenInfos(CXTranslationUnit cxTranslationUnit, CXToken *tokens, uint tokensCount);
+    ~FullTokenInfos();
+
+    bool isEmpty() const;
+    bool isNull() const;
+    size_t size() const;
+
+    FullTokenInfo operator[](size_t index) const;
+    QVector<TokenInfoContainer> toTokenInfoContainers() const;
+
+private:
+    mutable std::vector<CXSourceRange> currentOutputArgumentRanges;
+    CXTranslationUnit cxTranslationUnit = nullptr;
+    CXToken *const cxTokens = nullptr;
+    const uint cxTokenCount = 0;
+
+    std::vector<CXCursor> cxCursors;
+};
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/tokeninfo.cpp

@@ -23,8 +23,6 @@
 **
 ****************************************************************************/
 
-#include <tokeninfocontainer.h>
-
 #include "clangstring.h"
 #include "cursor.h"
 #include "tokeninfo.h"
@@ -42,8 +40,10 @@ TokenInfo::TokenInfo(const CXCursor &cxCursor,
                      CXToken *cxToken,
                      CXTranslationUnit cxTranslationUnit,
                      std::vector<CXSourceRange> &currentOutputArgumentRanges)
-    : m_currentOutputArgumentRanges(&currentOutputArgumentRanges),
-      m_originalCursor(cxCursor)
+    : m_originalCursor(cxCursor),
+      m_cxToken(cxToken),
+      m_cxTranslationUnit(cxTranslationUnit),
+      m_currentOutputArgumentRanges(&currentOutputArgumentRanges)
 {
     const SourceRange sourceRange {cxTranslationUnit,
                                    clang_getTokenExtent(cxTranslationUnit, *cxToken)};
@@ -54,24 +54,6 @@ TokenInfo::TokenInfo(const CXCursor &cxCursor,
     m_column = start.column();
     m_offset = start.offset();
     m_length = end.offset() - start.offset();
-    collectKinds(cxTranslationUnit, cxToken, m_originalCursor);
-}
-
-TokenInfo::TokenInfo(uint line, uint column, uint length, HighlightingTypes types)
-    : m_line(line),
-      m_column(column),
-      m_length(length),
-      m_types(types)
-{
-}
-
-TokenInfo::TokenInfo(uint line, uint column, uint length, HighlightingType type)
-    : m_line(line),
-      m_column(column),
-      m_length(length),
-      m_types(HighlightingTypes())
-{
-    m_types.mainHighlightingType = type;
 }
 
 bool TokenInfo::hasInvalidMainType() const
@@ -115,11 +97,7 @@ bool TokenInfo::hasFunctionArguments() const
 
 TokenInfo::operator TokenInfoContainer() const
 {
-    return TokenInfoContainer(m_line, m_column, m_length, m_types, m_token, m_typeSpelling,
-                              m_resultTypeSpelling, m_semanticParentTypeSpelling,
-                              m_accessSpecifier, m_storageClass,
-                              m_isIdentifier, m_isInclusion,
-                              m_isDeclaration, m_isDefinition);
+    return TokenInfoContainer(m_line, m_column, m_length, m_types);
 }
 
 namespace {
@@ -146,8 +124,6 @@ bool isFunctionInFinalClass(const Cursor &cursor)
 void TokenInfo::memberReferenceKind(const Cursor &cursor)
 {
     if (cursor.isDynamicCall()) {
-        m_storageClass = cursor.storageClass();
-        m_accessSpecifier = cursor.accessSpecifier();
         if (isFinalFunction(cursor) || isFunctionInFinalClass(cursor))
             m_types.mainHighlightingType = HighlightingType::Function;
         else
@@ -173,7 +149,6 @@ void TokenInfo::overloadedDeclRefKind(const Cursor &cursor)
 
 void TokenInfo::variableKind(const Cursor &cursor)
 {
-    m_storageClass = cursor.storageClass();
     if (cursor.isLocalVariable())
         m_types.mainHighlightingType = HighlightingType::LocalVariable;
     else
@@ -183,10 +158,8 @@ void TokenInfo::variableKind(const Cursor &cursor)
         m_types.mixinHighlightingTypes.push_back(HighlightingType::OutputArgument);
 }
 
-void TokenInfo::fieldKind(const Cursor &cursor)
+void TokenInfo::fieldKind(const Cursor &)
 {
-    m_accessSpecifier = cursor.accessSpecifier();
-    m_storageClass = cursor.storageClass();
     m_types.mainHighlightingType = HighlightingType::Field;
 
     if (isOutputArgument())
@@ -283,11 +256,6 @@ void TokenInfo::filterOutPreviousOutputArguments()
 
 void TokenInfo::functionKind(const Cursor &cursor, Recursion recursion)
 {
-    updateTypeSpelling(cursor, true);
-
-    m_accessSpecifier = cursor.accessSpecifier();
-    m_storageClass = cursor.storageClass();
-
     if (isRealDynamicCall(cursor) || isVirtualMethodDeclarationOrDefinition(cursor))
         m_types.mainHighlightingType = HighlightingType::VirtualFunction;
     else
@@ -302,37 +270,9 @@ void TokenInfo::functionKind(const Cursor &cursor, Recursion recursion)
         addExtraTypeIfFirstPass(HighlightingType::FunctionDefinition, recursion);
 }
 
-Utf8String fullyQualifiedType(const Cursor &cursor)
-{
-    Utf8String typeSpelling = cursor.type().canonical().utf8Spelling();
-    if (typeSpelling.isEmpty()) {
-        // Only if it's the namespaces level.
-        typeSpelling = cursor.unifiedSymbolResolution();
-        typeSpelling.replace(Utf8StringLiteral("c:@N@"), Utf8StringLiteral(""));
-        typeSpelling.replace(Utf8StringLiteral("@N@"), Utf8StringLiteral("::"));
-    }
-    return typeSpelling;
-}
-
-void TokenInfo::updateTypeSpelling(const Cursor &cursor, bool functionLike)
-{
-    m_typeSpelling = fullyQualifiedType(cursor);
-    m_semanticParentTypeSpelling = fullyQualifiedType(cursor.semanticParent());
-    if (!functionLike)
-        return;
-    Type type = cursor.type().canonical();
-    m_resultTypeSpelling = type.resultType().utf8Spelling();
-    const bool hasSpaceAfterReturnVal
-            = m_typeSpelling.constData()[m_resultTypeSpelling.byteSize()] == ' ';
-    m_typeSpelling = m_typeSpelling.mid(m_resultTypeSpelling.byteSize()
-                                        + (hasSpaceAfterReturnVal ? 1 : 0));
-}
-
 void TokenInfo::referencedTypeKind(const Cursor &cursor)
 {
-    const Cursor ref = cursor.referenced();
-    updateTypeSpelling(ref);
-    typeKind(ref);
+    typeKind(cursor.referenced());
 }
 
 void TokenInfo::typeKind(const Cursor &cursor)
@@ -390,12 +330,7 @@ void TokenInfo::typeKind(const Cursor &cursor)
 
 void TokenInfo::identifierKind(const Cursor &cursor, Recursion recursion)
 {
-    // For now save type spelling only for declarations.
-    if (m_isDeclaration)
-        updateTypeSpelling(cursor);
-
     const CXCursorKind kind = cursor.kind();
-    m_isIdentifier = (kind != CXCursor_PreprocessingDirective);
 
     switch (kind) {
         case CXCursor_Destructor:
@@ -403,31 +338,48 @@ void TokenInfo::identifierKind(const Cursor &cursor, Recursion recursion)
         case CXCursor_FunctionDecl:
         case CXCursor_FunctionTemplate:
         case CXCursor_CallExpr:
-        case CXCursor_CXXMethod:                 functionKind(cursor, recursion); break;
-        case CXCursor_NonTypeTemplateParameter:  m_types.mainHighlightingType = HighlightingType::LocalVariable; break;
+        case CXCursor_CXXMethod:
+            functionKind(cursor, recursion);
+            break;
+        case CXCursor_NonTypeTemplateParameter:
+            m_types.mainHighlightingType = HighlightingType::LocalVariable;
+            break;
         case CXCursor_ParmDecl:
-        case CXCursor_VarDecl:                   variableKind(cursor); break;
-        case CXCursor_DeclRefExpr:               identifierKind(cursor.referenced(), Recursion::RecursivePass); break;
-        case CXCursor_MemberRefExpr:             memberReferenceKind(cursor); break;
+        case CXCursor_VarDecl:
+            variableKind(cursor);
+            break;
+        case CXCursor_DeclRefExpr:
+            identifierKind(cursor.referenced(), Recursion::RecursivePass);
+            break;
+        case CXCursor_MemberRefExpr:
+            memberReferenceKind(cursor);
+            break;
         case CXCursor_FieldDecl:
-        case CXCursor_MemberRef:                 fieldKind(cursor); break;
+        case CXCursor_MemberRef:
+            fieldKind(cursor);
+            break;
         case CXCursor_ObjCIvarDecl:
         case CXCursor_ObjCPropertyDecl:
         case CXCursor_ObjCClassMethodDecl:
         case CXCursor_ObjCInstanceMethodDecl:
         case CXCursor_ObjCSynthesizeDecl:
-        case CXCursor_ObjCDynamicDecl:           m_types.mainHighlightingType = HighlightingType::Field; break;
+        case CXCursor_ObjCDynamicDecl:
+            m_types.mainHighlightingType = HighlightingType::Field;
+            break;
         case CXCursor_TemplateRef:
         case CXCursor_NamespaceRef:
-        case CXCursor_TypeRef:                   referencedTypeKind(cursor); break;
+        case CXCursor_TypeRef:
+            referencedTypeKind(cursor);
+            break;
         case CXCursor_ClassDecl:
         case CXCursor_ClassTemplate:
         case CXCursor_ClassTemplatePartialSpecialization:
         case CXCursor_UnionDecl:
         case CXCursor_StructDecl:
         case CXCursor_EnumDecl:
-        case CXCursor_Namespace:                 m_types.mixinHighlightingTypes.push_back(HighlightingType::Declaration);
-        Q_FALLTHROUGH();
+        case CXCursor_Namespace:
+            m_types.mixinHighlightingTypes.push_back(HighlightingType::Declaration);
+            Q_FALLTHROUGH();
         case CXCursor_TemplateTypeParameter:
         case CXCursor_TemplateTemplateParameter:
         case CXCursor_NamespaceAlias:
@@ -442,16 +394,33 @@ void TokenInfo::identifierKind(const Cursor &cursor, Recursion recursion)
         case CXCursor_ObjCProtocolDecl:
         case CXCursor_ObjCProtocolRef:
         case CXCursor_ObjCClassRef:
-        case CXCursor_ObjCSuperClassRef:         typeKind(cursor); break;
-        case CXCursor_OverloadedDeclRef:         overloadedDeclRefKind(cursor); break;
-        case CXCursor_EnumConstantDecl:          m_types.mainHighlightingType = HighlightingType::Enumeration; break;
-        case CXCursor_PreprocessingDirective:    m_types.mainHighlightingType = HighlightingType::Preprocessor; break;
-        case CXCursor_MacroExpansion:            m_types.mainHighlightingType = HighlightingType::PreprocessorExpansion; break;
-        case CXCursor_MacroDefinition:           m_types.mainHighlightingType = HighlightingType::PreprocessorDefinition; break;
-        case CXCursor_InclusionDirective:        m_types.mainHighlightingType = HighlightingType::StringLiteral; break;
+        case CXCursor_ObjCSuperClassRef:
+            typeKind(cursor);
+            break;
+        case CXCursor_OverloadedDeclRef:
+            overloadedDeclRefKind(cursor);
+            break;
+        case CXCursor_EnumConstantDecl:
+            m_types.mainHighlightingType = HighlightingType::Enumeration;
+            break;
+        case CXCursor_PreprocessingDirective:
+            m_types.mainHighlightingType = HighlightingType::Preprocessor;
+            break;
+        case CXCursor_MacroExpansion:
+            m_types.mainHighlightingType = HighlightingType::PreprocessorExpansion;
+            break;
+        case CXCursor_MacroDefinition:
+            m_types.mainHighlightingType = HighlightingType::PreprocessorDefinition;
+            break;
+        case CXCursor_InclusionDirective:
+            m_types.mainHighlightingType = HighlightingType::StringLiteral;
+            break;
         case CXCursor_LabelRef:
-        case CXCursor_LabelStmt:                 m_types.mainHighlightingType = HighlightingType::Label; break;
-        default:                                 break;
+        case CXCursor_LabelStmt:
+            m_types.mainHighlightingType = HighlightingType::Label;
+            break;
+        default:
+            break;
     }
 }
 
@@ -462,11 +431,14 @@ HighlightingType literalKind(const Cursor &cursor)
         case CXCursor_CharacterLiteral:
         case CXCursor_StringLiteral:
         case CXCursor_InclusionDirective:
-        case CXCursor_ObjCStringLiteral: return HighlightingType::StringLiteral;
+        case CXCursor_ObjCStringLiteral:
+            return HighlightingType::StringLiteral;
         case CXCursor_IntegerLiteral:
         case CXCursor_ImaginaryLiteral:
-        case CXCursor_FloatingLiteral:   return HighlightingType::NumberLiteral;
-        default:                         return HighlightingType::Invalid;
+        case CXCursor_FloatingLiteral:
+            return HighlightingType::NumberLiteral;
+        default:
+            return HighlightingType::Invalid;
     }
 
     Q_UNREACHABLE();
@@ -534,28 +506,31 @@ static HighlightingType highlightingTypeForKeyword(CXTranslationUnit cxTranslati
     return HighlightingType::Keyword;
 }
 
-void TokenInfo::collectKinds(CXTranslationUnit cxTranslationUnit,
-                                    CXToken *cxToken, const Cursor &cursor)
+void TokenInfo::evaluate()
 {
-    auto cxTokenKind = clang_getTokenKind(*cxToken);
+    auto cxTokenKind = clang_getTokenKind(*m_cxToken);
 
     m_types = HighlightingTypes();
-    m_token = ClangString(clang_getTokenSpelling(cxTranslationUnit, *cxToken));
-
-    if (cxTokenKind == CXToken_Identifier) {
-        m_isDeclaration = cursor.isDeclaration();
-        m_isDefinition = cursor.isDefinition();
-    }
 
     switch (cxTokenKind) {
-        case CXToken_Keyword:     m_types.mainHighlightingType = highlightingTypeForKeyword(cxTranslationUnit, cxToken, m_originalCursor); break;
-        case CXToken_Punctuation: m_types.mainHighlightingType = punctuationKind(cursor); break;
-        case CXToken_Identifier:  identifierKind(cursor, Recursion::FirstPass); break;
-        case CXToken_Comment:     m_types.mainHighlightingType = HighlightingType::Comment; break;
-        case CXToken_Literal:     m_types.mainHighlightingType = literalKind(cursor); break;
+        case CXToken_Keyword:
+            m_types.mainHighlightingType = highlightingTypeForKeyword(m_cxTranslationUnit,
+                                                                      m_cxToken,
+                                                                      m_originalCursor);
+            break;
+        case CXToken_Punctuation:
+            m_types.mainHighlightingType = punctuationKind(m_originalCursor);
+            break;
+        case CXToken_Identifier:
+            identifierKind(m_originalCursor, Recursion::FirstPass);
+            break;
+        case CXToken_Comment:
+            m_types.mainHighlightingType = HighlightingType::Comment;
+            break;
+        case CXToken_Literal:
+            m_types.mainHighlightingType = literalKind(m_originalCursor);
+            break;
     }
-
-    m_isInclusion = (cursor.kind() == CXCursor_InclusionDirective);
 }
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/tokeninfo.h

@@ -27,7 +27,8 @@
 
 #include "clangsupport_global.h"
 #include "cursor.h"
-#include "tokeninfocontainer.h"
+
+#include <clangsupport/tokeninfocontainer.h>
 
 #include <sqlite/utf8string.h>
 
@@ -39,18 +40,14 @@ class TokenInfo
 {
     friend bool operator==(const TokenInfo &first, const TokenInfo &second);
 
-    enum class Recursion {
-        FirstPass,
-        RecursivePass
-    };
-
 public:
     TokenInfo(const CXCursor &cxCursor,
               CXToken *cxToken,
               CXTranslationUnit cxTranslationUnit,
               std::vector<CXSourceRange> &m_currentOutputArgumentRanges);
-    TokenInfo(uint m_line, uint m_column, uint m_length, HighlightingTypes m_types);
-    TokenInfo(uint m_line, uint m_column, uint m_length, HighlightingType type);
+    virtual ~TokenInfo() = default;
+
+    virtual void evaluate();
 
     bool hasInvalidMainType() const;
     bool hasMainType(HighlightingType type) const;
@@ -60,7 +57,7 @@ public:
     bool hasOnlyType(HighlightingType type) const;
     bool hasFunctionArguments() const;
 
-    operator TokenInfoContainer() const;
+    virtual operator TokenInfoContainer() const;
 
     const HighlightingTypes &types() const
     {
@@ -82,45 +79,42 @@ public:
         return m_length;
     }
 
+protected:
+    enum class Recursion {
+        FirstPass,
+        RecursivePass
+    };
+
+    virtual void identifierKind(const Cursor &cursor, Recursion recursion);
+    virtual void referencedTypeKind(const Cursor &cursor);
+    virtual void overloadedDeclRefKind(const Cursor &cursor);
+    virtual void variableKind(const Cursor &cursor);
+    virtual void fieldKind(const Cursor &cursor);
+    virtual void functionKind(const Cursor &cursor, Recursion recursion);
+    virtual void memberReferenceKind(const Cursor &cursor);
+    virtual HighlightingType punctuationKind(const Cursor &cursor);
+    virtual void typeKind(const Cursor &cursor);
+
+    Cursor m_originalCursor;
+    CXToken *m_cxToken;
+    CXTranslationUnit m_cxTranslationUnit;
+
 private:
-    void identifierKind(const Cursor &cursor, Recursion recursion);
-    void referencedTypeKind(const Cursor &cursor);
-    void overloadedDeclRefKind(const Cursor &cursor);
-    void variableKind(const Cursor &cursor);
-    void fieldKind(const Cursor &cursor);
     bool isVirtualMethodDeclarationOrDefinition(const Cursor &cursor) const;
     bool isDefinition() const;
-    void functionKind(const Cursor &cursor, Recursion recursion);
-    void memberReferenceKind(const Cursor &cursor);
-    HighlightingType punctuationKind(const Cursor &cursor);
-    void typeKind(const Cursor &cursor);
-    void collectKinds(CXTranslationUnit cxTranslationUnit, CXToken *cxToken, const Cursor &cursor);
     bool isRealDynamicCall(const Cursor &cursor) const;
     void addExtraTypeIfFirstPass(HighlightingType type, Recursion recursion);
     bool isOutputArgument() const;
     void collectOutputArguments(const Cursor &cursor);
     void filterOutPreviousOutputArguments();
     bool isArgumentInCurrentOutputArgumentLocations() const;
-    void updateTypeSpelling(const Cursor &cursor, bool functionLike = false);
 
-private:
     std::vector<CXSourceRange> *m_currentOutputArgumentRanges = nullptr;
-    Cursor m_originalCursor;
     uint m_line;
     uint m_column;
     uint m_length;
     uint m_offset = 0;
     HighlightingTypes m_types;
-    Utf8String m_token;
-    Utf8String m_typeSpelling;
-    Utf8String m_resultTypeSpelling;
-    Utf8String m_semanticParentTypeSpelling;
-    AccessSpecifier m_accessSpecifier = AccessSpecifier::Invalid;
-    StorageClass m_storageClass = StorageClass::Invalid;
-    bool m_isIdentifier = false;
-    bool m_isInclusion = false;
-    bool m_isDeclaration = false;
-    bool m_isDefinition = false;
 };
 
 

src/tools/clangbackend/source/tokeninfos.cpp

@@ -33,30 +33,30 @@ namespace ClangBackEnd {
 
 TokenInfos::TokenInfos(CXTranslationUnit cxTranslationUnit, CXToken *tokens, uint tokensCount)
     : cxTranslationUnit(cxTranslationUnit),
-      cxToken(tokens),
+      cxTokens(tokens),
       cxTokenCount(tokensCount)
 {
-    cxCursor.resize(tokensCount);
-    clang_annotateTokens(cxTranslationUnit, cxToken, cxTokenCount, cxCursor.data());
+    cxCursors.resize(tokensCount);
+    clang_annotateTokens(cxTranslationUnit, cxTokens, cxTokenCount, cxCursors.data());
 }
 
 TokenInfos::~TokenInfos()
 {
-    clang_disposeTokens(cxTranslationUnit, cxToken, cxTokenCount);
+    clang_disposeTokens(cxTranslationUnit, cxTokens, cxTokenCount);
 }
 
 TokenInfos::const_iterator TokenInfos::begin() const
 {
-    return const_iterator(cxCursor.cbegin(),
-                          cxToken,
+    return const_iterator(cxCursors.cbegin(),
+                          cxTokens,
                           cxTranslationUnit,
                           currentOutputArgumentRanges);
 }
 
 TokenInfos::const_iterator TokenInfos::end() const
 {
-    return const_iterator(cxCursor.cend(),
-                          cxToken + cxTokenCount,
+    return const_iterator(cxCursors.cend(),
+                          cxTokens + cxTokenCount,
                           cxTranslationUnit,
                           currentOutputArgumentRanges);
 }
@@ -90,7 +90,7 @@ bool TokenInfos::isEmpty() const
 
 bool ClangBackEnd::TokenInfos::isNull() const
 {
-    return cxToken == nullptr;
+    return cxTokens == nullptr;
 }
 
 uint TokenInfos::size() const
@@ -100,10 +100,12 @@ uint TokenInfos::size() const
 
 TokenInfo TokenInfos::operator[](size_t index) const
 {
-    return TokenInfo(cxCursor[index],
-                            cxToken + index,
-                            cxTranslationUnit,
-                            currentOutputArgumentRanges);
+    TokenInfo tokenInfo(cxCursors[index],
+                        cxTokens + index,
+                        cxTranslationUnit,
+                        currentOutputArgumentRanges);
+    tokenInfo.evaluate();
+    return tokenInfo;
 }
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/tokeninfos.h

@@ -63,10 +63,10 @@ public:
 private:
     mutable std::vector<CXSourceRange> currentOutputArgumentRanges;
     CXTranslationUnit cxTranslationUnit = nullptr;
-    CXToken *const cxToken = nullptr;
+    CXToken *const cxTokens = nullptr;
     const uint cxTokenCount = 0;
 
-    std::vector<CXCursor> cxCursor;
+    std::vector<CXCursor> cxCursors;
 };
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/tokeninfositerator.h

@@ -63,9 +63,9 @@ public:
     TokenInfosIterator operator++(int)
     {
         return TokenInfosIterator(cxCursorIterator++,
-                                         cxToken++,
-                                         cxTranslationUnit,
-                                         currentOutputArgumentRanges);
+                                  cxToken++,
+                                  cxTranslationUnit,
+                                  currentOutputArgumentRanges);
     }
 
     bool operator==(TokenInfosIterator other) const
@@ -80,10 +80,12 @@ public:
 
     TokenInfo operator*()
     {
-        return TokenInfo(*cxCursorIterator,
-                                cxToken,
-                                cxTranslationUnit,
-                                currentOutputArgumentRanges);
+        TokenInfo tokenInfo(*cxCursorIterator,
+                            cxToken,
+                            cxTranslationUnit,
+                            currentOutputArgumentRanges);
+        tokenInfo.evaluate();
+        return tokenInfo;
     }
 
 private: