Clang: Introduce Token class to work with CXToken pointers

Make tokenization and tokens annotation simpler.

Task-number: QTCREATORBUG-21143
Task-number: QTCREATORBUG-21144
Change-Id: I580091b7b63dd973228fd2e21cf2e74c7d0e7df2
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@qt.io>

src/tools/clangbackend/source/clangbackendclangipc-source.pri

@@ -53,6 +53,7 @@ HEADERS += \
     $$PWD/skippedsourceranges.h \
     $$PWD/sourcelocation.h \
     $$PWD/sourcerange.h \
+    $$PWD/token.h \
     $$PWD/tokeninfo.h \
     $$PWD/tokenprocessor.h \
     $$PWD/tokenprocessoriterator.h \
@@ -107,6 +108,7 @@ SOURCES += \
     $$PWD/skippedsourceranges.cpp \
     $$PWD/sourcelocation.cpp \
     $$PWD/sourcerange.cpp \
+    $$PWD/token.cpp \
     $$PWD/tokeninfo.cpp \
     $$PWD/unsavedfile.cpp \
     $$PWD/unsavedfiles.cpp \

src/tools/clangbackend/source/clangfollowsymbol.cpp

@@ -29,6 +29,7 @@
 #include "cursor.h"
 #include "clangstring.h"
 #include "sourcerange.h"
+#include "token.h"
 #include "clangsupportdebugutils.h"
 
 #include <utils/qtcassert.h>
@@ -37,65 +38,35 @@
 
 namespace ClangBackEnd {
 
-namespace {
-
-struct Tokens
+static SourceRange getOperatorRange(const Tokens &tokens,
+                                    std::vector<Token>::const_iterator currentToken)
 {
-    Tokens(const Tokens &) = delete;
-    Tokens(const Cursor &cursor) {
-        tu = cursor.cxTranslationUnit();
-        clang_tokenize(tu, cursor.cxSourceRange(), &data, &tokenCount);
-    }
-    Tokens(const CXTranslationUnit &tu) {
-        const CXSourceRange range
-                = clang_getCursorExtent(clang_getTranslationUnitCursor(tu));
-        clang_tokenize(tu, range, &data, &tokenCount);
-    }
-    ~Tokens() {
-        clang_disposeTokens(tu, data, tokenCount);
-    }
+    const SourceLocation start = currentToken->location();
+    currentToken += 2;
+    while (currentToken != tokens.cend() && !(currentToken->spelling() == "("))
+        ++currentToken;
 
-    CXToken *data = nullptr;
-    uint tokenCount = 0;
-private:
-    CXTranslationUnit tu;
-};
-
-} // anonymous namespace
-
-static SourceRange getOperatorRange(const CXTranslationUnit tu,
-                                    const Tokens &tokens,
-                                    uint operatorIndex)
-{
-    const CXSourceLocation start = clang_getTokenLocation(tu, tokens.data[operatorIndex]);
-    operatorIndex += 2;
-    while (operatorIndex < tokens.tokenCount
-           && !(ClangString(clang_getTokenSpelling(tu, tokens.data[operatorIndex])) == "(")) {
-        ++operatorIndex;
-    }
-    const CXSourceLocation end = clang_getTokenLocation(tu, tokens.data[operatorIndex]);
-    return SourceRange(tu, clang_getRange(start, end));
+    return SourceRange(start, currentToken->location());
 }
 
 static SourceRangeContainer extractMatchingTokenRange(const Cursor &cursor,
                                                       const Utf8String &tokenStr)
 {
-    Tokens tokens(cursor);
-    const CXTranslationUnit tu = cursor.cxTranslationUnit();
-    for (uint i = 0; i < tokens.tokenCount; ++i) {
-        if (!(tokenStr == ClangString(clang_getTokenSpelling(tu, tokens.data[i]))))
+    Tokens tokens(cursor.sourceRange());
+    for (auto it = tokens.cbegin(); it != tokens.cend(); ++it) {
+        const Token &currentToken = *it;
+        if (!(tokenStr == currentToken.spelling()))
             continue;
 
         if (cursor.isFunctionLike() || cursor.isConstructorOrDestructor()) {
             if (tokenStr == "operator")
-                return getOperatorRange(tu, tokens, i);
+                return getOperatorRange(tokens, it);
 
-            if (i+1 > tokens.tokenCount
-                    || !(ClangString(clang_getTokenSpelling(tu, tokens.data[i+1])) == "(")) {
+            auto nextIt = it + 1;
+            if (nextIt == tokens.cend() || !(nextIt->spelling() == "("))
                 continue;
         }
-        }
-        return SourceRange(tu, clang_getTokenExtent(tu, tokens.data[i]));
+        return currentToken.extent();
     }
     return SourceRangeContainer();
 }
@@ -103,8 +74,8 @@ static SourceRangeContainer extractMatchingTokenRange(const Cursor &cursor,
 static int getTokenIndex(CXTranslationUnit tu, const Tokens &tokens, uint line, uint column)
 {
     int tokenIndex = -1;
-    for (int i = static_cast<int>(tokens.tokenCount - 1); i >= 0; --i) {
-        const SourceRange range(tu, clang_getTokenExtent(tu, tokens.data[i]));
+    for (int i = static_cast<int>(tokens.size() - 1); i >= 0; --i) {
+        const SourceRange range(tu, tokens[i].extent());
         if (range.contains(line, column)) {
             tokenIndex = i;
             break;
@@ -118,28 +89,28 @@ FollowSymbolResult FollowSymbol::followSymbol(CXTranslationUnit tu,
                                               uint line,
                                               uint column)
 {
-    std::unique_ptr<Tokens> tokens(new Tokens(fullCursor));
+    Tokens tokens(fullCursor.sourceRange());
 
-    if (!tokens->tokenCount)
-        tokens.reset(new Tokens(tu));
+    if (!tokens.size()) {
+        const Cursor tuCursor(clang_getTranslationUnitCursor(tu));
+        tokens = Tokens(tuCursor.sourceRange());
+    }
 
-    if (!tokens->tokenCount)
+    if (!tokens.size())
         return SourceRangeContainer();
 
-    QVector<CXCursor> cursors(static_cast<int>(tokens->tokenCount));
-    clang_annotateTokens(tu, tokens->data, tokens->tokenCount, cursors.data());
-    int tokenIndex = getTokenIndex(tu, *tokens, line, column);
+    std::vector<Cursor> cursors = tokens.annotate();
+    int tokenIndex = getTokenIndex(tu, tokens, line, column);
     QTC_ASSERT(tokenIndex >= 0, return SourceRangeContainer());
 
-    const Utf8String tokenSpelling = ClangString(
-                clang_getTokenSpelling(tu, tokens->data[tokenIndex]));
+    const Utf8String tokenSpelling = tokens[tokenIndex].spelling();
     if (tokenSpelling.isEmpty())
         return SourceRangeContainer();
 
     Cursor cursor{cursors[tokenIndex]};
 
     if (cursor.kind() == CXCursor_InclusionDirective) {
-        CXFile file = clang_getIncludedFile(cursors[tokenIndex]);
+        CXFile file = clang_getIncludedFile(cursors[tokenIndex].cx());
         const ClangString filename(clang_getFileName(file));
         const SourceLocation loc(tu, filename, 1, 1);
         FollowSymbolResult result;

src/tools/clangbackend/source/clangreferencescollector.cpp

@@ -28,6 +28,7 @@
 #include "clangstring.h"
 #include "cursor.h"
 #include "sourcerange.h"
+#include "token.h"
 
 #include <clangsupport/sourcerangecontainer.h>
 #include <utils/qtcassert.h>
@@ -119,52 +120,32 @@ class ReferencesCollector
 {
 public:
     ReferencesCollector(CXTranslationUnit cxTranslationUnit);
-    ~ReferencesCollector();
 
     ReferencesResult collect(uint line, uint column, bool localReferences = false) const;
 
 private:
-    bool isWithinTokenRange(CXToken token, uint line, uint column) const;
     bool pointsToIdentifier(uint line, uint column, unsigned *tokenIndex) const;
-    bool matchesIdentifier(const CXToken &token, const Utf8String &identifier) const;
+    bool matchesIdentifier(const Token &token, const Utf8String &identifier) const;
     bool checkToken(unsigned index, const Utf8String &identifier, const Utf8String &usr) const;
 
 private:
     CXTranslationUnit m_cxTranslationUnit = nullptr;
-    CXToken *m_cxTokens = nullptr;
-    uint m_cxTokenCount = 0;
-
-    QVector<CXCursor> m_cxCursors;
+    Tokens m_tokens;
+    std::vector<Cursor> m_cursors;
 };
 
 ReferencesCollector::ReferencesCollector(CXTranslationUnit cxTranslationUnit)
     : m_cxTranslationUnit(cxTranslationUnit)
+    , m_tokens(Cursor(clang_getTranslationUnitCursor(m_cxTranslationUnit)).sourceRange())
+    , m_cursors(m_tokens.annotate())
 {
-    const CXSourceRange range
-            = clang_getCursorExtent(clang_getTranslationUnitCursor(m_cxTranslationUnit));
-    clang_tokenize(cxTranslationUnit, range, &m_cxTokens, &m_cxTokenCount);
-
-    m_cxCursors.resize(static_cast<int>(m_cxTokenCount));
-    clang_annotateTokens(cxTranslationUnit, m_cxTokens, m_cxTokenCount, m_cxCursors.data());
-}
-
-ReferencesCollector::~ReferencesCollector()
-{
-    clang_disposeTokens(m_cxTranslationUnit, m_cxTokens, m_cxTokenCount);
-}
-
-bool ReferencesCollector::isWithinTokenRange(CXToken token, uint line, uint column) const
-{
-    const SourceRange range {m_cxTranslationUnit, clang_getTokenExtent(m_cxTranslationUnit, token)};
-    return range.contains(line, column);
 }
 
 bool ReferencesCollector::pointsToIdentifier(uint line, uint column, unsigned *tokenIndex) const
 {
-    for (uint i = 0; i < m_cxTokenCount; ++i) {
-        const CXToken token = m_cxTokens[i];
-        if (clang_getTokenKind(token) == CXToken_Identifier
-                && isWithinTokenRange(token, line, column)) {
+    for (uint i = 0; i < m_tokens.size(); ++i) {
+        const Token &token = m_tokens[i];
+        if (token.kind() == CXToken_Identifier && token.extent().contains(line, column)) {
             *tokenIndex = i;
             return true;
         }
@@ -173,15 +154,12 @@ bool ReferencesCollector::pointsToIdentifier(uint line, uint column, unsigned *t
     return false;
 }
 
-bool ReferencesCollector::matchesIdentifier(const CXToken &token,
+bool ReferencesCollector::matchesIdentifier(const Token &token,
                                             const Utf8String &identifier) const
 {
-    const CXTokenKind tokenKind = clang_getTokenKind(token);
-    if (tokenKind == CXToken_Identifier) {
-        const Utf8String candidateIdentifier
-                = ClangString(clang_getTokenSpelling(m_cxTranslationUnit, token));
-        return candidateIdentifier == identifier;
-    }
+    const CXTokenKind tokenKind = token.kind();
+    if (tokenKind == CXToken_Identifier)
+        return token.spelling() == identifier;
 
     return false;
 }
@@ -189,7 +167,7 @@ bool ReferencesCollector::matchesIdentifier(const CXToken &token,
 bool ReferencesCollector::checkToken(unsigned index, const Utf8String &identifier,
                                      const Utf8String &usr) const
 {
-    const CXToken token = m_cxTokens[index];
+    const Token &token = m_tokens[index];
     if (!matchesIdentifier(token, identifier))
         return false;
 
@@ -201,8 +179,7 @@ bool ReferencesCollector::checkToken(unsigned index, const Utf8String &identifie
 //        qWarning() << "ReferencesCollector::checkToken:" << line << spelling;
     }
 
-    const Cursor currentCursor(m_cxCursors[static_cast<int>(index)]);
-    const ReferencedCursor candidate = ReferencedCursor::find(currentCursor);
+    const ReferencedCursor candidate = ReferencedCursor::find(m_cursors[index]);
 
     return candidate.usr() == usr;
 }
@@ -215,9 +192,7 @@ ReferencesResult ReferencesCollector::collect(uint line, uint column, bool local
     if (!pointsToIdentifier(line, column, &index))
         return result;
 
-    const Cursor cursorFromUser = m_cxCursors[static_cast<int>(index)];
-
-    const ReferencedCursor refCursor = ReferencedCursor::find(cursorFromUser);
+    const ReferencedCursor refCursor = ReferencedCursor::find(m_cursors[index]);
     const Utf8String usr = refCursor.usr();
     if (usr.isEmpty())
         return result;
@@ -225,14 +200,11 @@ ReferencesResult ReferencesCollector::collect(uint line, uint column, bool local
     if (localReferences && !refCursor.isLocalVariable())
         return result;
 
-    const CXToken token = m_cxTokens[index];
-    const Utf8String identifier = ClangString(clang_getTokenSpelling(m_cxTranslationUnit, token));
-    for (uint i = 0; i < m_cxTokenCount; ++i) {
-        if (checkToken(i, identifier, usr)) {
-            const SourceRange range {m_cxTranslationUnit,
-                                     clang_getTokenExtent(m_cxTranslationUnit, m_cxTokens[i])};
-            result.references.append(range);
-        }
+    const Token &token = m_tokens[index];
+    const Utf8String identifier = token.spelling();
+    for (uint i = 0; i < m_tokens.size(); ++i) {
+        if (checkToken(i, identifier, usr))
+            result.references.append(m_tokens[i].extent());
     }
 
     result.isLocalVariable = refCursor.isLocalVariable();

src/tools/clangbackend/source/clangtooltipinfocollector.cpp

@@ -29,6 +29,7 @@
 #include "clangstring.h"
 #include "cursor.h"
 #include "sourcerange.h"
+#include "token.h"
 #include "unsavedfiles.h"
 #include "unsavedfile.h"
 
@@ -275,17 +276,12 @@ Utf8String ToolTipInfoCollector::textForNamespaceAlias(const Cursor &cursor) con
 {
     // TODO: Add some libclang API to get the aliased name straight away.
 
-    CXToken *cxTokens = nullptr;
-    uint cxTokenCount = 0;
-
-    clang_tokenize(m_cxTranslationUnit, cursor.cxSourceRange(), &cxTokens, &cxTokenCount);
+    const Tokens tokens(cursor.sourceRange());
 
     Utf8String aliasedName;
     // Start at 3 in order to skip these tokens: namespace X =
-    for (uint i = 3; i < cxTokenCount; ++i)
-        aliasedName += ClangString(clang_getTokenSpelling(m_cxTranslationUnit, cxTokens[i]));
-
-    clang_disposeTokens(m_cxTranslationUnit, cxTokens, cxTokenCount);
+    for (uint i = 3; i < tokens.size(); ++i)
+        aliasedName += tokens[i].spelling();
 
     return aliasedName;
 }

src/tools/clangbackend/source/clangtranslationunit.cpp

@@ -197,7 +197,7 @@ TokenProcessor<TokenInfo> TranslationUnit::tokenInfos() const
 
 TokenProcessor<TokenInfo> TranslationUnit::tokenInfosInRange(const SourceRange &range) const
 {
-    return TokenProcessor<TokenInfo>(m_cxTranslationUnit, range);
+    return TokenProcessor<TokenInfo>(range);
 }
 
 TokenProcessor<FullTokenInfo> TranslationUnit::fullTokenInfos() const
@@ -207,7 +207,7 @@ TokenProcessor<FullTokenInfo> TranslationUnit::fullTokenInfos() const
 
 TokenProcessor<FullTokenInfo> TranslationUnit::fullTokenInfosInRange(const SourceRange &range) const
 {
-    return TokenProcessor<FullTokenInfo>(m_cxTranslationUnit, range);
+    return TokenProcessor<FullTokenInfo>(range);
 }
 
 SkippedSourceRanges TranslationUnit::skippedSourceRanges() const

src/tools/clangbackend/source/cursor.h

@@ -35,8 +35,6 @@
 
 #include <vector>
 
-class Utf8String;
-
 namespace ClangBackEnd {
 
 class SourceLocation;

src/tools/clangbackend/source/fulltokeninfo.cpp

@@ -28,17 +28,17 @@
 #include "cursor.h"
 #include "fulltokeninfo.h"
 #include "sourcerange.h"
+#include "token.h"
 #include "tokenprocessor.h"
 
 #include <utils/predicates.h>
 
 namespace ClangBackEnd {
 
-FullTokenInfo::FullTokenInfo(const CXCursor &cxCursor,
-                             CXToken *cxToken,
-                             CXTranslationUnit cxTranslationUnit,
+FullTokenInfo::FullTokenInfo(const Cursor &cursor,
+                             const Token *token,
                              std::vector<CXSourceRange> &currentOutputArgumentRanges)
-    : TokenInfo(cxCursor, cxToken, cxTranslationUnit, currentOutputArgumentRanges)
+    : TokenInfo(cursor, token, currentOutputArgumentRanges)
 {
 }
 
@@ -101,15 +101,13 @@ static Utf8String propertyParentSpelling(CXTranslationUnit cxTranslationUnit,
     return parentSpelling;
 }
 
-static Utf8String getPropertyType(const CXSourceLocation &cxLocation,
-                                  CXTranslationUnit cxTranslationUnit,
-                                  uint propertyPosition)
+static Utf8String getPropertyType(const SourceLocation &location, uint propertyPosition)
 {
     // Extract property type from the source code
     CXFile cxFile;
     uint offset;
-    clang_getFileLocation(cxLocation, &cxFile, nullptr, nullptr, &offset);
-    const char *const contents = clang_getFileContents(cxTranslationUnit, cxFile, nullptr);
+    clang_getFileLocation(location.cx(), &cxFile, nullptr, nullptr, &offset);
+    const char *const contents = clang_getFileContents(location.tu(), cxFile, nullptr);
     const char *const lineContents = &contents[offset - propertyPosition];
 
     const char *typeStart = std::strstr(lineContents, "Q_PROPERTY") + 10;
@@ -125,18 +123,15 @@ static Utf8String getPropertyType(const CXSourceLocation &cxLocation,
 
 void FullTokenInfo::updatePropertyData()
 {
-    CXSourceRange cxRange(clang_getTokenExtent(m_cxTranslationUnit, *m_cxToken));
-    const SourceRange range(m_cxTranslationUnit, cxRange);
-    m_extraInfo.semanticParentTypeSpelling = propertyParentSpelling(m_cxTranslationUnit,
+    const SourceRange range = m_token->extent();
+    m_extraInfo.semanticParentTypeSpelling = propertyParentSpelling(m_token->tu(),
                                                                     range.start().filePath(),
                                                                     line(),
                                                                     column());
     m_extraInfo.cursorRange = range;
     m_extraInfo.declaration = true;
     m_extraInfo.definition = true;
-    m_extraInfo.typeSpelling = getPropertyType(clang_getRangeStart(cxRange),
-                                               m_cxTranslationUnit,
-                                               column() - 1);
+    m_extraInfo.typeSpelling = getPropertyType(range.start(), column() - 1);
 }
 
 void FullTokenInfo::identifierKind(const Cursor &cursor, Recursion recursion)
@@ -263,9 +258,8 @@ void FullTokenInfo::overloadedOperatorKind()
 
 void FullTokenInfo::evaluate()
 {
-    m_extraInfo.token = ClangString(clang_getTokenSpelling(m_cxTranslationUnit, *m_cxToken));
-
-    auto cxTokenKind = clang_getTokenKind(*m_cxToken);
+    m_extraInfo.token = m_token->spelling();
+    CXTokenKind cxTokenKind = m_token->kind();
     if (cxTokenKind == CXToken_Identifier) {
         m_extraInfo.declaration = m_originalCursor.isDeclaration();
         m_extraInfo.definition = m_originalCursor.isDefinition();

src/tools/clangbackend/source/fulltokeninfo.h

@@ -34,9 +34,8 @@ class FullTokenInfo : public TokenInfo
     template<class T> friend class TokenProcessor;
 public:
     FullTokenInfo() = default;
-    FullTokenInfo(const CXCursor &cxCursor,
-                  CXToken *cxToken,
-                  CXTranslationUnit cxTranslationUnit,
+    FullTokenInfo(const Cursor &cursor,
+                  const Token *token,
                   std::vector<CXSourceRange> &m_currentOutputArgumentRanges);
     void evaluate() override;
 

src/tools/clangbackend/source/sourcelocation.h

@@ -64,6 +64,9 @@ public:
 
     SourceLocationContainer toSourceLocationContainer() const;
 
+    CXTranslationUnit tu() const { return cxTranslationUnit; }
+    CXSourceLocation cx() const { return cxSourceLocation; }
+
 private:
     operator CXSourceLocation() const;
 

src/tools/clangbackend/source/sourcerange.h

@@ -56,6 +56,8 @@ public:
     operator CXSourceRange() const;
     operator SourceRangeContainer() const;
 
+    CXTranslationUnit tu() const { return cxTranslationUnit; }
+
 private:
     CXSourceRange cxSourceRange;
     CXTranslationUnit cxTranslationUnit = nullptr;

src/tools/clangbackend/source/token.cpp

@@ -0,0 +1,141 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+
+#include "token.h"
+
+#include "clangstring.h"
+#include "cursor.h"
+#include "sourcelocation.h"
+#include "sourcerange.h"
+
+namespace ClangBackEnd {
+
+Token::Token(CXTranslationUnit cxTranslationUnit, CXToken *cxToken)
+    : m_cxTranslationUnit(cxTranslationUnit)
+    , m_cxToken(cxToken)
+{
+}
+
+bool Token::isNull() const
+{
+    return !m_cxToken;
+}
+
+CXTokenKind Token::kind() const
+{
+    return clang_getTokenKind(*m_cxToken);
+}
+
+CXToken *Token::cx() const
+{
+    return m_cxToken;
+}
+
+SourceLocation Token::location() const
+{
+    return SourceLocation(m_cxTranslationUnit, clang_getTokenLocation(m_cxTranslationUnit, *m_cxToken));
+}
+
+SourceRange Token::extent() const
+{
+    return SourceRange(m_cxTranslationUnit, clang_getTokenExtent(m_cxTranslationUnit, *m_cxToken));
+}
+
+ClangString Token::spelling() const
+{
+    return clang_getTokenSpelling(m_cxTranslationUnit, *m_cxToken);
+}
+
+Tokens::Tokens(const SourceRange &range)
+    : m_cxTranslationUnit(range.tu())
+{
+    CXSourceRange cxRange(range);
+    CXToken *cxTokens;
+    unsigned numTokens;
+    clang_tokenize(m_cxTranslationUnit, cxRange, &cxTokens, &numTokens);
+
+    m_tokens.reserve(numTokens);
+    for (size_t i = 0; i < numTokens; ++i)
+        m_tokens.push_back(Token(m_cxTranslationUnit, cxTokens + i));
+}
+
+std::vector<Cursor> Tokens::annotate() const
+{
+    std::vector<Cursor> cursors;
+    if (m_tokens.empty())
+        return cursors;
+
+    std::vector<CXCursor> cxCursors;
+    cxCursors.resize(m_tokens.size());
+    clang_annotateTokens(m_cxTranslationUnit, m_tokens.front().cx(),
+                         static_cast<unsigned>(m_tokens.size()), cxCursors.data());
+    cursors.reserve(cxCursors.size());
+    for (const CXCursor &cxCursor : cxCursors)
+        cursors.emplace_back(cxCursor);
+
+    return cursors;
+}
+
+const Token &Tokens::operator[](size_t index) const
+{
+    return m_tokens[index];
+}
+
+Token &Tokens::operator[](size_t index)
+{
+    return m_tokens[index];
+}
+
+std::vector<Token>::const_iterator Tokens::cbegin() const
+{
+    return m_tokens.cbegin();
+}
+
+std::vector<Token>::const_iterator Tokens::cend() const
+{
+    return m_tokens.cend();
+}
+
+std::vector<Token>::iterator Tokens::begin()
+{
+    return m_tokens.begin();
+}
+
+std::vector<Token>::iterator Tokens::end()
+{
+    return m_tokens.end();
+}
+
+Tokens::~Tokens()
+{
+    if (m_tokens.empty())
+        return;
+
+    clang_disposeTokens(m_cxTranslationUnit, m_tokens.front().cx(),
+                        static_cast<unsigned>(m_tokens.size()));
+}
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/token.h

@@ -0,0 +1,90 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <clang-c/Index.h>
+
+#include <vector>
+
+namespace ClangBackEnd {
+
+class ClangString;
+class Cursor;
+class SourceLocation;
+class SourceRange;
+
+class Token
+{
+    friend class Tokens;
+public:
+    bool isNull() const;
+
+    CXTokenKind kind() const;
+
+    SourceLocation location() const;
+    SourceRange extent() const;
+    ClangString spelling() const;
+
+    CXToken *cx() const;
+    CXTranslationUnit tu() const { return m_cxTranslationUnit; }
+
+private:
+    Token(CXTranslationUnit m_cxTranslationUnit, CXToken *cxToken);
+
+    CXTranslationUnit m_cxTranslationUnit;
+    CXToken *m_cxToken;
+};
+
+class Tokens
+{
+public:
+    Tokens() = default;
+    Tokens(const SourceRange &range);
+    ~Tokens();
+
+    Tokens(Tokens &&other) = default;
+    Tokens(const Tokens &other) = delete;
+    Tokens &operator=(Tokens &&other) = default;
+    Tokens &operator=(const Tokens &other) = delete;
+
+    std::vector<Cursor> annotate() const;
+
+    size_t size() const { return m_tokens.size(); }
+    const Token &operator[](size_t index) const;
+    Token &operator[](size_t index);
+
+    std::vector<Token>::const_iterator cbegin() const;
+    std::vector<Token>::const_iterator cend() const;
+    std::vector<Token>::iterator begin();
+    std::vector<Token>::iterator end();
+
+    CXTranslationUnit tu() const { return m_cxTranslationUnit; }
+private:
+    CXTranslationUnit m_cxTranslationUnit;
+    std::vector<Token> m_tokens;
+};
+
+} // namespace ClangBackEnd

src/tools/clangbackend/source/tokeninfo.cpp

@@ -25,6 +25,7 @@
 
 #include "clangstring.h"
 #include "cursor.h"
+#include "token.h"
 #include "tokeninfo.h"
 #include "sourcelocation.h"
 #include "sourcerange.h"
@@ -34,17 +35,14 @@
 
 namespace ClangBackEnd {
 
-TokenInfo::TokenInfo(const CXCursor &cxCursor,
-                     CXToken *cxToken,
-                     CXTranslationUnit cxTranslationUnit,
+TokenInfo::TokenInfo(const Cursor &cursor,
+                     const Token *token,
                      std::vector<CXSourceRange> &currentOutputArgumentRanges)
-    : m_originalCursor(cxCursor),
-      m_cxToken(cxToken),
-      m_cxTranslationUnit(cxTranslationUnit),
+    : m_originalCursor(cursor),
+      m_token(token),
       m_currentOutputArgumentRanges(&currentOutputArgumentRanges)
 {
-    const SourceRange sourceRange {cxTranslationUnit,
-                                   clang_getTokenExtent(cxTranslationUnit, *cxToken)};
+    const SourceRange sourceRange = token->extent();
     const auto start = sourceRange.start();
     const auto end = sourceRange.end();
 
@@ -472,18 +470,18 @@ static HighlightingType literalKind(const Cursor &cursor)
     Q_UNREACHABLE();
 }
 
-static bool isTokenPartOfOperator(const Cursor &declarationCursor, CXToken *token)
+static bool isTokenPartOfOperator(const Cursor &declarationCursor, const Token &token)
 {
     Q_ASSERT(declarationCursor.isDeclaration());
     const CXTranslationUnit cxTranslationUnit = declarationCursor.cxTranslationUnit();
-    const ClangString tokenName = clang_getTokenSpelling(cxTranslationUnit, *token);
+    const ClangString tokenName = token.spelling();
     if (tokenName == "operator")
         return true;
 
     if (tokenName == "(") {
         // Valid operator declarations have at least one token after '(' so
         // it's safe to proceed to token + 1 without extra checks.
-        const ClangString nextToken = clang_getTokenSpelling(cxTranslationUnit, *(token + 1));
+        const ClangString nextToken = clang_getTokenSpelling(cxTranslationUnit, *(token.cx() + 1));
         if (nextToken != ")") {
             // Argument lists' parentheses are not operator tokens.
             // This '('-token opens a (non-empty) argument list.
@@ -493,7 +491,7 @@ static bool isTokenPartOfOperator(const Cursor &declarationCursor, CXToken *toke
 
     // It's safe to evaluate the preceding token because we will at least have
     // the 'operator'-keyword's token to the left.
-    CXToken *prevToken = token - 1;
+    CXToken *prevToken = token.cx() - 1;
     if (clang_getTokenKind(*prevToken) == CXToken_Punctuation) {
         if (tokenName == "(") {
             // In an operator declaration, when a '(' follows another punctuation
@@ -526,7 +524,7 @@ void TokenInfo::overloadedOperatorKind()
     if (!declarationCursor.displayName().startsWith("operator"))
         return;
 
-    if (inOperatorDeclaration && !isTokenPartOfOperator(declarationCursor, m_cxToken))
+    if (inOperatorDeclaration && !isTokenPartOfOperator(declarationCursor, *m_token))
         return;
 
     m_types.mixinHighlightingTypes.push_back(HighlightingType::Operator);
@@ -580,45 +578,36 @@ enum class QtMacroPart
     FunctionOrPrimitiveType
 };
 
-static bool isFirstTokenOfCursor(const Cursor &cursor, CXToken *token)
+static bool isFirstTokenOfCursor(const Cursor &cursor, const Token &token)
 {
-    const CXTranslationUnit cxTranslationUnit = cursor.cxTranslationUnit();
-    const SourceLocation tokenLocation = SourceLocation(cxTranslationUnit,
-                                                        clang_getTokenLocation(cxTranslationUnit,
-                                                                               *token));
-    return cursor.sourceLocation() == tokenLocation;
+    return cursor.sourceLocation() == token.location();
 }
 
-static bool isLastTokenOfCursor(const Cursor &cursor, CXToken *token)
+static bool isLastTokenOfCursor(const Cursor &cursor, const Token &token)
 {
-    const CXTranslationUnit cxTranslationUnit = cursor.cxTranslationUnit();
-    const SourceLocation tokenLocation = SourceLocation(cxTranslationUnit,
-                                                        clang_getTokenLocation(cxTranslationUnit,
-                                                                               *token));
-    return cursor.sourceRange().end() == tokenLocation;
+    return cursor.sourceRange().end() == token.location();
 }
 
-static bool isValidMacroToken(const Cursor &cursor, CXToken *token)
+static bool isValidMacroToken(const Cursor &cursor, const Token &token)
 {
     // Proper macro token has at least '(' and ')' around it.
     return !isFirstTokenOfCursor(cursor, token) && !isLastTokenOfCursor(cursor, token);
 }
 
-static QtMacroPart propertyPart(CXTranslationUnit cxTranslationUnit, CXToken *token)
+static QtMacroPart propertyPart(const Token &token)
 {
     static constexpr const char *propertyKeywords[]
             = {"READ", "WRITE", "MEMBER", "RESET", "NOTIFY", "REVISION", "DESIGNABLE",
                "SCRIPTABLE", "STORED", "USER", "CONSTANT", "FINAL"
               };
 
-    const ClangString currentToken = clang_getTokenSpelling(cxTranslationUnit, *token);
-    if (std::find(std::begin(propertyKeywords), std::end(propertyKeywords), currentToken)
+    if (std::find(std::begin(propertyKeywords), std::end(propertyKeywords), token.spelling())
             != std::end(propertyKeywords)) {
         return QtMacroPart::Keyword;
     }
 
-    const ClangString nextToken = clang_getTokenSpelling(cxTranslationUnit, *(token + 1));
-    const ClangString previousToken = clang_getTokenSpelling(cxTranslationUnit, *(token - 1));
+    const ClangString nextToken = clang_getTokenSpelling(token.tu(), *(token.cx() + 1));
+    const ClangString previousToken = clang_getTokenSpelling(token.tu(), *(token.cx() - 1));
     if (std::find(std::begin(propertyKeywords), std::end(propertyKeywords), nextToken)
             != std::end(propertyKeywords)) {
         if (std::find(std::begin(propertyKeywords), std::end(propertyKeywords), previousToken)
@@ -645,30 +634,30 @@ static QtMacroPart signalSlotPart(CXTranslationUnit cxTranslationUnit, CXToken *
     return (prevToken == "SLOT") ? QtMacroPart::SlotFunction : QtMacroPart::SlotType;
 }
 
-static QtMacroPart qtMacroPart(CXTranslationUnit cxTranslationUnit, CXToken *token)
+static QtMacroPart qtMacroPart(const Token &token)
 {
-    CXSourceLocation location = clang_getTokenLocation(cxTranslationUnit, *token);
+    const SourceLocation location = token.location();
 
     // If current token is inside macro then the cursor from token's position will be
     // the whole macro cursor.
-    Cursor possibleQtMacroCursor = clang_getCursor(cxTranslationUnit, location);
+    Cursor possibleQtMacroCursor = clang_getCursor(token.tu(), location.cx());
     if (!isValidMacroToken(possibleQtMacroCursor, token))
         return QtMacroPart::None;
 
     ClangString spelling = possibleQtMacroCursor.spelling();
     if (spelling == "Q_PROPERTY")
-        return propertyPart(cxTranslationUnit, token);
+        return propertyPart(token);
 
     if (spelling == "SIGNAL")
-        return signalSlotPart(cxTranslationUnit, token, true);
+        return signalSlotPart(token.tu(), token.cx(), true);
     if (spelling == "SLOT")
-        return signalSlotPart(cxTranslationUnit, token, false);
+        return signalSlotPart(token.tu(), token.cx(), false);
     return QtMacroPart::None;
 }
 
 void TokenInfo::invalidFileKind()
 {
-    const QtMacroPart macroPart = qtMacroPart(m_cxTranslationUnit, m_cxToken);
+    const QtMacroPart macroPart = qtMacroPart(*m_token);
 
     switch (macroPart) {
     case QtMacroPart::None:
@@ -708,7 +697,7 @@ void TokenInfo::keywordKind()
             break;
     }
 
-    const ClangString spelling = clang_getTokenSpelling(m_cxTranslationUnit, *m_cxToken);
+    const ClangString spelling = m_token->spelling();
     if (spelling == "bool"
             || spelling == "char"
             || spelling == "char16_t"
@@ -734,7 +723,7 @@ void TokenInfo::keywordKind()
 
 void TokenInfo::evaluate()
 {
-    auto cxTokenKind = clang_getTokenKind(*m_cxToken);
+    auto cxTokenKind = m_token->kind();
 
     m_types = HighlightingTypes();
 

src/tools/clangbackend/source/tokeninfo.h

@@ -36,15 +36,17 @@
 
 namespace ClangBackEnd {
 
+class Cursor;
+class Token;
+
 class TokenInfo
 {
     friend bool operator==(const TokenInfo &first, const TokenInfo &second);
     template<class T> friend class TokenProcessor;
 public:
     TokenInfo() = default;
-    TokenInfo(const CXCursor &cxCursor,
-              CXToken *cxToken,
-              CXTranslationUnit cxTranslationUnit,
+    TokenInfo(const Cursor &cursor,
+              const Token *token,
               std::vector<CXSourceRange> &m_currentOutputArgumentRanges);
     virtual ~TokenInfo() = default;
 
@@ -100,8 +102,7 @@ protected:
     virtual void punctuationOrOperatorKind();
 
     Cursor m_originalCursor;
-    CXToken *m_cxToken = nullptr;
-    CXTranslationUnit m_cxTranslationUnit = nullptr;
+    const Token *m_token;
     HighlightingTypes m_types;
 
 private:

src/tools/clangbackend/source/tokenprocessor.h

@@ -27,6 +27,7 @@
 
 #include "fulltokeninfo.h"
 #include "sourcerange.h"
+#include "token.h"
 #include "tokenprocessoriterator.h"
 #include "tokeninfocontainer.h"
 
@@ -52,53 +53,43 @@ public:
 
 public:
     TokenProcessor() = default;
-    TokenProcessor(CXTranslationUnit cxTranslationUnit, const SourceRange &range)
-        : cxTranslationUnit(cxTranslationUnit)
+    TokenProcessor(const SourceRange &range)
+        : tokens(range)
+        , cursors(tokens.annotate())
     {
-        unsigned cxTokensCount = 0;
-        clang_tokenize(cxTranslationUnit, range, &cxTokens, &cxTokensCount);
-        cxCursors.resize(cxTokensCount);
-        clang_annotateTokens(cxTranslationUnit, cxTokens, cxTokensCount, cxCursors.data());
-    }
-    ~TokenProcessor()
-    {
-        clang_disposeTokens(cxTranslationUnit, cxTokens, unsigned(cxCursors.size()));
     }
 
     bool isEmpty() const
     {
-        return cxCursors.empty();
+        return cursors.empty();
     }
     bool isNull() const
     {
-        return cxTokens == nullptr;
+        return !tokens.size();
     }
     size_t size() const
     {
-        return cxCursors.size();
+        return cursors.size();
     }
 
     const_iterator begin() const
     {
-        return const_iterator(cxCursors.cbegin(),
-                              cxTokens,
-                              cxTranslationUnit,
+        return const_iterator(cursors.cbegin(),
+                              tokens.cbegin(),
                               currentOutputArgumentRanges);
     }
 
     const_iterator end() const
     {
-        return const_iterator(cxCursors.cend(),
-                              cxTokens + cxCursors.size(),
-                              cxTranslationUnit,
+        return const_iterator(cursors.cend(),
+                              tokens.cend(),
                               currentOutputArgumentRanges);
     }
 
 
     T operator[](size_t index) const
     {
-        T tokenInfo(cxCursors[index], cxTokens + index, cxTranslationUnit,
-                    currentOutputArgumentRanges);
+        T tokenInfo(cursors[index], &tokens[index], currentOutputArgumentRanges);
         tokenInfo.evaluate();
         return tokenInfo;
     }
@@ -117,8 +108,8 @@ private:
     template<class TC>
     QVector<TC> toTokens() const
     {
-        QVector<TC> tokens;
-        tokens.reserve(int(size()));
+        QVector<TC> tokenInfos;
+        tokenInfos.reserve(int(size()));
 
         const auto isValidTokenInfo = [](const T &tokenInfo) {
             return !tokenInfo.hasInvalidMainType()
@@ -126,48 +117,48 @@ private:
                     && !tokenInfo.hasMainType(HighlightingType::Comment);
         };
 
-        for (size_t index = 0; index < cxCursors.size(); ++index) {
+        for (size_t index = 0; index < cursors.size(); ++index) {
             T tokenInfo = (*this)[index];
             if (isValidTokenInfo(tokenInfo))
-                tokens.push_back(tokenInfo);
+                tokenInfos.push_back(tokenInfo);
         }
 
-        return tokens;
+        return tokenInfos;
     }
 
     mutable std::vector<CXSourceRange> currentOutputArgumentRanges;
-    CXTranslationUnit cxTranslationUnit = nullptr;
-    CXToken *cxTokens = nullptr;
-
-    std::vector<CXCursor> cxCursors;
+    Tokens tokens;
+    std::vector<Cursor> cursors;
 };
 
 template <>
 inline
 QVector<TokenInfoContainer> TokenProcessor<FullTokenInfo>::toTokenInfoContainers() const
 {
-    QVector<FullTokenInfo> tokens = toTokens<FullTokenInfo>();
+    QVector<FullTokenInfo> tokenInfos = toTokens<FullTokenInfo>();
 
-    return Utils::transform(tokens,
-                            [&tokens](FullTokenInfo &token) -> TokenInfoContainer {
-        if (!token.m_extraInfo.declaration || token.hasMainType(HighlightingType::LocalVariable))
-            return token;
+    return Utils::transform(tokenInfos,
+                            [&tokenInfos](FullTokenInfo &tokenInfo) -> TokenInfoContainer {
+        if (!tokenInfo.m_extraInfo.declaration
+                || tokenInfo.hasMainType(HighlightingType::LocalVariable)) {
+            return tokenInfo;
+        }
 
-        const int index = tokens.indexOf(token);
-        const SourceLocationContainer &tokenStart = token.m_extraInfo.cursorRange.start;
-        for (auto it = tokens.rend() - index; it != tokens.rend(); ++it) {
+        const int index = tokenInfos.indexOf(tokenInfo);
+        const SourceLocationContainer &tokenStart = tokenInfo.m_extraInfo.cursorRange.start;
+        for (auto it = tokenInfos.rend() - index; it != tokenInfos.rend(); ++it) {
             if (it->m_extraInfo.declaration && !it->hasMainType(HighlightingType::LocalVariable)
-                    && it->m_originalCursor != token.m_originalCursor
+                    && it->m_originalCursor != tokenInfo.m_originalCursor
                     && it->m_extraInfo.cursorRange.contains(tokenStart)) {
-                if (token.m_originalCursor.lexicalParent() != it->m_originalCursor
-                        && !token.hasMainType(HighlightingType::QtProperty)) {
+                if (tokenInfo.m_originalCursor.lexicalParent() != it->m_originalCursor
+                        && !tokenInfo.hasMainType(HighlightingType::QtProperty)) {
                     continue;
                 }
-                token.m_extraInfo.lexicalParentIndex = std::distance(it, tokens.rend()) - 1;
+                tokenInfo.m_extraInfo.lexicalParentIndex = std::distance(it, tokenInfos.rend()) - 1;
                 break;
             }
         }
-        return token;
+        return tokenInfo;
     });
 }
 

src/tools/clangbackend/source/tokenprocessoriterator.h

@@ -41,53 +41,49 @@ template<class T>
 class TokenProcessorIterator : public std::iterator<std::forward_iterator_tag, TokenInfo, uint>
 {
 public:
-    TokenProcessorIterator(std::vector<CXCursor>::const_iterator cxCursorIterator,
-                           CXToken *cxToken,
-                           CXTranslationUnit cxTranslationUnit,
+    TokenProcessorIterator(std::vector<Cursor>::const_iterator cursorIterator,
+                           std::vector<Token>::const_iterator tokenIterator,
                            std::vector<CXSourceRange> &currentOutputArgumentRanges)
-        : cxCursorIterator(cxCursorIterator),
-          cxToken(cxToken),
-          cxTranslationUnit(cxTranslationUnit),
+        : cursorIterator(cursorIterator),
+          tokenIterator(tokenIterator),
           currentOutputArgumentRanges(currentOutputArgumentRanges)
     {}
 
     TokenProcessorIterator& operator++()
     {
-        ++cxCursorIterator;
-        ++cxToken;
+        ++cursorIterator;
+        ++tokenIterator;
 
         return *this;
     }
 
     TokenProcessorIterator operator++(int)
     {
-        return TokenProcessorIterator(cxCursorIterator++,
-                                      cxToken++,
-                                      cxTranslationUnit,
+        return TokenProcessorIterator(cursorIterator++,
+                                      tokenIterator++,
                                       currentOutputArgumentRanges);
     }
 
     bool operator==(TokenProcessorIterator other) const
     {
-        return cxCursorIterator == other.cxCursorIterator;
+        return cursorIterator == other.cursorIterator;
     }
 
     bool operator!=(TokenProcessorIterator other) const
     {
-        return cxCursorIterator != other.cxCursorIterator;
+        return cursorIterator != other.cursorIterator;
     }
 
     T operator*()
     {
-        T tokenInfo(*cxCursorIterator, cxToken, cxTranslationUnit, currentOutputArgumentRanges);
+        T tokenInfo(*cursorIterator, &(*tokenIterator), currentOutputArgumentRanges);
         tokenInfo.evaluate();
         return tokenInfo;
     }
 
 private:
-    std::vector<CXCursor>::const_iterator cxCursorIterator;
-    CXToken *cxToken;
-    CXTranslationUnit cxTranslationUnit;
+    std::vector<Cursor>::const_iterator cursorIterator;
+    std::vector<Token>::const_iterator tokenIterator;
     std::vector<CXSourceRange> &currentOutputArgumentRanges;
 };
 

tests/unit/unittest/data/token.cpp

@@ -0,0 +1,3 @@
+void function(int x)
+{
+}

tests/unit/unittest/token-test.cpp

@@ -0,0 +1,131 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "googletest.h"
+
+#include "testenvironment.h"
+
+#include <clangdocument.h>
+#include <clangdocuments.h>
+#include <clangstring.h>
+#include <clangtranslationunit.h>
+#include <cursor.h>
+#include <sourcelocation.h>
+#include <sourcerange.h>
+#include <token.h>
+#include <unsavedfiles.h>
+
+using ClangBackEnd::Cursor;
+using ClangBackEnd::Document;
+using ClangBackEnd::TranslationUnit;
+using ClangBackEnd::UnsavedFiles;
+using ClangBackEnd::Documents;
+using ClangBackEnd::ClangString;
+using ClangBackEnd::SourceLocation;
+using ClangBackEnd::SourceRange;
+using ClangBackEnd::Token;
+using ClangBackEnd::Tokens;
+
+namespace {
+
+struct Data {
+    ClangBackEnd::UnsavedFiles unsavedFiles;
+    ClangBackEnd::Documents documents{unsavedFiles};
+    Utf8String filePath{Utf8StringLiteral(TESTDATA_DIR"/token.cpp")};
+    Utf8StringVector compilationArguments{
+        TestEnvironment::addPlatformArguments({Utf8StringLiteral("-std=c++11")})};
+    Document document{filePath, compilationArguments, documents};
+    TranslationUnit translationUnit{filePath,
+                                    filePath,
+                                    document.translationUnit().cxIndex(),
+                                    document.translationUnit().cxTranslationUnit()};
+};
+
+class Token : public ::testing::Test
+{
+protected:
+    static void SetUpTestCase()
+    {
+        d = std::make_unique<const Data>();
+        d->document.parse();
+    }
+
+protected:
+    static std::unique_ptr<const Data> d;
+    const Document &document = d->document;
+    const TranslationUnit &translationUnit = d->translationUnit;
+    const SourceRange range{{translationUnit.cxTranslationUnit(), d->filePath, 1, 1},
+                            {translationUnit.cxTranslationUnit(), d->filePath, 3, 2}};
+    const Tokens tokens{range};
+};
+
+std::unique_ptr<const Data> Token::d;
+
+TEST_F(Token, CreateTokens)
+{
+    ASSERT_THAT(tokens.size(), 8u);
+}
+
+TEST_F(Token, AnnotateTokens)
+{
+    auto cursors = tokens.annotate();
+
+    ASSERT_THAT(cursors.size(), 8u);
+}
+
+TEST_F(Token, TokenKind)
+{
+    auto kind = tokens[0].kind();
+
+    ASSERT_THAT(kind, CXToken_Keyword);
+}
+
+TEST_F(Token, TokenLocation)
+{
+    auto location = range.start();
+
+    auto tokenLocation = tokens[0].location();
+
+    ASSERT_THAT(tokenLocation, location);
+}
+
+TEST_F(Token, TokenSpelling)
+{
+    auto spelling = tokens[0].spelling();
+
+    ASSERT_THAT(spelling, "void");
+}
+
+TEST_F(Token, TokenExtent)
+{
+    ::SourceRange tokenRange(range.start(), ::SourceLocation(translationUnit.cxTranslationUnit(), d->filePath, 1, 5));
+
+    auto extent = tokens[0].extent();
+
+    ASSERT_THAT(extent, tokenRange);
+}
+
+
+}

tests/unit/unittest/unittest.pro

@@ -101,7 +101,7 @@ SOURCES += \
     refactoringprojectupdater-test.cpp \
     projectpartqueue-test.cpp \
     processormanager-test.cpp \
-    taskscheduler-test.cpp
+    taskscheduler-test.cpp \
 
 !isEmpty(LIBCLANG_LIBS) {
 SOURCES += \
@@ -154,6 +154,7 @@ SOURCES += \
     sqlitestatement-test.cpp \
     sqlitetable-test.cpp \
     sqlstatementbuilder-test.cpp \
+    token-test.cpp \
     translationunitupdater-test.cpp \
     unsavedfiles-test.cpp \
     unsavedfile-test.cpp \