Removing SimpleToken

src/libs/cplusplus/BackwardsScanner.cpp

@@ -52,19 +52,19 @@ BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, int maxBlockCount,
     _startToken = _tokens.size();
 }
 
-SimpleToken BackwardsScanner::LA(int index) const
+Token BackwardsScanner::LA(int index) const
 { return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); }
 
-SimpleToken BackwardsScanner::operator[](int index) const
+Token BackwardsScanner::operator[](int index) const
 { return const_cast<BackwardsScanner *>(this)->fetchToken(index); }
 
-const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
+const Token &BackwardsScanner::fetchToken(int tokenIndex)
 {
     while (_offset + tokenIndex < 0) {
         _block = _block.previous();
         if (_blocksTokenized == _maxBlockCount || !_block.isValid()) {
             ++_offset;
-            _tokens.prepend(SimpleToken()); // sentinel
+            _tokens.prepend(Token()); // sentinel
             break;
         } else {
             ++_blocksTokenized;
@@ -73,10 +73,10 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
             _text.prepend(QLatin1Char('\n'));
             _text.prepend(blockText);
 
-            QList<SimpleToken> adaptedTokens;
+            QList<Token> adaptedTokens;
             for (int i = 0; i < _tokens.size(); ++i) {
-                SimpleToken t = _tokens.at(i);
+                Token t = _tokens.at(i);
-                t.setPosition(t.position() + blockText.length() + 1);
+                t.offset += + blockText.length() + 1;
                 adaptedTokens.append(t);
             }
 
@@ -100,19 +100,19 @@ QString BackwardsScanner::text() const
 
 QString BackwardsScanner::mid(int index) const
 {
-    const SimpleToken &firstToken = _tokens.at(index + _offset);
+    const Token &firstToken = _tokens.at(index + _offset);
     return _text.mid(firstToken.begin());
 }
 
 QString BackwardsScanner::text(int index) const
 {
-    const SimpleToken &firstToken = _tokens.at(index + _offset);
+    const Token &firstToken = _tokens.at(index + _offset);
     return _text.mid(firstToken.begin(), firstToken.length());
 }
 
 QStringRef BackwardsScanner::textRef(int index) const
 {
-    const SimpleToken &firstToken = _tokens.at(index + _offset);
+    const Token &firstToken = _tokens.at(index + _offset);
     return _text.midRef(firstToken.begin(), firstToken.length());
 }
 
@@ -181,11 +181,11 @@ int BackwardsScanner::startOfLine(int index) const
     const BackwardsScanner tk(*this);
 
     forever {
-        const SimpleToken &tok = tk[index - 1];
+        const Token &tok = tk[index - 1];
 
         if (tok.is(T_EOF_SYMBOL))
             break;
-        else if (tok.followsNewline())
+        else if (tok.newline())
             return index - 1;
 
         --index;
@@ -201,7 +201,7 @@ int BackwardsScanner::startOfBlock(int index) const
     const int start = index;
 
     forever {
-        SimpleToken token = tk[index - 1];
+        Token token = tk[index - 1];
 
         if (token.is(T_EOF_SYMBOL)) {
             break;
@@ -234,9 +234,10 @@ int BackwardsScanner::startOfBlock(int index) const
 
 QString BackwardsScanner::indentationString(int index) const
 {
-    const SimpleToken tokenAfterNewline = operator[](startOfLine(index + 1));
+    const Token tokenAfterNewline = operator[](startOfLine(index + 1));
-    const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'), tokenAfterNewline.position()));
+    const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'),
-    return _text.mid(newlinePos, tokenAfterNewline.position() - newlinePos);
+                                                     tokenAfterNewline.begin()));
+    return _text.mid(newlinePos, tokenAfterNewline.begin() - newlinePos);
 }
 
 

src/libs/cplusplus/BackwardsScanner.h

@@ -55,10 +55,10 @@ public:
     QString text(int index) const;
     QStringRef textRef(int index) const;
     // 1-based
-    SimpleToken LA(int index) const;
+    Token LA(int index) const;
 
     // n-la token is [startToken - n]
-    SimpleToken operator[](int index) const; // ### deprecate
+    Token operator[](int index) const; // ### deprecate
 
     QString indentationString(int index) const;
 
@@ -71,10 +71,10 @@ public:
     static int previousBlockState(const QTextBlock &block);
 
 private:
-    const SimpleToken &fetchToken(int tokenIndex);
+    const Token &fetchToken(int tokenIndex);
 
 private:
-    QList<SimpleToken> _tokens;
+    QList<Token> _tokens;
     int _offset;
     int _blocksTokenized;
     QTextBlock _block;

src/libs/cplusplus/ExpressionUnderCursor.cpp

@@ -56,7 +56,7 @@ int ExpressionUnderCursor::startOfExpression(BackwardsScanner &tk, int index)
     index = startOfExpression_helper(tk, index);
 
     if (_jumpedComma) {
-        const SimpleToken &tok = tk[index - 1];
+        const Token &tok = tk[index - 1];
 
         switch (tok.kind()) {
         case T_COMMA:
@@ -204,7 +204,7 @@ int ExpressionUnderCursor::startOfExpression_helper(BackwardsScanner &tk, int in
     return index;
 }
 
-bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
+bool ExpressionUnderCursor::isAccessToken(const Token &tk)
 {
     switch (tk.kind()) {
     case T_COLON_COLON:
@@ -237,12 +237,12 @@ int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
     int index = scanner.startToken();
 
     forever {
-        const SimpleToken &tk = scanner[index - 1];
+        const Token &tk = scanner[index - 1];
 
         if (tk.is(T_EOF_SYMBOL))
             break;
         else if (tk.is(T_LPAREN))
-            return scanner.startPosition() + tk.position();
+            return scanner.startPosition() + tk.begin();
         else if (tk.is(T_RPAREN)) {
             int matchingBrace = scanner.startOfMatchingBrace(index);
 

src/libs/cplusplus/ExpressionUnderCursor.h

@@ -42,7 +42,6 @@ QT_END_NAMESPACE
 namespace CPlusPlus {
 
 class BackwardsScanner;
-class SimpleToken;
 
 class CPLUSPLUS_EXPORT ExpressionUnderCursor
 {
@@ -56,7 +55,7 @@ public:
 private:
     int startOfExpression(BackwardsScanner &tk, int index);
     int startOfExpression_helper(BackwardsScanner &tk, int index);
-    bool isAccessToken(const SimpleToken &tk);
+    bool isAccessToken(const Token &tk);
 
 private:
     bool _jumpedComma;

src/libs/cplusplus/MatchingText.cpp

@@ -152,7 +152,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
     const int startToken = tk.startToken();
     int index = startToken;
 
-    const SimpleToken &token = tk[index - 1];
+    const Token &token = tk[index - 1];
 
     if (text.at(0) == QLatin1Char('"') && (token.is(T_STRING_LITERAL) || token.is(T_WIDE_STRING_LITERAL))) {
         if (text.length() != 1)
@@ -220,7 +220,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
 
     --index; // consume the `{'
 
-    const SimpleToken &token = tk[index - 1];
+    const Token &token = tk[index - 1];
 
     if (token.is(T_STRING_LITERAL) && tk[index - 2].is(T_EXTERN)) {
         // recognized extern "C"
@@ -230,7 +230,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
         int i = index - 1;
 
         forever {
-            const SimpleToken &current = tk[i - 1];
+            const Token &current = tk[i - 1];
 
             if (current.is(T_EOF_SYMBOL))
                 break;
@@ -290,7 +290,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
         }
 
         // look at the token before the matched brace
-        const SimpleToken &tokenBeforeBrace = tk[lparenIndex - 1];
+        const Token &tokenBeforeBrace = tk[lparenIndex - 1];
 
         if (tokenBeforeBrace.is(T_IF)) {
             // recognized an if statement

src/libs/cplusplus/SimpleLexer.cpp

@@ -37,47 +37,6 @@
 
 using namespace CPlusPlus;
 
-SimpleToken::SimpleToken(const Token &token)
-    : _kind(token.f.kind)
-    , _flags(0)
-    , _position(token.begin())
-    , _length(token.f.length)
-{
-    f._whitespace = token.f.whitespace;
-    f._newline = token.f.newline;
-}
-
-bool SimpleToken::isLiteral() const
-{
-    return _kind >= T_FIRST_LITERAL && _kind <= T_LAST_LITERAL;
-}
-
-bool SimpleToken::isOperator() const
-{
-    return _kind >= T_FIRST_OPERATOR && _kind <= T_LAST_OPERATOR;
-}
-
-bool SimpleToken::isKeyword() const
-{
-    return _kind >= T_FIRST_KEYWORD && _kind < T_FIRST_QT_KEYWORD;
-}
-
-bool SimpleToken::isComment() const
-{
-    return _kind == T_COMMENT || _kind == T_DOXY_COMMENT ||
-            _kind == T_CPP_COMMENT || _kind == T_CPP_DOXY_COMMENT;
-}
-
-bool SimpleToken::isObjCAtKeyword() const
-{
-    return _kind >= T_FIRST_OBJC_AT_KEYWORD && _kind <= T_LAST_OBJC_AT_KEYWORD;
-}
-
-const char *SimpleToken::name() const
-{
-    return Token::name(_kind);
-}
-
 SimpleLexer::SimpleLexer()
     : _lastState(0),
       _skipComments(false),
@@ -119,9 +78,9 @@ void SimpleLexer::setSkipComments(bool skipComments)
     _skipComments = skipComments;
 }
 
-QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
+QList<Token> SimpleLexer::operator()(const QString &text, int state)
 {
-    QList<SimpleToken> tokens;
+    QList<Token> tokens;
 
     const QByteArray bytes = text.toLatin1();
     const char *firstChar = bytes.constData();
@@ -131,6 +90,7 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
     lex.setQtMocRunEnabled(_qtMocRunEnabled);
     lex.setObjCEnabled(_objCEnabled);
     lex.setStartWithNewline(true);
+    lex.setObjCEnabled(_objCEnabled);
 
     if (! _skipComments)
         lex.setScanCommentTokens(true);
@@ -147,57 +107,53 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
             break;
 
         QStringRef spell = text.midRef(lex.tokenOffset(), lex.tokenLength());
-        SimpleToken simpleTk(tk);
         lex.setScanAngleStringLiteralTokens(false);
 
         if (tk.f.newline && tk.is(T_POUND))
             inPreproc = true;
-        else if (inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
+        else if (inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
                  spell == QLatin1String("include"))
             lex.setScanAngleStringLiteralTokens(true);
         else if (_objCEnabled
-                 && inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
+                 && inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
                  spell == QLatin1String("import"))
             lex.setScanAngleStringLiteralTokens(true);
 
-        if (_objCEnabled && tk.is(T_IDENTIFIER))
+        tokens.append(tk);
-            simpleTk.f._objcTypeQualifier = (classifyObjectiveCContextKeyword(firstChar + tk.offset, tk.f.length) != Token_identifier);
-
-        tokens.append(simpleTk);
     }
 
     _lastState = lex.state();
     return tokens;
 }
 
-int SimpleLexer::tokenAt(const QList<SimpleToken> &tokens, int offset)
+int SimpleLexer::tokenAt(const QList<Token> &tokens, unsigned offset)
 {
     for (int index = tokens.size() - 1; index >= 0; --index) {
-        const SimpleToken &tk = tokens.at(index);
+        const Token &tk = tokens.at(index);
-        if (tk.position() <= offset && tk.end() >= offset)
+        if (tk.begin() <= offset && tk.end() >= offset)
             return index;
     }
 
     return -1;
 }
 
-SimpleToken SimpleLexer::tokenAt(const QString &text,
+Token SimpleLexer::tokenAt(const QString &text,
-                                 int offset,
+                           unsigned offset,
-                                 int state,
+                           int state,
-                                 bool qtMocRunEnabled)
+                           bool qtMocRunEnabled)
 {
     SimpleLexer tokenize;
     tokenize.setQtMocRunEnabled(qtMocRunEnabled);
-    const QList<SimpleToken> tokens = tokenize(text, state);
+    const QList<Token> tokens = tokenize(text, state);
     const int tokenIdx = tokenAt(tokens, offset);
-    return (tokenIdx == -1) ? SimpleToken() : tokens.at(tokenIdx);
+    return (tokenIdx == -1) ? Token() : tokens.at(tokenIdx);
 }
 
-int SimpleLexer::tokenBefore(const QList<SimpleToken> &tokens, int offset)
+int SimpleLexer::tokenBefore(const QList<Token> &tokens, unsigned offset)
 {
     for (int index = tokens.size() - 1; index >= 0; --index) {
-        const SimpleToken &tk = tokens.at(index);
+        const Token &tk = tokens.at(index);
-        if (tk.position() <= offset)
+        if (tk.begin() <= offset)
             return index;
     }
 

src/libs/cplusplus/SimpleLexer.h

@@ -39,73 +39,6 @@ namespace CPlusPlus {
 class SimpleLexer;
 class Token;
 
-class CPLUSPLUS_EXPORT SimpleToken
-{
-public:
-    SimpleToken(const Token &token);
-
-    SimpleToken()
-        : _kind(0)
-        , _flags(0)
-        , _position(0)
-        , _length(0)
-    { }
-
-    inline int kind() const
-    { return _kind; }
-
-    inline int position() const
-    { return _position; }
-
-    inline int length() const
-    { return _length; }
-
-    inline int begin() const
-    { return _position; }
-
-    inline int end() const
-    { return _position + _length; }
-
-    inline bool followsNewline() const
-    { return f._newline; }
-
-    inline bool followsWhitespace() const
-    { return f._whitespace; }
-
-    inline bool is(int k) const    { return _kind == k; }
-    inline bool isNot(int k) const { return _kind != k; }
-
-    bool isLiteral() const;
-    bool isOperator() const;
-    bool isKeyword() const;
-    bool isComment() const;
-    bool isObjCAtKeyword() const;
-    bool isObjCTypeQualifier() const { return f._objcTypeQualifier; }
-
-    const char *name() const;
-
-    // internal
-    inline void setPosition(int position)
-    { _position = position; }
-
-public:
-    short _kind;
-    union {
-        short _flags;
-
-        struct {
-            unsigned _newline: 1;
-            unsigned _whitespace: 1;
-            unsigned _objcTypeQualifier: 1;
-        } f;
-    };
-
-    int _position;
-    int _length;
-
-    friend class SimpleLexer;
-};
-
 class CPLUSPLUS_EXPORT SimpleLexer
 {
 public:
@@ -121,15 +54,18 @@ public:
     bool objCEnabled() const;
     void setObjCEnabled(bool onoff);
 
-    QList<SimpleToken> operator()(const QString &text, int state = 0);
+    QList<Token> operator()(const QString &text, int state = 0);
 
     int state() const
     { return _lastState; }
 
-    static int tokenAt(const QList<SimpleToken> &tokens, int offset);
+    static int tokenAt(const QList<Token> &tokens, unsigned offset);
-    static SimpleToken tokenAt(const QString &text, int offset, int state, bool qtMocRunEnabled = false);
+    static Token tokenAt(const QString &text,
+                         unsigned offset,
+                         int state,
+                         bool qtMocRunEnabled = false);
 
-    static int tokenBefore(const QList<SimpleToken> &tokens, int offset);
+    static int tokenBefore(const QList<Token> &tokens, unsigned offset);
 
 private:
     int _lastState;

src/plugins/cppeditor/cppeditor.cpp

@@ -1221,14 +1221,14 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
     SimpleLexer tokenize;
     tokenize.setQtMocRunEnabled(true);
     const QString blockText = cursor.block().text();
-    const QList<SimpleToken> tokens = tokenize(blockText, BackwardsScanner::previousBlockState(cursor.block()));
+    const QList<Token> tokens = tokenize(blockText, BackwardsScanner::previousBlockState(cursor.block()));
 
     bool recognizedQtMethod = false;
 
     for (int i = 0; i < tokens.size(); ++i) {
-        const SimpleToken &tk = tokens.at(i);
+        const Token &tk = tokens.at(i);
 
-        if (column >= tk.begin() && column <= tk.end()) {
+        if (((unsigned) column) >= tk.begin() && ((unsigned) column) <= tk.end()) {
             if (i >= 2 && tokens.at(i).is(T_IDENTIFIER) && tokens.at(i - 1).is(T_LPAREN)
                 && (tokens.at(i - 2).is(T_SIGNAL) || tokens.at(i - 2).is(T_SLOT))) {
 
@@ -1269,7 +1269,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
 
     if (! recognizedQtMethod) {
         const QTextBlock block = tc.block();
-        const SimpleToken tk = SimpleLexer::tokenAt(block.text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(block), true);
+        const Token tk = SimpleLexer::tokenAt(block.text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(block), true);
 
         beginOfToken = block.position() + tk.begin();
         endOfToken = block.position() + tk.end();
@@ -1431,11 +1431,11 @@ bool CPPEditor::contextAllowsAutoParentheses(const QTextCursor &cursor,
 
 bool CPPEditor::contextAllowsElectricCharacters(const QTextCursor &cursor) const
 {
-    const SimpleToken tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
+    const Token tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
 
     // XXX Duplicated from CPPEditor::isInComment to avoid tokenizing twice
     if (tk.isComment()) {
-        const int pos = cursor.selectionEnd() - cursor.block().position();
+        const unsigned pos = cursor.selectionEnd() - cursor.block().position();
 
         if (pos == tk.end()) {
             if (tk.is(T_CPP_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))
@@ -1452,7 +1452,7 @@ bool CPPEditor::contextAllowsElectricCharacters(const QTextCursor &cursor) const
     else if (tk.is(T_STRING_LITERAL) || tk.is(T_WIDE_STRING_LITERAL)
         || tk.is(T_CHAR_LITERAL) || tk.is(T_WIDE_CHAR_LITERAL)) {
 
-        const int pos = cursor.selectionEnd() - cursor.block().position();
+        const unsigned pos = cursor.selectionEnd() - cursor.block().position();
         if (pos <= tk.end())
             return false;
     }
@@ -1462,10 +1462,10 @@ bool CPPEditor::contextAllowsElectricCharacters(const QTextCursor &cursor) const
 
 bool CPPEditor::isInComment(const QTextCursor &cursor) const
 {
-    const SimpleToken tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
+    const Token tk = SimpleLexer::tokenAt(cursor.block().text(), cursor.positionInBlock(), BackwardsScanner::previousBlockState(cursor.block()));
 
     if (tk.isComment()) {
-        const int pos = cursor.selectionEnd() - cursor.block().position();
+        const unsigned pos = cursor.selectionEnd() - cursor.block().position();
 
         if (pos == tk.end()) {
             if (tk.is(T_CPP_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))
@@ -1520,7 +1520,7 @@ void CPPEditor::indentBlock(QTextDocument *doc, QTextBlock block, QChar typedCha
     const int tokenCount = tk.startToken();
 
     if (tokenCount != 0) {
-        const SimpleToken firstToken = tk[0];
+        const Token firstToken = tk[0];
 
         if (firstToken.is(T_COLON)) {
             const int previousLineIndent = indentationColumn(ts, tk, -1);
@@ -1556,8 +1556,8 @@ void CPPEditor::indentBlock(QTextDocument *doc, QTextBlock block, QChar typedCha
         }
 
         const QString spell = tk.text(tokenIndex);
-        if (tk[tokenIndex].followsNewline() && (spell.startsWith(QLatin1String("QT_")) ||
+        if (tk[tokenIndex].newline() && (spell.startsWith(QLatin1String("QT_")) ||
-                                                spell.startsWith(QLatin1String("Q_")))) {
+                                         spell.startsWith(QLatin1String("Q_")))) {
             const int indent = indentationColumn(ts, tk, tokenIndex);
             ts.indentLine(block, indent);
             return;

src/plugins/cppeditor/cpphighlighter.cpp

@@ -62,7 +62,7 @@ void CppHighlighter::highlightBlock(const QString &text)
     tokenize.setObjCEnabled(false);
 
     int initialState = state;
-    const QList<SimpleToken> tokens = tokenize(text, initialState);
+    const QList<Token> tokens = tokenize(text, initialState);
     state = tokenize.state(); // refresh the state
 
     int foldingIndent = initialBraceDepth;
@@ -81,7 +81,7 @@ void CppHighlighter::highlightBlock(const QString &text)
         return;
     }
 
-    const int firstNonSpace = tokens.first().position();
+    const unsigned firstNonSpace = tokens.first().begin();
 
     Parentheses parentheses;
     parentheses.reserve(20); // assume wizard level ;-)
@@ -89,36 +89,36 @@ void CppHighlighter::highlightBlock(const QString &text)
     bool highlightAsPreprocessor = false;
 
     for (int i = 0; i < tokens.size(); ++i) {
-        const SimpleToken &tk = tokens.at(i);
+        const Token &tk = tokens.at(i);
 
-        int previousTokenEnd = 0;
+        unsigned previousTokenEnd = 0;
         if (i != 0) {
             // mark the whitespaces
-            previousTokenEnd = tokens.at(i - 1).position() +
+            previousTokenEnd = tokens.at(i - 1).begin() +
                                tokens.at(i - 1).length();
         }
 
-        if (previousTokenEnd != tk.position()) {
+        if (previousTokenEnd != tk.begin()) {
-            setFormat(previousTokenEnd, tk.position() - previousTokenEnd,
+            setFormat(previousTokenEnd, tk.begin() - previousTokenEnd,
                       m_formats[CppVisualWhitespace]);
         }
 
         if (tk.is(T_LPAREN) || tk.is(T_LBRACE) || tk.is(T_LBRACKET)) {
-            const QChar c = text.at(tk.position());
+            const QChar c = text.at(tk.begin());
-            parentheses.append(Parenthesis(Parenthesis::Opened, c, tk.position()));
+            parentheses.append(Parenthesis(Parenthesis::Opened, c, tk.begin()));
             if (tk.is(T_LBRACE)) {
                 ++braceDepth;
 
                 // if a folding block opens at the beginning of a line, treat the entire line
                 // as if it were inside the folding block
-                if (tk.position() == firstNonSpace) {
+                if (tk.begin() == firstNonSpace) {
                     ++foldingIndent;
                     BaseTextDocumentLayout::userData(currentBlock())->setFoldingStartIncluded(true);
                 }
             }
         } else if (tk.is(T_RPAREN) || tk.is(T_RBRACE) || tk.is(T_RBRACKET)) {
-            const QChar c = text.at(tk.position());
+            const QChar c = text.at(tk.begin());
-            parentheses.append(Parenthesis(Parenthesis::Closed, c, tk.position()));
+            parentheses.append(Parenthesis(Parenthesis::Closed, c, tk.begin()));
             if (tk.is(T_RBRACE)) {
                 --braceDepth;
                 if (braceDepth < foldingIndent) {
@@ -137,30 +137,30 @@ void CppHighlighter::highlightBlock(const QString &text)
             highlightAsPreprocessor = false;
 
         if (i == 0 && tk.is(T_POUND)) {
-            highlightLine(text, tk.position(), tk.length(), m_formats[CppPreprocessorFormat]);
+            highlightLine(text, tk.begin(), tk.length(), m_formats[CppPreprocessorFormat]);
             highlightAsPreprocessor = true;
 
         } else if (highlightCurrentWordAsPreprocessor &&
-                   (tk.isKeyword() || tk.is(T_IDENTIFIER)) && isPPKeyword(text.midRef(tk.position(), tk.length())))
+                   (tk.isKeyword() || tk.is(T_IDENTIFIER)) && isPPKeyword(text.midRef(tk.begin(), tk.length())))
-            setFormat(tk.position(), tk.length(), m_formats[CppPreprocessorFormat]);
+            setFormat(tk.begin(), tk.length(), m_formats[CppPreprocessorFormat]);
 
         else if (tk.is(T_NUMERIC_LITERAL))
-            setFormat(tk.position(), tk.length(), m_formats[CppNumberFormat]);
+            setFormat(tk.begin(), tk.length(), m_formats[CppNumberFormat]);
 
         else if (tk.is(T_STRING_LITERAL) || tk.is(T_CHAR_LITERAL) || tk.is(T_ANGLE_STRING_LITERAL) ||
                  tk.is(T_AT_STRING_LITERAL))
-            highlightLine(text, tk.position(), tk.length(), m_formats[CppStringFormat]);
+            highlightLine(text, tk.begin(), tk.length(), m_formats[CppStringFormat]);
 
         else if (tk.is(T_WIDE_STRING_LITERAL) || tk.is(T_WIDE_CHAR_LITERAL))
-            highlightLine(text, tk.position(), tk.length(), m_formats[CppStringFormat]);
+            highlightLine(text, tk.begin(), tk.length(), m_formats[CppStringFormat]);
 
         else if (tk.isComment()) {
 
             if (tk.is(T_COMMENT) || tk.is(T_CPP_COMMENT))
-                highlightLine(text, tk.position(), tk.length(), m_formats[CppCommentFormat]);
+                highlightLine(text, tk.begin(), tk.length(), m_formats[CppCommentFormat]);
 
             else // a doxygen comment
-                highlightDoxygenComment(text, tk.position(), tk.length());
+                highlightDoxygenComment(text, tk.begin(), tk.length());
 
             // we need to insert a close comment parenthesis, if
             //  - the line starts in a C Comment (initalState != 0)
@@ -173,38 +173,38 @@ void CppHighlighter::highlightBlock(const QString &text)
                     BaseTextDocumentLayout::userData(currentBlock())->setFoldingEndIncluded(true);
                 else
                     foldingIndent = qMin(braceDepth, foldingIndent);
-                const int tokenEnd = tk.position() + tk.length() - 1;
+                const int tokenEnd = tk.begin() + tk.length() - 1;
                 parentheses.append(Parenthesis(Parenthesis::Closed, QLatin1Char('-'), tokenEnd));
 
                 // clear the initial state.
                 initialState = 0;
             }
 
-        } else if (tk.isKeyword() || isQtKeyword(text.midRef(tk.position(), tk.length())) || tk.isObjCAtKeyword() || tk.isObjCTypeQualifier())
+        } else if (tk.isKeyword() || isQtKeyword(text.midRef(tk.begin(), tk.length())) || tk.isObjCAtKeyword())
-            setFormat(tk.position(), tk.length(), m_formats[CppKeywordFormat]);
+            setFormat(tk.begin(), tk.length(), m_formats[CppKeywordFormat]);
 
         else if (tk.isOperator())
-            setFormat(tk.position(), tk.length(), m_formats[CppOperatorFormat]);
+            setFormat(tk.begin(), tk.length(), m_formats[CppOperatorFormat]);
 
         else if (i == 0 && tokens.size() > 1 && tk.is(T_IDENTIFIER) && tokens.at(1).is(T_COLON))
-            setFormat(tk.position(), tk.length(), m_formats[CppLabelFormat]);
+            setFormat(tk.begin(), tk.length(), m_formats[CppLabelFormat]);
 
         else if (tk.is(T_IDENTIFIER))
-            highlightWord(text.midRef(tk.position(), tk.length()), tk.position(), tk.length());
+            highlightWord(text.midRef(tk.begin(), tk.length()), tk.begin(), tk.length());
 
     }
 
     // mark the trailing white spaces
     {
-        const SimpleToken tk = tokens.last();
+        const Token tk = tokens.last();
-        const int lastTokenEnd = tk.position() + tk.length();
+        const int lastTokenEnd = tk.begin() + tk.length();
         if (text.length() > lastTokenEnd)
             highlightLine(text, lastTokenEnd, text.length() - lastTokenEnd, QTextCharFormat());
     }
 
     if (! initialState && state && ! tokens.isEmpty()) {
         parentheses.append(Parenthesis(Parenthesis::Opened, QLatin1Char('+'),
-                                       tokens.last().position()));
+                                       tokens.last().begin()));
         ++braceDepth;
     }
 

src/plugins/cpptools/cppcodecompletion.cpp

@@ -319,9 +319,9 @@ void FunctionArgumentWidget::updateArgumentHighlight()
     int argnr = 0;
     int parcount = 0;
     SimpleLexer tokenize;
-    QList<SimpleToken> tokens = tokenize(str);
+    QList<Token> tokens = tokenize(str);
     for (int i = 0; i < tokens.count(); ++i) {
-        const SimpleToken &tk = tokens.at(i);
+        const Token &tk = tokens.at(i);
         if (tk.is(T_LPAREN))
             ++parcount;
         else if (tk.is(T_RPAREN))
@@ -555,9 +555,9 @@ static int startOfOperator(TextEditor::ITextEditable *editor,
     SimpleLexer tokenize;
     tokenize.setQtMocRunEnabled(true);
     tokenize.setSkipComments(false);
-    const QList<SimpleToken> &tokens = tokenize(tc.block().text());
+    const QList<Token> &tokens = tokenize(tc.block().text());
     const int tokenIdx = SimpleLexer::tokenAt(tokens, tc.positionInBlock());
-    const SimpleToken &tk = (tokenIdx == -1) ? SimpleToken() : tokens.at(tokenIdx);
+    const Token &tk = (tokenIdx == -1) ? Token() : tokens.at(tokenIdx);
 
     if (completionKind == T_DOXY_COMMENT && !(tk.is(T_DOXY_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))) {
         completionKind = T_EOF_SYMBOL;
@@ -579,11 +579,11 @@ static int startOfOperator(TextEditor::ITextEditable *editor,
     else if (completionKind == T_LPAREN) {
         int i = 0;
         for (; i < tokens.size(); ++i) {
-            const SimpleToken &token = tokens.at(i);
+            const Token &token = tokens.at(i);
-            if (token.position() == tk.position()) {
+            if (token.begin() == tk.begin()) {
                 if (i == 0) // no token on the left, but might be on a previous line
                     break;
-                const SimpleToken &previousToken = tokens.at(i - 1);
+                const Token &previousToken = tokens.at(i - 1);
                 if (previousToken.is(T_IDENTIFIER) || previousToken.is(T_GREATER)
                     || previousToken.is(T_SIGNAL) || previousToken.is(T_SLOT))
                     break;
@@ -601,8 +601,8 @@ static int startOfOperator(TextEditor::ITextEditable *editor,
         if (tokens.size() >= 3) {
             if (tokens.at(0).is(T_POUND) && tokens.at(1).is(T_IDENTIFIER) && (tokens.at(2).is(T_STRING_LITERAL) ||
                                                                               tokens.at(2).is(T_ANGLE_STRING_LITERAL))) {
-                const SimpleToken &directiveToken = tokens.at(1);
+                const Token &directiveToken = tokens.at(1);
-                QString directive = tc.block().text().mid(directiveToken.position(),
+                QString directive = tc.block().text().mid(directiveToken.begin(),
                                                           directiveToken.length());
                 if (directive == QLatin1String("include") ||
                     directive == QLatin1String("include_next") ||