Introduced token caching to prevent repetetive tokenizing.

Also removed TokenUnderCursor as it's functionality is in the token cache.

Reviewed-by: ckamm

src/libs/cplusplus/BackwardsScanner.cpp

@@ -27,33 +27,52 @@
 **
 **************************************************************************/
 #include "BackwardsScanner.h"
+#include "TokenCache.h"
 #include <Token.h>
 #include <QtGui/QTextCursor>
+#include <QTextDocument>
 
 using namespace CPlusPlus;
 
-BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, const QString &suffix, int maxBlockCount)
-    : _offset(0)
+BackwardsScanner::BackwardsScanner(TokenCache *tokenCache, const QTextCursor &cursor, int maxBlockCount, const QString &suffix)
+    : _tokenCache(tokenCache)
+    , _offset(0)
     , _blocksTokenized(0)
     , _block(cursor.block())
     , _maxBlockCount(maxBlockCount)
 {
-    _tokenize.setQtMocRunEnabled(true);
-    _tokenize.setSkipComments(true);
-    _tokenize.setObjCEnabled(true);
-    _text = _block.text().left(cursor.position() - cursor.block().position());
-
-    if (! suffix.isEmpty())
+    int pos = cursor.position() - cursor.block().position();
+    _text = _block.text().left(pos);
     _text += suffix;
 
-    _tokens.append(_tokenize(_text, previousBlockState(_block)));
+    _tokens.append(tokenCache->tokensForBlock(_block));
+
+    for (int i = _tokens.size() - 1; i >= 0; --i) {
+        const int tokenEnd = _tokens.at(i).end();
+
+        if ((tokenEnd < pos) ||
+                (tokenEnd == pos && suffix.isEmpty())) {
+            break;
+        } else {
+            _tokens.removeAt(i);
+        }
+    }
+
+    QString remainingText;
+    if (!_tokens.isEmpty())
+        remainingText = _text.mid(_tokens.last().end());
+    if (!remainingText.isEmpty()) {
+        SimpleLexer tokenize;
+        tokenize.setQtMocRunEnabled(true);
+        tokenize.setSkipComments(true);
+        tokenize.setObjCEnabled(true);
+
+        _tokens.append(tokenize(remainingText, TokenCache::previousBlockState(_block)));
+    }
 
     _startToken = _tokens.size();
 }
 
-int BackwardsScanner::state() const
-{ return _tokenize.state(); }
-
 SimpleToken BackwardsScanner::LA(int index) const
 { return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); }
 
@@ -71,6 +90,7 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
         } else {
             ++_blocksTokenized;
 
+            QList<SimpleToken> newTokens = _tokenCache->tokensForBlock(_block);
             QString blockText = _block.text();
             _text.prepend(QLatin1Char('\n'));
             _text.prepend(blockText);
@@ -79,12 +99,11 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
             for (int i = 0; i < _tokens.size(); ++i) {
                 SimpleToken t = _tokens.at(i);
                 t.setPosition(t.position() + blockText.length() + 1);
-                t.setText(_text.midRef(t.position(), t.length()));
                 adaptedTokens.append(t);
             }
 
-            _tokens = _tokenize(blockText, previousBlockState(_block));
-            _offset += _tokens.size();
+            _tokens = newTokens;
+            _offset += newTokens.size();
             _tokens += adaptedTokens;
         }
     }
@@ -119,20 +138,6 @@ QStringRef BackwardsScanner::textRef(int index) const
     return _text.midRef(firstToken.begin(), firstToken.length());
 }
 
-int BackwardsScanner::previousBlockState(const QTextBlock &block)
-{
-    const QTextBlock prevBlock = block.previous();
-
-    if (prevBlock.isValid()) {
-        int state = prevBlock.userState();
-
-        if (state != -1)
-            return state;
-    }
-
-    return 0;
-}
-
 int BackwardsScanner::size() const
 {
     return _tokens.size();

src/libs/cplusplus/BackwardsScanner.h

@@ -36,16 +36,18 @@
 
 namespace CPlusPlus {
 
+class TokenCache;
+
 class CPLUSPLUS_EXPORT BackwardsScanner
 {
     enum { MAX_BLOCK_COUNT = 10 };
 
 public:
-    BackwardsScanner(const QTextCursor &cursor,
-                     const QString &suffix = QString(),
-                     int maxBlockCount = MAX_BLOCK_COUNT);
+    BackwardsScanner(TokenCache *cache,
+                     const QTextCursor &cursor,
+                     int maxBlockCount = MAX_BLOCK_COUNT,
+                     const QString &suffix = QString());
 
-    int state() const;
     int startToken() const;
 
     int startPosition() const;
@@ -67,20 +69,18 @@ public:
     int startOfMatchingBrace(int index) const;
     int startOfBlock(int index) const;
 
-    static int previousBlockState(const QTextBlock &block);
-
     int size() const;
 
 private:
     const SimpleToken &fetchToken(int tokenIndex);
 
 private:
+    TokenCache *_tokenCache;
     QList<SimpleToken> _tokens;
     int _offset;
     int _blocksTokenized;
     QTextBlock _block;
     QString _text;
-    SimpleLexer _tokenize;
     int _maxBlockCount;
     int _startToken;
 };

src/libs/cplusplus/ExpressionUnderCursor.cpp

@@ -30,6 +30,7 @@
 #include "ExpressionUnderCursor.h"
 #include "SimpleLexer.h"
 #include "BackwardsScanner.h"
+#include "TokenCache.h"
 #include <Token.h>
 
 #include <QTextCursor>
@@ -37,8 +38,8 @@
 
 using namespace CPlusPlus;
 
-ExpressionUnderCursor::ExpressionUnderCursor()
-    : _jumpedComma(false)
+ExpressionUnderCursor::ExpressionUnderCursor(TokenCache *tokenCache)
+    : _tokenCache(tokenCache), _jumpedComma(false)
 { }
 
 ExpressionUnderCursor::~ExpressionUnderCursor()
@@ -218,7 +219,7 @@ bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
 
 QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
 {
-    BackwardsScanner scanner(cursor);
+    BackwardsScanner scanner(_tokenCache, cursor);
 
     _jumpedComma = false;
 
@@ -232,7 +233,7 @@ QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
 
 int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
 {
-    BackwardsScanner scanner(cursor);
+    BackwardsScanner scanner(_tokenCache, cursor);
 
     int index = scanner.startToken();
 

src/libs/cplusplus/ExpressionUnderCursor.h

@@ -43,11 +43,12 @@ namespace CPlusPlus {
 
 class BackwardsScanner;
 class SimpleToken;
+class TokenCache;
 
 class CPLUSPLUS_EXPORT ExpressionUnderCursor
 {
 public:
-    ExpressionUnderCursor();
+    ExpressionUnderCursor(TokenCache *tokenCache);
     ~ExpressionUnderCursor();
 
     QString operator()(const QTextCursor &cursor);
@@ -59,6 +60,7 @@ private:
     bool isAccessToken(const SimpleToken &tk);
 
 private:
+    TokenCache *_tokenCache;
     bool _jumpedComma;
 };
 

src/libs/cplusplus/MatchingText.cpp

@@ -28,6 +28,7 @@
 **************************************************************************/
 #include "MatchingText.h"
 #include "BackwardsScanner.h"
+#include "TokenCache.h"
 
 #include <Token.h>
 
@@ -75,7 +76,8 @@ static bool isCompleteCharLiteral(const BackwardsScanner &tk, int index)
     return false;
 }
 
-MatchingText::MatchingText()
+MatchingText::MatchingText(TokenCache *tokenCache)
+    : _tokenCache(tokenCache)
 { }
 
 bool MatchingText::shouldInsertMatchingText(const QTextCursor &tc)
@@ -151,7 +153,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
     if (text.isEmpty() || !shouldInsertMatchingText(la))
         return QString();
 
-    BackwardsScanner tk(tc, textToProcess.left(*skippedChars), MAX_NUM_LINES);
+    BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES, textToProcess.left(*skippedChars));
     const int startToken = tk.startToken();
     int index = startToken;
 
@@ -211,7 +213,7 @@ bool MatchingText::shouldInsertNewline(const QTextCursor &tc) const
 
 QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
 {
-    BackwardsScanner tk(tc, QString(), MAX_NUM_LINES);
+    BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES);
     int index = tk.startToken();
 
     if (tk[index - 1].isNot(T_LBRACE))

src/libs/cplusplus/MatchingText.h

@@ -35,11 +35,12 @@
 namespace CPlusPlus {
 
 class BackwardsScanner;
+class TokenCache;
 
 class CPLUSPLUS_EXPORT MatchingText
 {
 public:
-    MatchingText();
+    MatchingText(TokenCache *tokenCache);
 
     static bool shouldInsertMatchingText(const QTextCursor &tc);
     static bool shouldInsertMatchingText(const QChar &lookAhead);
@@ -50,6 +51,8 @@ public:
 
 private:
     bool shouldInsertNewline(const QTextCursor &tc) const;
+
+    TokenCache *_tokenCache;
 };
 
 } // end of namespace CPlusPlus

src/libs/cplusplus/SimpleLexer.cpp

@@ -37,12 +37,11 @@
 
 using namespace CPlusPlus;
 
-SimpleToken::SimpleToken(const Token &token, const QStringRef &text)
+SimpleToken::SimpleToken(const Token &token)
     : _kind(token.f.kind)
     , _flags(0)
     , _position(token.begin())
     , _length(token.f.length)
-    , _text(text)
 {
     f._whitespace = token.f.whitespace;
     f._newline = token.f.newline;
@@ -148,17 +147,17 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
             break;
 
         QStringRef spell = text.midRef(lex.tokenOffset(), lex.tokenLength());
-        SimpleToken simpleTk(tk, spell);
+        SimpleToken simpleTk(tk);
         lex.setScanAngleStringLiteralTokens(false);
 
         if (tk.f.newline && tk.is(T_POUND))
             inPreproc = true;
         else if (inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
-                 simpleTk.text() == QLatin1String("include"))
+                 spell == QLatin1String("include"))
             lex.setScanAngleStringLiteralTokens(true);
         else if (_objCEnabled
                  && inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
-                 simpleTk.text() == QLatin1String("import"))
+                 spell == QLatin1String("import"))
             lex.setScanAngleStringLiteralTokens(true);
 
         if (_objCEnabled && tk.is(T_IDENTIFIER))

src/libs/cplusplus/SimpleLexer.h

@@ -42,7 +42,7 @@ class Token;
 class CPLUSPLUS_EXPORT SimpleToken
 {
 public:
-    SimpleToken(const Token &token, const QStringRef &text);
+    SimpleToken(const Token &token);
 
     SimpleToken()
         : _kind(0)
@@ -66,9 +66,6 @@ public:
     inline int end() const
     { return _position + _length; }
 
-    inline QStringRef text() const
-    { return _text; }
-
     inline bool followsNewline() const
     { return f._newline; }
 
@@ -91,10 +88,6 @@ public:
     inline void setPosition(int position)
     { _position = position; }
 
-    // internal
-    inline void setText(const QStringRef &text)
-    { _text = text; }
-
 public:
     short _kind;
     union {
@@ -109,7 +102,6 @@ public:
 
     int _position;
     int _length;
-    QStringRef _text;
 
     friend class SimpleLexer;
 };

src/libs/cplusplus/TokenCache.cpp

@@ -0,0 +1,81 @@
+#include "SimpleLexer.h"
+#include "TokenCache.h"
+
+using namespace CPlusPlus;
+
+TokenCache::TokenCache()
+    : m_doc(0)
+    , m_revision(-1)
+{}
+
+void TokenCache::setDocument(QTextDocument *doc)
+{
+    m_doc = doc;
+    m_revision = -1;
+}
+
+QList<SimpleToken> TokenCache::tokensForBlock(const QTextBlock &block) const
+{
+    Q_ASSERT(m_doc);
+    Q_ASSERT(m_doc == block.document());
+
+    const int documentRevision = m_doc->revision();
+
+    if (documentRevision != m_revision) {
+        m_tokensByBlock.clear();
+        m_revision = documentRevision;
+    }
+
+    const int blockNr = block.blockNumber();
+
+    if (m_tokensByBlock.contains(blockNr)) {
+        return m_tokensByBlock.value(blockNr);
+    } else {
+
+        SimpleLexer tokenize;
+        tokenize.setObjCEnabled(true);
+        tokenize.setQtMocRunEnabled(true);
+        tokenize.setSkipComments(false);
+
+        const int prevState = previousBlockState(block);
+        QList<SimpleToken> tokens = tokenize(block.text(), prevState);
+        m_tokensByBlock.insert(blockNr, tokens);
+
+        return tokens;
+    }
+}
+
+SimpleToken TokenCache::tokenUnderCursor(const QTextCursor &cursor) const
+{
+    const QTextBlock block = cursor.block();
+    const QList<SimpleToken> tokens = tokensForBlock(block);
+    const int column = cursor.position() - block.position();
+
+    for (int index = tokens.size() - 1; index >= 0; --index) {
+        const SimpleToken &tk = tokens.at(index);
+        if (tk.position() < column)
+            return tk;
+    }
+
+    return SimpleToken();
+}
+
+QString TokenCache::text(const QTextBlock &block, int tokenIndex) const
+{
+    SimpleToken tk = tokensForBlock(block).at(tokenIndex);
+    return block.text().mid(tk.position(), tk.length());
+}
+
+int TokenCache::previousBlockState(const QTextBlock &block)
+{
+    const QTextBlock prevBlock = block.previous();
+
+    if (prevBlock.isValid()) {
+        int state = prevBlock.userState();
+
+        if (state != -1)
+            return state;
+    }
+
+    return 0;
+}

src/libs/cplusplus/TokenCache.h

@@ -0,0 +1,39 @@
+#ifndef TOKENCACHE_H
+#define TOKENCACHE_H
+
+#include <CPlusPlusForwardDeclarations.h>
+#include <cplusplus/SimpleLexer.h>
+
+#include <QtCore/QHash>
+#include <QtCore/QList>
+
+#include <QtGui/QTextBlock>
+#include <QtGui/QTextCursor>
+#include <QtGui/QTextDocument>
+
+namespace CPlusPlus {
+
+class CPLUSPLUS_EXPORT TokenCache
+{
+public:
+    TokenCache();
+
+    void setDocument(QTextDocument *doc);
+
+    QList<CPlusPlus::SimpleToken> tokensForBlock(const QTextBlock &block) const;
+    CPlusPlus::SimpleToken tokenUnderCursor(const QTextCursor &cursor) const;
+
+    QString text(const QTextBlock &block, int tokenIndex) const;
+
+    static int previousBlockState(const QTextBlock &block);
+
+private:
+    QTextDocument *m_doc;
+
+    mutable int m_revision;
+    mutable QHash<int, QList<CPlusPlus::SimpleToken> > m_tokensByBlock;
+};
+
+} // namespace CPlusPlus
+
+#endif // TOKENCACHE_H

src/libs/cplusplus/TokenUnderCursor.cpp

@@ -1,67 +0,0 @@
-/**************************************************************************
-**
-** This file is part of Qt Creator
-**
-** Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
-**
-** Contact: Nokia Corporation (qt-info@nokia.com)
-**
-** Commercial Usage
-**
-** Licensees holding valid Qt Commercial licenses may use this file in
-** accordance with the Qt Commercial License Agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and Nokia.
-**
-** GNU Lesser General Public License Usage
-**
-** Alternatively, this file may be used under the terms of the GNU Lesser
-** General Public License version 2.1 as published by the Free Software
-** Foundation and appearing in the file LICENSE.LGPL included in the
-** packaging of this file.  Please review the following information to
-** ensure the GNU Lesser General Public License version 2.1 requirements
-** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
-**
-** If you are unsure which license is appropriate for your use, please
-** contact the sales department at http://qt.nokia.com/contact.
-**
-**************************************************************************/
-
-#include "TokenUnderCursor.h"
-#include "BackwardsScanner.h"
-#include <Token.h>
-
-#include <QTextCursor>
-#include <QTextBlock>
-#include <climits>
-
-using namespace CPlusPlus;
-
-TokenUnderCursor::TokenUnderCursor()
-{ }
-
-TokenUnderCursor::~TokenUnderCursor()
-{ }
-
-SimpleToken TokenUnderCursor::operator()(const QTextCursor &cursor, QTextBlock *b)
-{
-    SimpleLexer tokenize;
-    tokenize.setObjCEnabled(true);
-    tokenize.setSkipComments(false);
-
-    QTextBlock block = cursor.block();
-    int column = cursor.position() - cursor.block().position();
-
-    _text = block.text();
-    _tokens = tokenize(_text, BackwardsScanner::previousBlockState(block));
-    for (int index = _tokens.size() - 1; index != -1; --index) {
-        const SimpleToken &tk = _tokens.at(index);
-        if (tk.position() < column) {
-            if (b)
-                *b = block;
-            return tk;
-        }
-    }
-
-    return SimpleToken();
-}

src/libs/cplusplus/TokenUnderCursor.h

@@ -1,61 +0,0 @@
-/**************************************************************************
-**
-** This file is part of Qt Creator
-**
-** Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
-**
-** Contact: Nokia Corporation (qt-info@nokia.com)
-**
-** Commercial Usage
-**
-** Licensees holding valid Qt Commercial licenses may use this file in
-** accordance with the Qt Commercial License Agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and Nokia.
-**
-** GNU Lesser General Public License Usage
-**
-** Alternatively, this file may be used under the terms of the GNU Lesser
-** General Public License version 2.1 as published by the Free Software
-** Foundation and appearing in the file LICENSE.LGPL included in the
-** packaging of this file.  Please review the following information to
-** ensure the GNU Lesser General Public License version 2.1 requirements
-** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
-**
-** If you are unsure which license is appropriate for your use, please
-** contact the sales department at http://qt.nokia.com/contact.
-**
-**************************************************************************/
-#ifndef CPLUSPLUS_TOKENUNDERCURSOR_H
-#define CPLUSPLUS_TOKENUNDERCURSOR_H
-
-#include "SimpleLexer.h"
-#include <QList>
-
-QT_BEGIN_NAMESPACE
-class QString;
-class QTextCursor;
-class QTextBlock;
-QT_END_NAMESPACE
-
-namespace CPlusPlus {
-
-class CPLUSPLUS_EXPORT TokenUnderCursor
-{
-public:
-    TokenUnderCursor();
-    ~TokenUnderCursor();
-
-    SimpleToken operator()(const QTextCursor &cursor, QTextBlock *block = 0);
-
-    const QList<SimpleToken> &tokens() const
-    { return _tokens; }
-
-private:
-    QList<SimpleToken> _tokens;
-    QString _text;
-};
-
-} // end of namespace CPlusPlus
-
-#endif // CPLUSPLUS_TOKENUNDERCURSOR_H

src/libs/cplusplus/cplusplus-lib.pri

@@ -12,18 +12,18 @@ contains(QT, gui) {
 HEADERS += \
     $$PWD/Icons.h \
     $$PWD/ExpressionUnderCursor.h \
-    $$PWD/TokenUnderCursor.h \
     $$PWD/BackwardsScanner.h \
     $$PWD/MatchingText.h \
-    $$PWD/OverviewModel.h
+    $$PWD/OverviewModel.h \
+    $$PWD/TokenCache.h
 
 SOURCES += \
     $$PWD/Icons.cpp \
     $$PWD/ExpressionUnderCursor.cpp \
-    $$PWD/TokenUnderCursor.cpp \
     $$PWD/BackwardsScanner.cpp \
     $$PWD/MatchingText.cpp \
-    $$PWD/OverviewModel.cpp
+    $$PWD/OverviewModel.cpp \
+    $$PWD/TokenCache.cpp
 }
 
 HEADERS += \

src/plugins/cppeditor/cppeditor.cpp

@@ -51,11 +51,11 @@
 #include <cplusplus/Overview.h>
 #include <cplusplus/OverviewModel.h>
 #include <cplusplus/SimpleLexer.h>
-#include <cplusplus/TokenUnderCursor.h>
 #include <cplusplus/MatchingText.h>
 #include <cplusplus/BackwardsScanner.h>
 #include <cplusplus/FastPreprocessor.h>
 #include <cplusplus/CheckUndefinedSymbols.h>
+#include <cplusplus/TokenCache.h>
 
 #include <cpptools/cppmodelmanagerinterface.h>
 
@@ -534,7 +534,7 @@ struct FindCanonicalSymbol
     SemanticInfo info;
 
     FindCanonicalSymbol(CPPEditor *editor, const SemanticInfo &info)
-        : editor(editor), info(info)
+        : editor(editor), expressionUnderCursor(editor->tokenCache()), info(info)
     {
         typeOfExpression.init(info.doc, info.snapshot);
     }
@@ -728,6 +728,11 @@ void CPPEditor::cut()
     finishRename();
 }
 
+TokenCache *CPPEditor::tokenCache() const
+{
+    return m_modelManager->tokenCache(editableInterface());
+}
+
 void CPPEditor::startRename()
 {
     m_inRenameChanged = false;
@@ -1207,7 +1212,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
     SimpleLexer tokenize;
     tokenize.setQtMocRunEnabled(true);
     const QString blockText = cursor.block().text();
-    const QList<SimpleToken> tokens = tokenize(blockText, BackwardsScanner::previousBlockState(cursor.block()));
+    const QList<SimpleToken> tokens = tokenize(blockText, TokenCache::previousBlockState(cursor.block()));
 
     bool recognizedQtMethod = false;
 
@@ -1254,10 +1259,8 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
     }
 
     if (! recognizedQtMethod) {
-        static TokenUnderCursor tokenUnderCursor;
-
-        QTextBlock block;
-        const SimpleToken tk = tokenUnderCursor(tc, &block);
+        const QTextBlock block = tc.block();
+        const SimpleToken tk = tokenCache()->tokenUnderCursor(tc);
 
         beginOfToken = block.position() + tk.begin();
         endOfToken = block.position() + tk.end();
@@ -1287,7 +1290,7 @@ CPPEditor::Link CPPEditor::findLinkAt(const QTextCursor &cursor,
         return link;
 
     // Evaluate the type of the expression under the cursor
-    ExpressionUnderCursor expressionUnderCursor;
+    ExpressionUnderCursor expressionUnderCursor(tokenCache());
     const QString expression = expressionUnderCursor(tc);
 
     TypeOfExpression typeOfExpression;
@@ -1386,13 +1389,13 @@ bool CPPEditor::isElectricCharacter(const QChar &ch) const
 QString CPPEditor::insertMatchingBrace(const QTextCursor &tc, const QString &text,
                                        const QChar &la, int *skippedChars) const
 {
-    MatchingText m;
+    MatchingText m(tokenCache());
     return m.insertMatchingBrace(tc, text, la, skippedChars);
 }
 
 QString CPPEditor::insertParagraphSeparator(const QTextCursor &tc) const
 {
-    MatchingText m;
+    MatchingText m(tokenCache());
     return m.insertParagraphSeparator(tc);
 }
 
@@ -1415,8 +1418,7 @@ bool CPPEditor::contextAllowsAutoParentheses(const QTextCursor &cursor,
 
 bool CPPEditor::isInComment(const QTextCursor &cursor) const
 {
-    CPlusPlus::TokenUnderCursor tokenUnderCursor;
-    const SimpleToken tk = tokenUnderCursor(cursor);
+    const SimpleToken tk = tokenCache()->tokenUnderCursor(cursor);
 
     if (tk.isComment()) {
         const int pos = cursor.selectionEnd() - cursor.block().position();
@@ -1470,7 +1472,7 @@ void CPPEditor::indentBlock(QTextDocument *doc, QTextBlock block, QChar typedCha
 
     const TabSettings &ts = tabSettings();
 
-    BackwardsScanner tk(tc, QString(), 400);
+    BackwardsScanner tk(tokenCache(), tc, 400);
     const int tokenCount = tk.startToken();
 
     if (tokenCount != 0) {

src/plugins/cppeditor/cppeditor.h

@@ -48,6 +48,7 @@ QT_END_NAMESPACE
 namespace CPlusPlus {
 class OverviewModel;
 class Symbol;
+class TokenCache;
 }
 
 namespace CppTools {
@@ -198,6 +199,8 @@ public:
     virtual void paste(); // reimplemented from BaseTextEditor
     virtual void cut(); // reimplemented from BaseTextEditor
 
+    CPlusPlus::TokenCache *tokenCache() const;
+
 public Q_SLOTS:
     virtual void setFontSettings(const TextEditor::FontSettings &);
     void setSortedMethodOverview(bool sort);

src/plugins/cppeditor/cpphighlighter.cpp

@@ -104,7 +104,7 @@ void CppHighlighter::highlightBlock(const QString &text)
         }
 
         if (tk.is(T_LPAREN) || tk.is(T_LBRACE) || tk.is(T_LBRACKET)) {
-            const QChar c(tk.text().at(0));
+            const QChar c = text.at(tk.position());
             parentheses.append(Parenthesis(Parenthesis::Opened, c, tk.position()));
             if (tk.is(T_LBRACE)) {
                 ++braceDepth;
@@ -117,7 +117,7 @@ void CppHighlighter::highlightBlock(const QString &text)
                 }
             }
         } else if (tk.is(T_RPAREN) || tk.is(T_RBRACE) || tk.is(T_RBRACKET)) {
-            const QChar c(tk.text().at(0));
+            const QChar c = text.at(tk.position());
             parentheses.append(Parenthesis(Parenthesis::Closed, c, tk.position()));
             if (tk.is(T_RBRACE)) {
                 --braceDepth;
@@ -141,7 +141,7 @@ void CppHighlighter::highlightBlock(const QString &text)
             highlightAsPreprocessor = true;
 
         } else if (highlightCurrentWordAsPreprocessor &&
-                   (tk.isKeyword() || tk.is(T_IDENTIFIER)) && isPPKeyword(tk.text()))
+                   (tk.isKeyword() || tk.is(T_IDENTIFIER)) && isPPKeyword(text.midRef(tk.position(), tk.length())))
             setFormat(tk.position(), tk.length(), m_formats[CppPreprocessorFormat]);
 
         else if (tk.is(T_NUMERIC_LITERAL))
@@ -180,7 +180,7 @@ void CppHighlighter::highlightBlock(const QString &text)
                 initialState = 0;
             }
 
-        } else if (tk.isKeyword() || isQtKeyword(tk.text()) || tk.isObjCAtKeyword() || tk.isObjCTypeQualifier())
+        } else if (tk.isKeyword() || isQtKeyword(text.midRef(tk.position(), tk.length())) || tk.isObjCAtKeyword() || tk.isObjCTypeQualifier())
             setFormat(tk.position(), tk.length(), m_formats[CppKeywordFormat]);
 
         else if (tk.isOperator())
@@ -190,7 +190,7 @@ void CppHighlighter::highlightBlock(const QString &text)
             setFormat(tk.position(), tk.length(), m_formats[CppLabelFormat]);
 
         else if (tk.is(T_IDENTIFIER))
-            highlightWord(tk.text(), tk.position(), tk.length());
+            highlightWord(text.midRef(tk.position(), tk.length()), tk.position(), tk.length());
 
     }
 

src/plugins/cppeditor/cpphoverhandler.cpp

@@ -232,7 +232,7 @@ void CppHoverHandler::updateHelpIdAndTooltip(TextEditor::ITextEditor *editor, in
         }
 
         // Fetch the expression's code
-        ExpressionUnderCursor expressionUnderCursor;
+        ExpressionUnderCursor expressionUnderCursor(m_modelManager->tokenCache(editor));
         const QString expression = expressionUnderCursor(tc);
 
         const QList<LookupItem> types = typeOfExpression(expression, scope);

src/plugins/cppeditor/cpphoverhandler.h

@@ -36,6 +36,10 @@ QT_BEGIN_NAMESPACE
 class QPoint;
 QT_END_NAMESPACE
 
+namespace CPlusPlus {
+class TokenCache;
+}
+
 namespace Core {
 class IEditor;
 }

src/plugins/cpptools/cppcodecompletion.cpp

@@ -50,7 +50,6 @@
 #include <cplusplus/Overview.h>
 #include <cplusplus/ExpressionUnderCursor.h>
 #include <cplusplus/BackwardsScanner.h>
-#include <cplusplus/TokenUnderCursor.h>
 #include <cplusplus/LookupContext.h>
 
 #include <coreplugin/icore.h>
@@ -65,7 +64,6 @@
 #include <utils/faketooltip.h>
 #include <utils/qtcassert.h>
 
-#include <QtCore/QDebug>
 #include <QtCore/QMap>
 #include <QtCore/QFile>
 #include <QtGui/QAction>
@@ -454,7 +452,8 @@ QIcon CppCodeCompletion::iconForSymbol(Symbol *symbol) const
 /*
   Searches backwards for an access operator.
 */
-static int startOfOperator(TextEditor::ITextEditable *editor,
+static int startOfOperator(TokenCache *tokenCache,
+                           TextEditor::ITextEditable *editor,
                            int pos, unsigned *kind,
                            bool wantFunctionCall)
 {
@@ -547,15 +546,14 @@ static int startOfOperator(TextEditor::ITextEditable *editor,
     }
 
     if (completionKind == T_COMMA) {
-        ExpressionUnderCursor expressionUnderCursor;
+        ExpressionUnderCursor expressionUnderCursor(tokenCache);
         if (expressionUnderCursor.startOfFunctionCall(tc) == -1) {
             completionKind = T_EOF_SYMBOL;
             start = pos;
         }
     }
 
-    static CPlusPlus::TokenUnderCursor tokenUnderCursor;
-    const SimpleToken tk = tokenUnderCursor(tc);
+    const SimpleToken tk = tokenCache->tokenUnderCursor(tc);
 
     if (completionKind == T_DOXY_COMMENT && !(tk.is(T_DOXY_COMMENT) || tk.is(T_CPP_DOXY_COMMENT))) {
         completionKind = T_EOF_SYMBOL;
@@ -575,7 +573,7 @@ static int startOfOperator(TextEditor::ITextEditable *editor,
         start = pos;
     }
     else if (completionKind == T_LPAREN) {
-        const QList<SimpleToken> &tokens = tokenUnderCursor.tokens();
+        const QList<SimpleToken> &tokens = tokenCache->tokensForBlock(tc.block());
         int i = 0;
         for (; i < tokens.size(); ++i) {
             const SimpleToken &token = tokens.at(i);
@@ -597,11 +595,11 @@ static int startOfOperator(TextEditor::ITextEditable *editor,
     // Check for include preprocessor directive
     else if (completionKind == T_STRING_LITERAL || completionKind == T_ANGLE_STRING_LITERAL || completionKind == T_SLASH) {
         bool include = false;
-        const QList<SimpleToken> &tokens = tokenUnderCursor.tokens();
+        const QList<SimpleToken> &tokens = tokenCache->tokensForBlock(tc.block());
         if (tokens.size() >= 3) {
             if (tokens.at(0).is(T_POUND) && tokens.at(1).is(T_IDENTIFIER) && (tokens.at(2).is(T_STRING_LITERAL) ||
                                                                               tokens.at(2).is(T_ANGLE_STRING_LITERAL))) {
-                QStringRef directive = tokens.at(1).text();
+                QString directive = tokenCache->text(tc.block(), 1);
                 if (directive == QLatin1String("include") ||
                     directive == QLatin1String("include_next") ||
                     directive == QLatin1String("import")) {
@@ -634,9 +632,10 @@ int CppCodeCompletion::startPosition() const
 bool CppCodeCompletion::triggersCompletion(TextEditor::ITextEditable *editor)
 {
     const int pos = editor->position();
+    TokenCache *tokenCache = m_manager->tokenCache(editor);
     unsigned token = T_EOF_SYMBOL;
 
-    if (startOfOperator(editor, pos, &token, /*want function call=*/ true) != pos) {
+    if (startOfOperator(tokenCache, editor, pos, &token, /*want function call=*/ true) != pos) {
         if (token == T_POUND) {
             if (TextEditor::BaseTextEditor *edit = qobject_cast<TextEditor::BaseTextEditor *>(editor->widget())) {
                 QTextCursor tc(edit->document());
@@ -684,7 +683,8 @@ int CppCodeCompletion::startCompletionHelper(TextEditor::ITextEditable *editor)
     while (editor->characterAt(endOfOperator - 1).isSpace())
         --endOfOperator;
 
-    int endOfExpression = startOfOperator(editor, endOfOperator,
+    TokenCache *tokenCache = m_manager->tokenCache(editor);
+    int endOfExpression = startOfOperator(tokenCache, editor, endOfOperator,
                                           &m_completionOperator,
                                           /*want function call =*/ true);
 
@@ -725,7 +725,7 @@ int CppCodeCompletion::startCompletionHelper(TextEditor::ITextEditable *editor)
         return m_startPosition;
     }
 
-    ExpressionUnderCursor expressionUnderCursor;
+    ExpressionUnderCursor expressionUnderCursor(m_manager->tokenCache(editor));
     QTextCursor tc(edit->document());
 
     if (m_completionOperator == T_COMMA) {
@@ -800,14 +800,8 @@ int CppCodeCompletion::startCompletionInternal(TextEditor::BaseTextEditor *edit,
         }
     }
 
-    if (debug)
-        qDebug() << "scope:" << scope->owner()->fileName() << scope->owner()->line() << scope->owner()->column();
-
     QList<LookupItem> results = typeOfExpression(expression, scope, TypeOfExpression::Preprocess);
 
-    if (debug)
-        qDebug() << "got:" << results.size() << "results";
-
     if (results.isEmpty()) {
         if (m_completionOperator == T_SIGNAL || m_completionOperator == T_SLOT) {
             if (! (expression.isEmpty() || expression == QLatin1String("this"))) {
@@ -828,7 +822,8 @@ int CppCodeCompletion::startCompletionInternal(TextEditor::BaseTextEditor *edit,
             QTextCursor tc(edit->document());
             tc.setPosition(index);
 
-            ExpressionUnderCursor expressionUnderCursor;
+            TokenCache *tokenCache = m_manager->tokenCache(edit->editableInterface());
+            ExpressionUnderCursor expressionUnderCursor(tokenCache);
             const QString baseExpression = expressionUnderCursor(tc);
 
             // Resolve the type of this expression
@@ -1084,7 +1079,8 @@ bool CppCodeCompletion::completeConstructorOrFunction(const QList<LookupItem> &r
 
             QTextCursor tc(edit->document());
             tc.setPosition(endOfExpression);
-            BackwardsScanner bs(tc);
+            TokenCache *tokenCache = m_manager->tokenCache(m_editor);
+            BackwardsScanner bs(tokenCache, tc);
             const int startToken = bs.startToken();
             const int lineStartToken = bs.startOfLine(startToken);
             // make sure the required tokens are actually available
@@ -1143,8 +1139,8 @@ bool CppCodeCompletion::completeMember(const QList<LookupItem> &baseResults)
 {
     const LookupContext &context = typeOfExpression.context();
 
-    if (debug)
-        qDebug() << Q_FUNC_INFO << __LINE__;
+//    if (debug)
+//        qDebug() << Q_FUNC_INFO << __LINE__;
 
     if (baseResults.isEmpty())
         return false;
@@ -1156,8 +1152,8 @@ bool CppCodeCompletion::completeMember(const QList<LookupItem> &baseResults)
     if (ClassOrNamespace *binding = resolveExpression.baseExpression(baseResults,
                                                                      m_completionOperator,
                                                                      &replacedDotOperator)) {
-        if (debug)
-            qDebug() << "cool we got a binding for the base expression";
+//        if (debug)
+//            qDebug() << "cool we got a binding for the base expression";
 
         if (replacedDotOperator && binding) {
             // Replace . with ->
@@ -1173,10 +1169,10 @@ bool CppCodeCompletion::completeMember(const QList<LookupItem> &baseResults)
         return ! m_completions.isEmpty();
     }
 
-    if (debug) {
-        Overview oo;
-        qDebug() << "hmm, got:" << oo(baseResults.first().type());
-    }
+//    if (debug) {
+//        Overview oo;
+//        qDebug() << "hmm, got:" << oo(baseResults.first().type());
+//    }
 
     return false;
 }

src/plugins/cpptools/cppmodelmanager.cpp

@@ -962,6 +962,9 @@ bool CppModelManager::isCppEditor(Core::IEditor *editor) const
     return editor->context().contains(uid);
 }
 
+TokenCache *CppModelManager::tokenCache(TextEditor::ITextEditor *editor) const
+{ return editorSupport(editor)->tokenCache(); }
+
 void CppModelManager::emitDocumentUpdated(Document::Ptr doc)
 { emit documentUpdated(doc); }
 

src/plugins/cpptools/cppmodelmanager.h

@@ -108,6 +108,8 @@ public:
     CppEditorSupport *editorSupport(TextEditor::ITextEditor *editor) const
     { return m_editorSupport.value(editor); }
 
+    virtual CPlusPlus::TokenCache *tokenCache(TextEditor::ITextEditor *editor) const;
+
     void emitDocumentUpdated(CPlusPlus::Document::Ptr doc);
 
     void stopEditorSelectionsUpdate()

src/plugins/cpptools/cppmodelmanagerinterface.h

@@ -44,16 +44,25 @@ namespace Core {
 
 namespace CPlusPlus {
     class LookupContext;
+    class TokenCache;
 }
 
 namespace ProjectExplorer {
     class Project;
 }
 
+namespace TextEditor {
+    class ITextEditor;
+}
+
 namespace CppTools {
 
 class AbstractEditorSupport;
 
+namespace Internal {
+class CppEditorSupport;
+}
+
 class CPPTOOLS_EXPORT CppModelManagerInterface : public QObject
 {
     Q_OBJECT
@@ -136,6 +145,8 @@ public:
 
     virtual void findMacroUsages(const CPlusPlus::Macro &macro) = 0;
 
+    virtual CPlusPlus::TokenCache *tokenCache(TextEditor::ITextEditor *editor) const = 0;
+
 Q_SIGNALS:
     void documentUpdated(CPlusPlus::Document::Ptr doc);
 

src/plugins/cpptools/cpptoolseditorsupport.cpp

@@ -67,8 +67,12 @@ void CppEditorSupport::setTextEditor(TextEditor::ITextEditor *textEditor)
 {
     _textEditor = textEditor;
 
-    if (! _textEditor)
+    if (_textEditor) {
+        if (TextEditor::BaseTextEditor *ed = qobject_cast<TextEditor::BaseTextEditor *>(_textEditor->widget()))
+            _tokenCache.setDocument(ed->document());
+    } else {
         return;
+    }
 
     connect(_textEditor, SIGNAL(contentsChanged()), this, SIGNAL(contentsChanged()));
     connect(this, SIGNAL(contentsChanged()), this, SLOT(updateDocument()));
@@ -96,6 +100,11 @@ unsigned CppEditorSupport::editorRevision() const
     return 0;
 }
 
+TokenCache *CppEditorSupport::tokenCache()
+{
+    return &_tokenCache;
+}
+
 int CppEditorSupport::updateDocumentInterval() const
 { return _updateDocumentInterval; }
 

src/plugins/cpptools/cpptoolseditorsupport.h

@@ -36,6 +36,7 @@
 #include <QSharedPointer>
 #include <QTextCursor>
 #include <cplusplus/CppDocument.h>
+#include <cplusplus/TokenCache.h>
 
 QT_BEGIN_NAMESPACE
 class QTimer;
@@ -72,6 +73,8 @@ public:
     QString contents();
     unsigned editorRevision() const;
 
+    CPlusPlus::TokenCache *tokenCache();
+
 Q_SIGNALS:
     void contentsChanged();
 
@@ -89,6 +92,7 @@ private:
     QFuture<void> _documentParser;
     QString _cachedContents;
     unsigned _revision;
+    CPlusPlus::TokenCache _tokenCache;
 };
 
 } // namespace Internal

src/plugins/debugger/watchutils.cpp

@@ -736,7 +736,8 @@ QString cppExpressionAt(TextEditor::ITextEditor *editor, int pos,
         return QString();
 
     QString expr = plaintext->textCursor().selectedText();
-    if (expr.isEmpty()) {
+    CppTools::CppModelManagerInterface *modelManager = ExtensionSystem::PluginManager::instance()->getObject<CppTools::CppModelManagerInterface>();
+    if (expr.isEmpty() && modelManager) {
         QTextCursor tc(plaintext->document());
         tc.setPosition(pos);
 
@@ -745,7 +746,7 @@ QString cppExpressionAt(TextEditor::ITextEditor *editor, int pos,
             tc.movePosition(QTextCursor::EndOfWord);
 
         // Fetch the expression's code.
-        CPlusPlus::ExpressionUnderCursor expressionUnderCursor;
+        CPlusPlus::ExpressionUnderCursor expressionUnderCursor(modelManager->tokenCache(editor));
         expr = expressionUnderCursor(tc);
         *column = tc.columnNumber();
         *line = tc.blockNumber();
@@ -757,7 +758,7 @@ QString cppExpressionAt(TextEditor::ITextEditor *editor, int pos,
 
     if (function && !expr.isEmpty())
         if (const Core::IFile *file = editor->file())
-            if (CppTools::CppModelManagerInterface *modelManager = ExtensionSystem::PluginManager::instance()->getObject<CppTools::CppModelManagerInterface>())
+            if (modelManager)
                 *function = CppTools::AbstractEditorSupport::functionAt(modelManager, file->fileName(), *line, *column);
 
     return expr;