Introduced a token cache for the C++ editor.

This should speed things up a bit, because before, the line was tokenized at
least 3 times.

src/libs/cplusplus/BackwardsScanner.cpp

@@ -27,33 +27,48 @@
 **
 **************************************************************************/
 #include "BackwardsScanner.h"
+#include "TokenCache.h"
 #include <Token.h>
 #include <QtGui/QTextCursor>
+#include <QTextDocument>
 
 using namespace CPlusPlus;
 
-BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, const QString &suffix, int maxBlockCount)
-    : _offset(0)
+BackwardsScanner::BackwardsScanner(TokenCache *tokenCache, const QTextCursor &cursor, int maxBlockCount, const QString &suffix)
+    : _tokenCache(tokenCache)
+    , _offset(0)
     , _blocksTokenized(0)
     , _block(cursor.block())
     , _maxBlockCount(maxBlockCount)
 {
-    _tokenize.setQtMocRunEnabled(true);
-    _tokenize.setSkipComments(true);
-    _tokenize.setObjCEnabled(true);
-    _text = _block.text().left(cursor.position() - cursor.block().position());
+    int pos = cursor.position() - cursor.block().position();
+    _text = _block.text().left(pos);
+
+    if (suffix.isEmpty()) {
+        _tokens.append(tokenCache->tokensForBlock(_block));
+        int last = -1;
+        for (int i = _tokens.size() - 1; i >= 0; --i) {
+            if (_tokens.at(i).begin() < pos) {
+                last = i;
+                break;
+            }
+        }
+        for (int i = _tokens.size() - 1; i > last && i >= 0; --i)
+            _tokens.removeAt(i);
+    } else {
+        SimpleLexer tokenize;
+        tokenize.setQtMocRunEnabled(true);
+        tokenize.setSkipComments(true);
+        tokenize.setObjCEnabled(true);
 
-    if (! suffix.isEmpty())
         _text += suffix;
 
-    _tokens.append(_tokenize(_text, previousBlockState(_block)));
+        _tokens.append(tokenize(_text, TokenCache::previousBlockState(_block)));
+    }
 
     _startToken = _tokens.size();
 }
 
-int BackwardsScanner::state() const
-{ return _tokenize.state(); }
-
 SimpleToken BackwardsScanner::LA(int index) const
 { return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); }
 
@@ -83,7 +98,7 @@ const SimpleToken &BackwardsScanner::fetchToken(int i)
                 adaptedTokens.append(t);
             }
 
-            _tokens = _tokenize(blockText, previousBlockState(_block));
+            _tokens = _tokenCache->tokensForBlock(_block);
             _offset += _tokens.size();
             _tokens += adaptedTokens;
         }
@@ -119,20 +134,6 @@ QStringRef BackwardsScanner::textRef(int index) const
     return _text.midRef(firstToken.begin(), firstToken.length());
 }
 
-int BackwardsScanner::previousBlockState(const QTextBlock &block)
-{
-    const QTextBlock prevBlock = block.previous();
-
-    if (prevBlock.isValid()) {
-        int state = prevBlock.userState();
-
-        if (state != -1)
-            return state;
-    }
-
-    return 0;
-}
-
 int BackwardsScanner::size() const
 {
     return _tokens.size();

src/libs/cplusplus/BackwardsScanner.h

@@ -36,16 +36,18 @@
 
 namespace CPlusPlus {
 
+class TokenCache;
+
 class CPLUSPLUS_EXPORT BackwardsScanner
 {
     enum { MAX_BLOCK_COUNT = 10 };
 
 public:
-    BackwardsScanner(const QTextCursor &cursor,
-                     const QString &suffix = QString(),
-                     int maxBlockCount = MAX_BLOCK_COUNT);
+    BackwardsScanner(TokenCache *cache,
+                     const QTextCursor &cursor,
+                     int maxBlockCount = MAX_BLOCK_COUNT,
+                     const QString &suffix = QString());
 
-    int state() const;
     int startToken() const;
 
     int startPosition() const;
@@ -67,20 +69,18 @@ public:
     int startOfMatchingBrace(int index) const;
     int startOfBlock(int index) const;
 
-    static int previousBlockState(const QTextBlock &block);
-
     int size() const;
 
 private:
     const SimpleToken &fetchToken(int i);
 
 private:
+    TokenCache *_tokenCache;
     QList<SimpleToken> _tokens;
     int _offset;
     int _blocksTokenized;
     QTextBlock _block;
     QString _text;
-    SimpleLexer _tokenize;
     int _maxBlockCount;
     int _startToken;
 };

src/libs/cplusplus/ExpressionUnderCursor.cpp

@@ -30,6 +30,7 @@
 #include "ExpressionUnderCursor.h"
 #include "SimpleLexer.h"
 #include "BackwardsScanner.h"
+#include "TokenCache.h"
 #include <Token.h>
 
 #include <QTextCursor>
@@ -37,8 +38,8 @@
 
 using namespace CPlusPlus;
 
-ExpressionUnderCursor::ExpressionUnderCursor()
-    : _jumpedComma(false)
+ExpressionUnderCursor::ExpressionUnderCursor(TokenCache *tokenCache)
+    : _tokenCache(tokenCache), _jumpedComma(false)
 { }
 
 ExpressionUnderCursor::~ExpressionUnderCursor()
@@ -218,7 +219,7 @@ bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
 
 QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
 {
-    BackwardsScanner scanner(cursor);
+    BackwardsScanner scanner(_tokenCache, cursor);
 
     _jumpedComma = false;
 
@@ -232,7 +233,7 @@ QString ExpressionUnderCursor::operator()(const QTextCursor &cursor)
 
 int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
 {
-    BackwardsScanner scanner(cursor);
+    BackwardsScanner scanner(_tokenCache, cursor);
 
     int index = scanner.startToken();
 

src/libs/cplusplus/ExpressionUnderCursor.h

@@ -43,11 +43,12 @@ namespace CPlusPlus {
 
 class BackwardsScanner;
 class SimpleToken;
+class TokenCache;
 
 class CPLUSPLUS_EXPORT ExpressionUnderCursor
 {
 public:
-    ExpressionUnderCursor();
+    ExpressionUnderCursor(TokenCache *tokenCache);
     ~ExpressionUnderCursor();
 
     QString operator()(const QTextCursor &cursor);
@@ -59,6 +60,7 @@ private:
     bool isAccessToken(const SimpleToken &tk);
 
 private:
+    TokenCache *_tokenCache;
     bool _jumpedComma;
 };
 

src/libs/cplusplus/MatchingText.cpp

@@ -28,6 +28,7 @@
 **************************************************************************/
 #include "MatchingText.h"
 #include "BackwardsScanner.h"
+#include "TokenCache.h"
 
 #include <Token.h>
 
@@ -75,7 +76,8 @@ static bool isCompleteCharLiteral(const BackwardsScanner &tk, int index)
     return false;
 }
 
-MatchingText::MatchingText()
+MatchingText::MatchingText(TokenCache *tokenCache)
+    : _tokenCache(tokenCache)
 { }
 
 bool MatchingText::shouldInsertMatchingText(const QTextCursor &tc)
@@ -151,7 +153,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
     if (text.isEmpty() || !shouldInsertMatchingText(la))
         return QString();
 
-    BackwardsScanner tk(tc, textToProcess.left(*skippedChars), MAX_NUM_LINES);
+    BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES, textToProcess.left(*skippedChars));
     const int startToken = tk.startToken();
     int index = startToken;
 
@@ -211,7 +213,7 @@ bool MatchingText::shouldInsertNewline(const QTextCursor &tc) const
 
 QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
 {
-    BackwardsScanner tk(tc, QString(), MAX_NUM_LINES);
+    BackwardsScanner tk(_tokenCache, tc, MAX_NUM_LINES);
     int index = tk.startToken();
 
     if (tk[index - 1].isNot(T_LBRACE))

src/libs/cplusplus/MatchingText.h

@@ -35,11 +35,12 @@
 namespace CPlusPlus {
 
 class BackwardsScanner;
+class TokenCache;
 
 class CPLUSPLUS_EXPORT MatchingText
 {
 public:
-    MatchingText();
+    MatchingText(TokenCache *tokenCache);
 
     static bool shouldInsertMatchingText(const QTextCursor &tc);
     static bool shouldInsertMatchingText(const QChar &lookAhead);
@@ -50,6 +51,8 @@ public:
 
 private:
     bool shouldInsertNewline(const QTextCursor &tc) const;
+
+    TokenCache *_tokenCache;
 };
 
 } // end of namespace CPlusPlus

src/libs/cplusplus/TokenCache.cpp

@@ -0,0 +1,64 @@
+#include "SimpleLexer.h"
+#include "TokenCache.h"
+
+#include <QtCore/QDebug>
+
+using namespace CPlusPlus;
+
+TokenCache::TokenCache()
+    : m_doc(0)
+    , m_revision(-1)
+{}
+
+void TokenCache::setDocument(QTextDocument *doc)
+{
+    m_doc = doc;
+    m_revision = -1;
+}
+
+QList<SimpleToken> TokenCache::tokensForBlock(const QTextBlock &block) const
+{
+    Q_ASSERT(m_doc);
+
+    const int documentRevision = m_doc->revision();
+
+    if (documentRevision != m_revision) {
+        m_tokensByBlock.clear();
+        m_revision = documentRevision;
+//        qDebug() << "** revision changed to" << documentRevision;
+    }
+
+    const int blockNr = block.blockNumber();
+
+    if (m_tokensByBlock.contains(blockNr)) {
+//        qDebug()<<"Cache hit on line" << line;
+        return m_tokensByBlock.value(blockNr);
+    } else {
+//        qDebug()<<"Cache miss on line" << line;
+
+        SimpleLexer tokenize;
+        tokenize.setObjCEnabled(true);
+        tokenize.setQtMocRunEnabled(true);
+        tokenize.setSkipComments(false);
+
+        const int prevState = previousBlockState(block);
+        QList<SimpleToken> tokens = tokenize(block.text(), prevState);
+        m_tokensByBlock.insert(blockNr, tokens);
+
+        return tokens;
+    }
+}
+
+int TokenCache::previousBlockState(const QTextBlock &block)
+{
+    const QTextBlock prevBlock = block.previous();
+
+    if (prevBlock.isValid()) {
+        int state = prevBlock.userState();
+
+        if (state != -1)
+            return state;
+    }
+
+    return 0;
+}

src/libs/cplusplus/TokenCache.h

@@ -0,0 +1,35 @@
+#ifndef TOKENCACHE_H
+#define TOKENCACHE_H
+
+#include <CPlusPlusForwardDeclarations.h>
+#include <cplusplus/SimpleLexer.h>
+
+#include <QtCore/QHash>
+#include <QtCore/QList>
+
+#include <QtGui/QTextBlock>
+#include <QtGui/QTextDocument>
+
+namespace CPlusPlus {
+
+class CPLUSPLUS_EXPORT TokenCache
+{
+public:
+    TokenCache();
+
+    void setDocument(QTextDocument *doc);
+
+    QList<CPlusPlus::SimpleToken> tokensForBlock(const QTextBlock &block) const;
+
+    static int previousBlockState(const QTextBlock &block);
+
+private:
+    QTextDocument *m_doc;
+
+    mutable int m_revision;
+    mutable QHash<int, QList<CPlusPlus::SimpleToken> > m_tokensByBlock;
+};
+
+} // namespace CPlusPlus
+
+#endif // TOKENCACHE_H

src/libs/cplusplus/TokenUnderCursor.cpp

@@ -28,12 +28,14 @@
 **************************************************************************/
 
 #include "TokenUnderCursor.h"
-#include "BackwardsScanner.h"
+#include "TokenCache.h"
+#include "TokenCache.h"
 #include <Token.h>
 
 #include <QTextCursor>
 #include <QTextBlock>
 #include <climits>
+#include <QTextDocument>
 
 using namespace CPlusPlus;
 
@@ -43,17 +45,13 @@ TokenUnderCursor::TokenUnderCursor()
 TokenUnderCursor::~TokenUnderCursor()
 { }
 
-SimpleToken TokenUnderCursor::operator()(const QTextCursor &cursor, QTextBlock *b)
+SimpleToken TokenUnderCursor::operator()(TokenCache *cache, const QTextCursor &cursor, QTextBlock *b)
 {
-    SimpleLexer tokenize;
-    tokenize.setObjCEnabled(true);
-    tokenize.setSkipComments(false);
-
     QTextBlock block = cursor.block();
     int column = cursor.position() - cursor.block().position();
 
     _text = block.text();
-    _tokens = tokenize(_text, BackwardsScanner::previousBlockState(block));
+    _tokens = cache->tokensForBlock(block);
     for (int index = _tokens.size() - 1; index != -1; --index) {
         const SimpleToken &tk = _tokens.at(index);
         if (tk.position() < column) {

src/libs/cplusplus/TokenUnderCursor.h

@@ -40,13 +40,15 @@ QT_END_NAMESPACE
 
 namespace CPlusPlus {
 
+class TokenCache;
+
 class CPLUSPLUS_EXPORT TokenUnderCursor
 {
 public:
     TokenUnderCursor();
     ~TokenUnderCursor();
 
-    SimpleToken operator()(const QTextCursor &cursor, QTextBlock *block = 0);
+    SimpleToken operator()(TokenCache *cache, const QTextCursor &cursor, QTextBlock *block = 0);
 
     const QList<SimpleToken> &tokens() const
     { return _tokens; }

src/libs/cplusplus/cplusplus-lib.pri

@@ -15,7 +15,8 @@ HEADERS += \
     $$PWD/TokenUnderCursor.h \
     $$PWD/BackwardsScanner.h \
     $$PWD/MatchingText.h \
-    $$PWD/OverviewModel.h
+    $$PWD/OverviewModel.h \
+    $$PWD/TokenCache.h
 
 SOURCES += \
     $$PWD/Icons.cpp \
@@ -23,7 +24,8 @@ SOURCES += \
     $$PWD/TokenUnderCursor.cpp \
     $$PWD/BackwardsScanner.cpp \
     $$PWD/MatchingText.cpp \
-    $$PWD/OverviewModel.cpp
+    $$PWD/OverviewModel.cpp \
+    $$PWD/TokenCache.cpp
 }
 
 HEADERS += \