feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

Removing SimpleToken

Browse Source
This commit is contained in:
Erik Verbruggen
2010-06-29 17:57:15 +02:00
parent e3e8b1a5c0
commit 8e4fb678fd
10 changed files with 105 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/libs/cplusplus/BackwardsScanner.cpp
View File

@@ -52,19 +52,19 @@ BackwardsScanner::BackwardsScanner(const QTextCursor &cursor, int maxBlockCount,
_startToken = _tokens.size();
}
SimpleToken BackwardsScanner::LA(int index) const
Token BackwardsScanner::LA(int index) const
{ return const_cast<BackwardsScanner *>(this)->fetchToken(_startToken - index); }
SimpleToken BackwardsScanner::operator[](int index) const
Token BackwardsScanner::operator[](int index) const
{ return const_cast<BackwardsScanner *>(this)->fetchToken(index); }
const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
const Token &BackwardsScanner::fetchToken(int tokenIndex)
{
while (_offset + tokenIndex < 0) {
_block = _block.previous();
if (_blocksTokenized == _maxBlockCount || !_block.isValid()) {
++_offset;
_tokens.prepend(SimpleToken()); // sentinel
_tokens.prepend(Token()); // sentinel
break;
} else {
++_blocksTokenized;
@@ -73,10 +73,10 @@ const SimpleToken &BackwardsScanner::fetchToken(int tokenIndex)
_text.prepend(QLatin1Char('\n'));
_text.prepend(blockText);
QList<SimpleToken> adaptedTokens;
QList<Token> adaptedTokens;
for (int i = 0; i < _tokens.size(); ++i) {
SimpleToken t = _tokens.at(i);
t.setPosition(t.position() + blockText.length() + 1);
Token t = _tokens.at(i);
t.offset += + blockText.length() + 1;
adaptedTokens.append(t);
}
@@ -100,19 +100,19 @@ QString BackwardsScanner::text() const
QString BackwardsScanner::mid(int index) const
{
const SimpleToken &firstToken = _tokens.at(index + _offset);
const Token &firstToken = _tokens.at(index + _offset);
return _text.mid(firstToken.begin());
}
QString BackwardsScanner::text(int index) const
{
const SimpleToken &firstToken = _tokens.at(index + _offset);
const Token &firstToken = _tokens.at(index + _offset);
return _text.mid(firstToken.begin(), firstToken.length());
}
QStringRef BackwardsScanner::textRef(int index) const
{
const SimpleToken &firstToken = _tokens.at(index + _offset);
const Token &firstToken = _tokens.at(index + _offset);
return _text.midRef(firstToken.begin(), firstToken.length());
}
@@ -181,11 +181,11 @@ int BackwardsScanner::startOfLine(int index) const
const BackwardsScanner tk(*this);
forever {
const SimpleToken &tok = tk[index - 1];
const Token &tok = tk[index - 1];
if (tok.is(T_EOF_SYMBOL))
break;
else if (tok.followsNewline())
else if (tok.newline())
return index - 1;
--index;
@@ -201,7 +201,7 @@ int BackwardsScanner::startOfBlock(int index) const
const int start = index;
forever {
SimpleToken token = tk[index - 1];
Token token = tk[index - 1];
if (token.is(T_EOF_SYMBOL)) {
break;
@@ -234,9 +234,10 @@ int BackwardsScanner::startOfBlock(int index) const
QString BackwardsScanner::indentationString(int index) const
{
const SimpleToken tokenAfterNewline = operator[](startOfLine(index + 1));
const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'), tokenAfterNewline.position()));
return _text.mid(newlinePos, tokenAfterNewline.position() - newlinePos);
const Token tokenAfterNewline = operator[](startOfLine(index + 1));
const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'),
tokenAfterNewline.begin()));
return _text.mid(newlinePos, tokenAfterNewline.begin() - newlinePos);
}

8
src/libs/cplusplus/BackwardsScanner.h
View File

@@ -55,10 +55,10 @@ public:
QString text(int index) const;
QStringRef textRef(int index) const;
// 1-based
SimpleToken LA(int index) const;
Token LA(int index) const;
// n-la token is [startToken - n]
SimpleToken operator[](int index) const; // ### deprecate
Token operator[](int index) const; // ### deprecate
QString indentationString(int index) const;
@@ -71,10 +71,10 @@ public:
static int previousBlockState(const QTextBlock &block);
private:
const SimpleToken &fetchToken(int tokenIndex);
const Token &fetchToken(int tokenIndex);
private:
QList<SimpleToken> _tokens;
QList<Token> _tokens;
int _offset;
int _blocksTokenized;
QTextBlock _block;

8
src/libs/cplusplus/ExpressionUnderCursor.cpp
View File

@@ -56,7 +56,7 @@ int ExpressionUnderCursor::startOfExpression(BackwardsScanner &tk, int index)
index = startOfExpression_helper(tk, index);
if (_jumpedComma) {
const SimpleToken &tok = tk[index - 1];
const Token &tok = tk[index - 1];
switch (tok.kind()) {
case T_COMMA:
@@ -204,7 +204,7 @@ int ExpressionUnderCursor::startOfExpression_helper(BackwardsScanner &tk, int in
return index;
}
bool ExpressionUnderCursor::isAccessToken(const SimpleToken &tk)
bool ExpressionUnderCursor::isAccessToken(const Token &tk)
{
switch (tk.kind()) {
case T_COLON_COLON:
@@ -237,12 +237,12 @@ int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
int index = scanner.startToken();
forever {
const SimpleToken &tk = scanner[index - 1];
const Token &tk = scanner[index - 1];
if (tk.is(T_EOF_SYMBOL))
break;
else if (tk.is(T_LPAREN))
return scanner.startPosition() + tk.position();
return scanner.startPosition() + tk.begin();
else if (tk.is(T_RPAREN)) {
int matchingBrace = scanner.startOfMatchingBrace(index);

3
src/libs/cplusplus/ExpressionUnderCursor.h
View File

@@ -42,7 +42,6 @@ QT_END_NAMESPACE
namespace CPlusPlus {
class BackwardsScanner;
class SimpleToken;
class CPLUSPLUS_EXPORT ExpressionUnderCursor
{
@@ -56,7 +55,7 @@ public:
private:
int startOfExpression(BackwardsScanner &tk, int index);
int startOfExpression_helper(BackwardsScanner &tk, int index);
bool isAccessToken(const SimpleToken &tk);
bool isAccessToken(const Token &tk);
private:
bool _jumpedComma;

8
src/libs/cplusplus/MatchingText.cpp
View File

@@ -152,7 +152,7 @@ QString MatchingText::insertMatchingBrace(const QTextCursor &cursor, const QStri
const int startToken = tk.startToken();
int index = startToken;
const SimpleToken &token = tk[index - 1];
const Token &token = tk[index - 1];
if (text.at(0) == QLatin1Char('"') && (token.is(T_STRING_LITERAL) || token.is(T_WIDE_STRING_LITERAL))) {
if (text.length() != 1)
@@ -220,7 +220,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
--index; // consume the `{'
const SimpleToken &token = tk[index - 1];
const Token &token = tk[index - 1];
if (token.is(T_STRING_LITERAL) && tk[index - 2].is(T_EXTERN)) {
// recognized extern "C"
@@ -230,7 +230,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
int i = index - 1;
forever {
const SimpleToken &current = tk[i - 1];
const Token &current = tk[i - 1];
if (current.is(T_EOF_SYMBOL))
break;
@@ -290,7 +290,7 @@ QString MatchingText::insertParagraphSeparator(const QTextCursor &tc) const
}
// look at the token before the matched brace
const SimpleToken &tokenBeforeBrace = tk[lparenIndex - 1];
const Token &tokenBeforeBrace = tk[lparenIndex - 1];
if (tokenBeforeBrace.is(T_IF)) {
// recognized an if statement

80
src/libs/cplusplus/SimpleLexer.cpp
View File

@@ -37,47 +37,6 @@
using namespace CPlusPlus;
SimpleToken::SimpleToken(const Token &token)
: _kind(token.f.kind)
, _flags(0)
, _position(token.begin())
, _length(token.f.length)
{
f._whitespace = token.f.whitespace;
f._newline = token.f.newline;
}
bool SimpleToken::isLiteral() const
{
return _kind >= T_FIRST_LITERAL && _kind <= T_LAST_LITERAL;
}
bool SimpleToken::isOperator() const
{
return _kind >= T_FIRST_OPERATOR && _kind <= T_LAST_OPERATOR;
}
bool SimpleToken::isKeyword() const
{
return _kind >= T_FIRST_KEYWORD && _kind < T_FIRST_QT_KEYWORD;
}
bool SimpleToken::isComment() const
{
return _kind == T_COMMENT || _kind == T_DOXY_COMMENT ||
_kind == T_CPP_COMMENT || _kind == T_CPP_DOXY_COMMENT;
}
bool SimpleToken::isObjCAtKeyword() const
{
return _kind >= T_FIRST_OBJC_AT_KEYWORD && _kind <= T_LAST_OBJC_AT_KEYWORD;
}
const char *SimpleToken::name() const
{
return Token::name(_kind);
}
SimpleLexer::SimpleLexer()
: _lastState(0),
_skipComments(false),
@@ -119,9 +78,9 @@ void SimpleLexer::setSkipComments(bool skipComments)
_skipComments = skipComments;
}
QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
QList<Token> SimpleLexer::operator()(const QString &text, int state)
{
QList<SimpleToken> tokens;
QList<Token> tokens;
const QByteArray bytes = text.toLatin1();
const char *firstChar = bytes.constData();
@@ -131,6 +90,7 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
lex.setQtMocRunEnabled(_qtMocRunEnabled);
lex.setObjCEnabled(_objCEnabled);
lex.setStartWithNewline(true);
lex.setObjCEnabled(_objCEnabled);
if (! _skipComments)
lex.setScanCommentTokens(true);
@@ -147,57 +107,53 @@ QList<SimpleToken> SimpleLexer::operator()(const QString &text, int state)
break;
QStringRef spell = text.midRef(lex.tokenOffset(), lex.tokenLength());
SimpleToken simpleTk(tk);
lex.setScanAngleStringLiteralTokens(false);
if (tk.f.newline && tk.is(T_POUND))
inPreproc = true;
else if (inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
else if (inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
spell == QLatin1String("include"))
lex.setScanAngleStringLiteralTokens(true);
else if (_objCEnabled
&& inPreproc && tokens.size() == 1 && simpleTk.is(T_IDENTIFIER) &&
&& inPreproc && tokens.size() == 1 && tk.is(T_IDENTIFIER) &&
spell == QLatin1String("import"))
lex.setScanAngleStringLiteralTokens(true);
if (_objCEnabled && tk.is(T_IDENTIFIER))
simpleTk.f._objcTypeQualifier = (classifyObjectiveCContextKeyword(firstChar + tk.offset, tk.f.length) != Token_identifier);
tokens.append(simpleTk);
tokens.append(tk);
}
_lastState = lex.state();
return tokens;
}
int SimpleLexer::tokenAt(const QList<SimpleToken> &tokens, int offset)
int SimpleLexer::tokenAt(const QList<Token> &tokens, unsigned offset)
{
for (int index = tokens.size() - 1; index >= 0; --index) {
const SimpleToken &tk = tokens.at(index);
if (tk.position() <= offset && tk.end() >= offset)
const Token &tk = tokens.at(index);
if (tk.begin() <= offset && tk.end() >= offset)
return index;
}
return -1;
}
SimpleToken SimpleLexer::tokenAt(const QString &text,
int offset,
int state,
bool qtMocRunEnabled)
Token SimpleLexer::tokenAt(const QString &text,
unsigned offset,
int state,
bool qtMocRunEnabled)
{
SimpleLexer tokenize;
tokenize.setQtMocRunEnabled(qtMocRunEnabled);
const QList<SimpleToken> tokens = tokenize(text, state);
const QList<Token> tokens = tokenize(text, state);
const int tokenIdx = tokenAt(tokens, offset);
return (tokenIdx == -1) ? SimpleToken() : tokens.at(tokenIdx);
return (tokenIdx == -1) ? Token() : tokens.at(tokenIdx);
}
int SimpleLexer::tokenBefore(const QList<SimpleToken> &tokens, int offset)
int SimpleLexer::tokenBefore(const QList<Token> &tokens, unsigned offset)
{
for (int index = tokens.size() - 1; index >= 0; --index) {
const SimpleToken &tk = tokens.at(index);
if (tk.position() <= offset)
const Token &tk = tokens.at(index);
if (tk.begin() <= offset)
return index;
}

78
src/libs/cplusplus/SimpleLexer.h
View File

@@ -39,73 +39,6 @@ namespace CPlusPlus {
class SimpleLexer;
class Token;
class CPLUSPLUS_EXPORT SimpleToken
{
public:
SimpleToken(const Token &token);
SimpleToken()
: _kind(0)
, _flags(0)
, _position(0)
, _length(0)
{ }
inline int kind() const
{ return _kind; }
inline int position() const
{ return _position; }
inline int length() const
{ return _length; }
inline int begin() const
{ return _position; }
inline int end() const
{ return _position + _length; }
inline bool followsNewline() const
{ return f._newline; }
inline bool followsWhitespace() const
{ return f._whitespace; }
inline bool is(int k) const { return _kind == k; }
inline bool isNot(int k) const { return _kind != k; }
bool isLiteral() const;
bool isOperator() const;
bool isKeyword() const;
bool isComment() const;
bool isObjCAtKeyword() const;
bool isObjCTypeQualifier() const { return f._objcTypeQualifier; }
const char *name() const;
// internal
inline void setPosition(int position)
{ _position = position; }
public:
short _kind;
union {
short _flags;
struct {
unsigned _newline: 1;
unsigned _whitespace: 1;
unsigned _objcTypeQualifier: 1;
} f;
};
int _position;
int _length;
friend class SimpleLexer;
};
class CPLUSPLUS_EXPORT SimpleLexer
{
public:
@@ -121,15 +54,18 @@ public:
bool objCEnabled() const;
void setObjCEnabled(bool onoff);
QList<SimpleToken> operator()(const QString &text, int state = 0);
QList<Token> operator()(const QString &text, int state = 0);
int state() const
{ return _lastState; }
static int tokenAt(const QList<SimpleToken> &tokens, int offset);
static SimpleToken tokenAt(const QString &text, int offset, int state, bool qtMocRunEnabled = false);
static int tokenAt(const QList<Token> &tokens, unsigned offset);
static Token tokenAt(const QString &text,
unsigned offset,
int state,
bool qtMocRunEnabled = false);
static int tokenBefore(const QList<SimpleToken> &tokens, int offset);
static int tokenBefore(const QList<Token> &tokens, unsigned offset);
private:
int _lastState;