feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

C++: Clarify units of a Token

Browse Source 
This will avoid confusion when later more length and indices methods are
added.

In Token:
    length() --> bytes()
    begin() --> bytesBegin()
    end() --> bytesEnd()

Change-Id: I244c69b022e239ee762b4114559e707f93ff344f
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>
This commit is contained in:
Nikolai Kosjar
2013-12-13 18:41:15 +01:00
parent ba76baa65f
commit 126e69137a
32 changed files with 272 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/libs/3rdparty/cplusplus/ASTVisitor.cpp vendored
View File

@@ -86,7 +86,7 @@ void ASTVisitor::getTokenPosition(unsigned index,
{ translationUnit()->getTokenPosition(index, line, column, fileName); }
void ASTVisitor::getTokenStartPosition(unsigned index, unsigned *line, unsigned *column) const
{ getPosition(tokenAt(index).begin(), line, column); }
{ getPosition(tokenAt(index).bytesBegin(), line, column); }
void ASTVisitor::getTokenEndPosition(unsigned index, unsigned *line, unsigned *column) const
{ getPosition(tokenAt(index).end(), line, column); }
{ getPosition(tokenAt(index).bytesEnd(), line, column); }

94
src/libs/3rdparty/cplusplus/Bind.cpp vendored
View File

@@ -1094,8 +1094,8 @@ void Bind::lambdaDeclarator(LambdaDeclaratorAST *ast)
Function *fun = control()->newFunction(0, 0);
fun->setStartOffset(tokenAt(ast->firstToken()).begin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).end());
fun->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
if (ast->trailing_return_type)
_type = this->trailingReturnType(ast->trailing_return_type, _type);
fun->setReturnType(_type);
@@ -1192,8 +1192,8 @@ bool Bind::visit(CompoundStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
unsigned startScopeToken = ast->lbrace_token ? ast->lbrace_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).end());
block->setEndOffset(tokenAt(ast->lastToken() - 1).end());
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
ast->symbol = block;
_scope->addMember(block);
Scope *previousScope = switchScope(block);
@@ -1235,8 +1235,8 @@ bool Bind::visit(ForeachStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).end());
block->setEndOffset(tokenAt(ast->lastToken()).begin());
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1285,8 +1285,8 @@ bool Bind::visit(RangeBasedForStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).end());
block->setEndOffset(tokenAt(ast->lastToken()).begin());
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1334,8 +1334,8 @@ bool Bind::visit(ForStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).end());
block->setEndOffset(tokenAt(ast->lastToken()).begin());
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1354,8 +1354,8 @@ bool Bind::visit(IfStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).end());
block->setEndOffset(tokenAt(ast->lastToken()).begin());
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1410,8 +1410,8 @@ bool Bind::visit(SwitchStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).end());
block->setEndOffset(tokenAt(ast->lastToken()).begin());
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1436,8 +1436,8 @@ bool Bind::visit(CatchClauseAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).end());
block->setEndOffset(tokenAt(ast->lastToken()).begin());
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1453,8 +1453,8 @@ bool Bind::visit(WhileStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).end());
block->setEndOffset(tokenAt(ast->lastToken()).begin());
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1469,8 +1469,8 @@ bool Bind::visit(ObjCFastEnumerationAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).end());
block->setEndOffset(tokenAt(ast->lastToken()).begin());
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -2139,7 +2139,7 @@ bool Bind::visit(FunctionDefinitionAST *ast)
if (fun) {
setDeclSpecifiers(fun, declSpecifiers);
fun->setEndOffset(tokenAt(ast->lastToken() - 1).end());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
if (_scope->isClass()) {
fun->setVisibility(_visibility);
@@ -2201,8 +2201,8 @@ bool Bind::visit(NamespaceAST *ast)
}
Namespace *ns = control()->newNamespace(sourceLocation, namespaceName);
ns->setStartOffset(tokenAt(sourceLocation).end()); // the scope starts after the namespace or the identifier token.
ns->setEndOffset(tokenAt(ast->lastToken() - 1).end());
ns->setStartOffset(tokenAt(sourceLocation).bytesEnd()); // the scope starts after the namespace or the identifier token.
ns->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
ns->setInline(ast->inline_token != 0);
ast->symbol = ns;
_scope->addMember(ns);
@@ -2261,8 +2261,8 @@ bool Bind::visit(ParameterDeclarationAST *ast)
bool Bind::visit(TemplateDeclarationAST *ast)
{
Template *templ = control()->newTemplate(ast->firstToken(), 0);
templ->setStartOffset(tokenAt(ast->firstToken()).begin());
templ->setEndOffset(tokenAt(ast->lastToken() - 1).end());
templ->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
templ->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
ast->symbol = templ;
Scope *previousScope = switchScope(templ);
@@ -2375,34 +2375,34 @@ unsigned Bind::calculateScopeStart(ObjCClassDeclarationAST *ast) const
{
if (ast->inst_vars_decl)
if (unsigned pos = ast->inst_vars_decl->lbrace_token)
return tokenAt(pos).end();
return tokenAt(pos).bytesEnd();
if (ast->protocol_refs)
if (unsigned pos = ast->protocol_refs->lastToken())
return tokenAt(pos - 1).end();
return tokenAt(pos - 1).bytesEnd();
if (ast->superclass)
if (unsigned pos = ast->superclass->lastToken())
return tokenAt(pos - 1).end();
return tokenAt(pos - 1).bytesEnd();
if (ast->colon_token)
return tokenAt(ast->colon_token).end();
return tokenAt(ast->colon_token).bytesEnd();
if (ast->rparen_token)
return tokenAt(ast->rparen_token).end();
return tokenAt(ast->rparen_token).bytesEnd();
if (ast->category_name)
if (unsigned pos = ast->category_name->lastToken())
return tokenAt(pos - 1).end();
return tokenAt(pos - 1).bytesEnd();
if (ast->lparen_token)
return tokenAt(ast->lparen_token).end();
return tokenAt(ast->lparen_token).bytesEnd();
if (ast->class_name)
if (unsigned pos = ast->class_name->lastToken())
return tokenAt(pos - 1).end();
return tokenAt(pos - 1).bytesEnd();
return tokenAt(ast->firstToken()).begin();
return tokenAt(ast->firstToken()).bytesBegin();
}
bool Bind::visit(ObjCClassDeclarationAST *ast)
@@ -2420,7 +2420,7 @@ bool Bind::visit(ObjCClassDeclarationAST *ast)
_scope->addMember(klass);
klass->setStartOffset(calculateScopeStart(ast));
klass->setEndOffset(tokenAt(ast->lastToken() - 1).end());
klass->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
if (ast->interface_token)
klass->setInterface(true);
@@ -2479,12 +2479,12 @@ unsigned Bind::calculateScopeStart(ObjCProtocolDeclarationAST *ast) const
{
if (ast->protocol_refs)
if (unsigned pos = ast->protocol_refs->lastToken())
return tokenAt(pos - 1).end();
return tokenAt(pos - 1).bytesEnd();
if (ast->name)
if (unsigned pos = ast->name->lastToken())
return tokenAt(pos - 1).end();
return tokenAt(pos - 1).bytesEnd();
return tokenAt(ast->firstToken()).begin();
return tokenAt(ast->firstToken()).bytesBegin();
}
bool Bind::visit(ObjCProtocolDeclarationAST *ast)
@@ -2499,7 +2499,7 @@ bool Bind::visit(ObjCProtocolDeclarationAST *ast)
const unsigned sourceLocation = location(ast->name, ast->firstToken());
ObjCProtocol *protocol = control()->newObjCProtocol(sourceLocation, name);
protocol->setStartOffset(calculateScopeStart(ast));
protocol->setEndOffset(tokenAt(ast->lastToken() - 1).end());
protocol->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
ast->symbol = protocol;
_scope->addMember(protocol);
@@ -2920,7 +2920,7 @@ bool Bind::visit(ClassSpecifierAST *ast)
{
// unsigned classkey_token = ast->classkey_token;
unsigned sourceLocation = ast->firstToken();
unsigned startScopeOffset = tokenAt(sourceLocation).end(); // at the end of the class key
unsigned startScopeOffset = tokenAt(sourceLocation).bytesEnd(); // at the end of the class key
for (SpecifierListAST *it = ast->attribute_list; it; it = it->next) {
_type = this->specifier(it->value, _type);
@@ -2930,12 +2930,12 @@ bool Bind::visit(ClassSpecifierAST *ast)
if (ast->name && ! ast->name->asAnonymousName()) {
sourceLocation = location(ast->name, sourceLocation);
startScopeOffset = tokenAt(sourceLocation).end(); // at the end of the class name
startScopeOffset = tokenAt(sourceLocation).bytesEnd(); // at the end of the class name
if (QualifiedNameAST *q = ast->name->asQualifiedName()) {
if (q->unqualified_name) {
sourceLocation = q->unqualified_name->firstToken();
startScopeOffset = tokenAt(q->unqualified_name->lastToken() - 1).end(); // at the end of the unqualified name
startScopeOffset = tokenAt(q->unqualified_name->lastToken() - 1).bytesEnd(); // at the end of the unqualified name
}
}
@@ -2944,7 +2944,7 @@ bool Bind::visit(ClassSpecifierAST *ast)
Class *klass = control()->newClass(sourceLocation, className);
klass->setStartOffset(startScopeOffset);
klass->setEndOffset(tokenAt(ast->lastToken() - 1).end());
klass->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
_scope->addMember(klass);
if (_scope->isClass())
@@ -3003,8 +3003,8 @@ bool Bind::visit(EnumSpecifierAST *ast)
const Name *enumName = this->name(ast->name);
Enum *e = control()->newEnum(sourceLocation, enumName);
e->setStartOffset(tokenAt(sourceLocation).end()); // at the end of the enum or identifier token.
e->setEndOffset(tokenAt(ast->lastToken() - 1).end());
e->setStartOffset(tokenAt(sourceLocation).bytesEnd()); // at the end of the enum or identifier token.
e->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
if (ast->key_token)
e->setScoped(true);
ast->symbol = e;
@@ -3126,8 +3126,8 @@ bool Bind::visit(NestedDeclaratorAST *ast)
bool Bind::visit(FunctionDeclaratorAST *ast)
{
Function *fun = control()->newFunction(0, 0);
fun->setStartOffset(tokenAt(ast->firstToken()).begin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).end());
fun->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
if (ast->trailing_return_type)
_type = this->trailingReturnType(ast->trailing_return_type, _type);
fun->setReturnType(_type);

4
src/libs/3rdparty/cplusplus/Lexer.cpp vendored
View File

@@ -108,7 +108,7 @@ void Lexer::scan(Token *tok)
{
tok->reset();
scan_helper(tok);
tok->f.length = _currentChar - _tokenStart;
tok->f.bytes = _currentChar - _tokenStart;
}
void Lexer::scan_helper(Token *tok)
@@ -141,7 +141,7 @@ void Lexer::scan_helper(Token *tok)
tok->lineno = _currentLine;
_tokenStart = _currentChar;
tok->offset = _currentChar - _firstChar;
tok->byteOffset = _currentChar - _firstChar;
if (_yychar) {
s._newlineExpected = false;

2
src/libs/3rdparty/cplusplus/Symbol.cpp vendored
View File

@@ -166,7 +166,7 @@ void Symbol::setSourceLocation(unsigned sourceLocation, TranslationUnit *transla
if (translationUnit) {
const Token &tk = translationUnit->tokenAt(sourceLocation);
_isGenerated = tk.generated();
translationUnit->getPosition(tk.begin(), &_line, &_column, &_fileId);
translationUnit->getPosition(tk.bytesBegin(), &_line, &_column, &_fileId);
} else {
_isGenerated = false;
_line = 0;

2
src/libs/3rdparty/cplusplus/Token.cpp vendored
View File

@@ -84,7 +84,7 @@ const char *token_names[] = {
void Token::reset()
{
flags = 0;
offset = 0;
byteOffset = 0;
ptr = 0;
}

34
src/libs/3rdparty/cplusplus/Token.h vendored
View File

@@ -285,7 +285,7 @@ enum Kind {
class CPLUSPLUS_EXPORT Token
{
public:
Token() : flags(0), offset(0), ptr(0) {}
Token() : flags(0), byteOffset(0), ptr(0) {}
inline bool is(unsigned k) const { return f.kind == k; }
inline bool isNot(unsigned k) const { return f.kind != k; }
@@ -298,13 +298,13 @@ public:
inline bool joined() const { return f.joined; }
inline bool expanded() const { return f.expanded; }
inline bool generated() const { return f.generated; }
inline unsigned length() const { return f.length; }
inline unsigned bytes() const { return f.bytes; }
inline unsigned begin() const
{ return offset; }
inline unsigned bytesBegin() const
{ return byteOffset; }
inline unsigned end() const
{ return offset + f.length; }
inline unsigned bytesEnd() const
{ return byteOffset + f.bytes; }
inline bool isLiteral() const
{ return f.kind >= T_FIRST_LITERAL && f.kind <= T_LAST_LITERAL; }
@@ -333,15 +333,15 @@ public:
public:
struct Flags {
// The token kind.
unsigned kind : 8;
unsigned kind : 8;
// The token starts a new line.
unsigned newline : 1;
// The token is preceded by whitespace(s).
unsigned whitespace : 1;
unsigned newline : 1;
// The token is preceeded by whitespace(s).
unsigned whitespace : 1;
// The token is joined with the previous one.
unsigned joined : 1;
unsigned joined : 1;
// The token originates from a macro expansion.
unsigned expanded : 1;
unsigned expanded : 1;
// The token originates from a macro expansion and does not correspond to an
// argument that went through substitution. Notice the example:
//
@@ -351,18 +351,18 @@ public:
// After preprocessing we would expect the following tokens: 1 + 2;
// Tokens '1', '+', '2', and ';' are all expanded. However only tokens '+' and ';'
// are generated.
unsigned generated : 1;
unsigned generated : 1;
// Unused...
unsigned pad : 3;
// The token length.
unsigned length : 16;
unsigned pad : 3;
// The token length in bytes.
unsigned bytes : 16;
};
union {
unsigned flags;
Flags f;
};
unsigned offset;
unsigned byteOffset;
union {
void *ptr;

24
src/libs/3rdparty/cplusplus/TranslationUnit.cpp vendored
View File

@@ -164,7 +164,7 @@ void TranslationUnit::tokenize()
_Lrecognize:
if (tk.is(T_POUND) && tk.newline()) {
unsigned offset = tk.begin();
unsigned offset = tk.byteOffset;
lex(&tk);
if (! tk.newline() && tk.is(T_IDENTIFIER) && tk.identifier == expansionId) {
@@ -264,7 +264,7 @@ void TranslationUnit::tokenize()
currentExpanded = true;
const std::pair<unsigned, unsigned> &p = lineColumn[lineColumnIdx];
if (p.first)
_expandedLineColumn.insert(std::make_pair(tk.begin(), p));
_expandedLineColumn.insert(std::make_pair(tk.bytesBegin(), p));
else
currentGenerated = true;
@@ -382,17 +382,17 @@ void TranslationUnit::getTokenPosition(unsigned index,
unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).begin(), line, column, fileName); }
{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
void TranslationUnit::getTokenStartPosition(unsigned index, unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).begin(), line, column, fileName); }
{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
void TranslationUnit::getTokenEndPosition(unsigned index, unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).end(), line, column, fileName); }
{ return getPosition(tokenAt(index).bytesEnd(), line, column, fileName); }
void TranslationUnit::getPosition(unsigned tokenOffset,
unsigned *line,
@@ -508,7 +508,7 @@ void TranslationUnit::fatal(unsigned index, const char *format, ...)
unsigned TranslationUnit::findPreviousLineOffset(unsigned tokenIndex) const
{
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).begin())];
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).bytesBegin())];
return lineOffset;
}
@@ -521,21 +521,21 @@ bool TranslationUnit::maybeSplitGreaterGreaterToken(unsigned tokenIndex)
return false;
tok.f.kind = T_GREATER;
tok.f.length = 1;
tok.f.bytes = 1;
Token newGreater;
newGreater.f.kind = T_GREATER;
newGreater.f.expanded = tok.expanded();
newGreater.f.generated = tok.generated();
newGreater.f.length = 1;
newGreater.offset = tok.offset + 1;
newGreater.f.bytes = 1;
newGreater.byteOffset = tok.byteOffset + 1;
_tokens->insert(_tokens->begin() + tokenIndex + 1, newGreater);
TokenLineColumn::const_iterator it = _expandedLineColumn.find(tok.begin());
TokenLineColumn::const_iterator it = _expandedLineColumn.find(tok.bytesBegin());
if (it != _expandedLineColumn.end()) {
const std::pair<unsigned, unsigned> newPosition(it->second.first, it->second.second + 1);
_expandedLineColumn.insert(std::make_pair(newGreater.begin(), newPosition));
_expandedLineColumn.insert(std::make_pair(newGreater.bytesBegin(), newPosition));
}
return true;
@@ -551,7 +551,7 @@ void TranslationUnit::releaseTokensAndComments()
void TranslationUnit::showErrorLine(unsigned index, unsigned column, FILE *out)
{
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).begin())];
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).bytesBegin())];
for (const char *cp = _firstSourceChar + lineOffset + 1; *cp && *cp != '\n'; ++cp) {
fputc(*cp, out);
}