feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

C++: TranslationUnit::getPosition takes utf16char offsets

Browse Source 
...and not byte offsets anymore. This is necessary in order to calculate
the line and column numbers correctly with respect to unicode code
points.

Change-Id: I5d79857b3eaefeb8d563b4f1e3938a64debc5e08
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>
This commit is contained in:
Nikolai Kosjar
2014-05-06 14:48:24 -04:00
parent 70122b3061
commit 587eb49c4e
11 changed files with 359 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/libs/3rdparty/cplusplus/ASTVisitor.cpp vendored
View File

@@ -86,7 +86,7 @@ void ASTVisitor::getTokenPosition(unsigned index,
{ translationUnit()->getTokenPosition(index, line, column, fileName); }
void ASTVisitor::getTokenStartPosition(unsigned index, unsigned *line, unsigned *column) const
{ getPosition(tokenAt(index).bytesBegin(), line, column); }
{ getPosition(tokenAt(index).utf16charsBegin(), line, column); }
void ASTVisitor::getTokenEndPosition(unsigned index, unsigned *line, unsigned *column) const
{ getPosition(tokenAt(index).bytesEnd(), line, column); }
{ getPosition(tokenAt(index).utf16charsEnd(), line, column); }

94
src/libs/3rdparty/cplusplus/Bind.cpp vendored
View File

@@ -1094,8 +1094,8 @@ void Bind::lambdaDeclarator(LambdaDeclaratorAST *ast)
Function *fun = control()->newFunction(0, 0);
fun->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
fun->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->trailing_return_type)
_type = this->trailingReturnType(ast->trailing_return_type, _type);
fun->setReturnType(_type);
@@ -1192,8 +1192,8 @@ bool Bind::visit(CompoundStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
unsigned startScopeToken = ast->lbrace_token ? ast->lbrace_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ast->symbol = block;
_scope->addMember(block);
Scope *previousScope = switchScope(block);
@@ -1235,8 +1235,8 @@ bool Bind::visit(ForeachStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1285,8 +1285,8 @@ bool Bind::visit(RangeBasedForStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1334,8 +1334,8 @@ bool Bind::visit(ForStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1354,8 +1354,8 @@ bool Bind::visit(IfStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1410,8 +1410,8 @@ bool Bind::visit(SwitchStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1436,8 +1436,8 @@ bool Bind::visit(CatchClauseAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1453,8 +1453,8 @@ bool Bind::visit(WhileStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1469,8 +1469,8 @@ bool Bind::visit(ObjCFastEnumerationAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -2139,7 +2139,7 @@ bool Bind::visit(FunctionDefinitionAST *ast)
if (fun) {
setDeclSpecifiers(fun, declSpecifiers);
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (_scope->isClass()) {
fun->setVisibility(_visibility);
@@ -2201,8 +2201,8 @@ bool Bind::visit(NamespaceAST *ast)
}
Namespace *ns = control()->newNamespace(sourceLocation, namespaceName);
ns->setStartOffset(tokenAt(sourceLocation).bytesEnd()); // the scope starts after the namespace or the identifier token.
ns->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
ns->setStartOffset(tokenAt(sourceLocation).utf16charsEnd()); // the scope starts after the namespace or the identifier token.
ns->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ns->setInline(ast->inline_token != 0);
ast->symbol = ns;
_scope->addMember(ns);
@@ -2261,8 +2261,8 @@ bool Bind::visit(ParameterDeclarationAST *ast)
bool Bind::visit(TemplateDeclarationAST *ast)
{
Template *templ = control()->newTemplate(ast->firstToken(), 0);
templ->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
templ->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
templ->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
templ->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ast->symbol = templ;
Scope *previousScope = switchScope(templ);
@@ -2375,34 +2375,34 @@ unsigned Bind::calculateScopeStart(ObjCClassDeclarationAST *ast) const
{
if (ast->inst_vars_decl)
if (unsigned pos = ast->inst_vars_decl->lbrace_token)
return tokenAt(pos).bytesEnd();
return tokenAt(pos).utf16charsEnd();
if (ast->protocol_refs)
if (unsigned pos = ast->protocol_refs->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->superclass)
if (unsigned pos = ast->superclass->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->colon_token)
return tokenAt(ast->colon_token).bytesEnd();
return tokenAt(ast->colon_token).utf16charsEnd();
if (ast->rparen_token)
return tokenAt(ast->rparen_token).bytesEnd();
return tokenAt(ast->rparen_token).utf16charsEnd();
if (ast->category_name)
if (unsigned pos = ast->category_name->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->lparen_token)
return tokenAt(ast->lparen_token).bytesEnd();
return tokenAt(ast->lparen_token).utf16charsEnd();
if (ast->class_name)
if (unsigned pos = ast->class_name->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
return tokenAt(ast->firstToken()).bytesBegin();
return tokenAt(ast->firstToken()).utf16charsBegin();
}
bool Bind::visit(ObjCClassDeclarationAST *ast)
@@ -2420,7 +2420,7 @@ bool Bind::visit(ObjCClassDeclarationAST *ast)
_scope->addMember(klass);
klass->setStartOffset(calculateScopeStart(ast));
klass->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
klass->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->interface_token)
klass->setInterface(true);
@@ -2479,12 +2479,12 @@ unsigned Bind::calculateScopeStart(ObjCProtocolDeclarationAST *ast) const
{
if (ast->protocol_refs)
if (unsigned pos = ast->protocol_refs->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->name)
if (unsigned pos = ast->name->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
return tokenAt(ast->firstToken()).bytesBegin();
return tokenAt(ast->firstToken()).utf16charsBegin();
}
bool Bind::visit(ObjCProtocolDeclarationAST *ast)
@@ -2499,7 +2499,7 @@ bool Bind::visit(ObjCProtocolDeclarationAST *ast)
const unsigned sourceLocation = location(ast->name, ast->firstToken());
ObjCProtocol *protocol = control()->newObjCProtocol(sourceLocation, name);
protocol->setStartOffset(calculateScopeStart(ast));
protocol->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
protocol->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ast->symbol = protocol;
_scope->addMember(protocol);
@@ -2920,7 +2920,7 @@ bool Bind::visit(ClassSpecifierAST *ast)
{
// unsigned classkey_token = ast->classkey_token;
unsigned sourceLocation = ast->firstToken();
unsigned startScopeOffset = tokenAt(sourceLocation).bytesEnd(); // at the end of the class key
unsigned startScopeOffset = tokenAt(sourceLocation).utf16charsEnd(); // at the end of the class key
for (SpecifierListAST *it = ast->attribute_list; it; it = it->next) {
_type = this->specifier(it->value, _type);
@@ -2930,12 +2930,12 @@ bool Bind::visit(ClassSpecifierAST *ast)
if (ast->name && ! ast->name->asAnonymousName()) {
sourceLocation = location(ast->name, sourceLocation);
startScopeOffset = tokenAt(sourceLocation).bytesEnd(); // at the end of the class name
startScopeOffset = tokenAt(sourceLocation).utf16charsEnd(); // at the end of the class name
if (QualifiedNameAST *q = ast->name->asQualifiedName()) {
if (q->unqualified_name) {
sourceLocation = q->unqualified_name->firstToken();
startScopeOffset = tokenAt(q->unqualified_name->lastToken() - 1).bytesEnd(); // at the end of the unqualified name
startScopeOffset = tokenAt(q->unqualified_name->lastToken() - 1).utf16charsEnd(); // at the end of the unqualified name
}
}
@@ -2944,7 +2944,7 @@ bool Bind::visit(ClassSpecifierAST *ast)
Class *klass = control()->newClass(sourceLocation, className);
klass->setStartOffset(startScopeOffset);
klass->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
klass->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
_scope->addMember(klass);
if (_scope->isClass())
@@ -3003,8 +3003,8 @@ bool Bind::visit(EnumSpecifierAST *ast)
const Name *enumName = this->name(ast->name);
Enum *e = control()->newEnum(sourceLocation, enumName);
e->setStartOffset(tokenAt(sourceLocation).bytesEnd()); // at the end of the enum or identifier token.
e->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
e->setStartOffset(tokenAt(sourceLocation).utf16charsEnd()); // at the end of the enum or identifier token.
e->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->key_token)
e->setScoped(true);
ast->symbol = e;
@@ -3126,8 +3126,8 @@ bool Bind::visit(NestedDeclaratorAST *ast)
bool Bind::visit(FunctionDeclaratorAST *ast)
{
Function *fun = control()->newFunction(0, 0);
fun->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
fun->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->trailing_return_type)
_type = this->trailingReturnType(ast->trailing_return_type, _type);
fun->setReturnType(_type);

2
src/libs/3rdparty/cplusplus/Lexer.cpp vendored
View File

@@ -109,7 +109,7 @@ void Lexer::pushLineStartOffset()
++_currentLine;
if (_translationUnit)
_translationUnit->pushLineOffset(_currentChar - _firstChar);
_translationUnit->pushLineOffset(_currentCharUtf16);
}
void Lexer::scan(Token *tok)

2
src/libs/3rdparty/cplusplus/Symbol.cpp vendored
View File

@@ -166,7 +166,7 @@ void Symbol::setSourceLocation(unsigned sourceLocation, TranslationUnit *transla
if (translationUnit) {
const Token &tk = translationUnit->tokenAt(sourceLocation);
_isGenerated = tk.generated();
translationUnit->getPosition(tk.bytesBegin(), &_line, &_column, &_fileId);
translationUnit->getPosition(tk.utf16charsBegin(), &_line, &_column, &_fileId);
} else {
_isGenerated = false;
_line = 0;

33
src/libs/3rdparty/cplusplus/TranslationUnit.cpp vendored
View File

@@ -348,10 +348,10 @@ void TranslationUnit::pushPreprocessorLine(unsigned offset,
const StringLiteral *fileName)
{ _ppLines.push_back(PPLine(offset, line, fileName)); }
unsigned TranslationUnit::findLineNumber(unsigned offset) const
unsigned TranslationUnit::findLineNumber(unsigned utf16charOffset) const
{
std::vector<unsigned>::const_iterator it =
std::lower_bound(_lineOffsets.begin(), _lineOffsets.end(), offset);
std::lower_bound(_lineOffsets.begin(), _lineOffsets.end(), utf16charOffset);
if (it != _lineOffsets.begin())
--it;
@@ -370,31 +370,31 @@ TranslationUnit::PPLine TranslationUnit::findPreprocessorLine(unsigned offset) c
return *it;
}
unsigned TranslationUnit::findColumnNumber(unsigned offset, unsigned lineNumber) const
unsigned TranslationUnit::findColumnNumber(unsigned utf16CharOffset, unsigned lineNumber) const
{
if (! offset)
if (! utf16CharOffset)
return 0;
return offset - _lineOffsets[lineNumber];
return utf16CharOffset - _lineOffsets[lineNumber];
}
void TranslationUnit::getTokenPosition(unsigned index,
unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsBegin(), line, column, fileName); }
void TranslationUnit::getTokenStartPosition(unsigned index, unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsBegin(), line, column, fileName); }
void TranslationUnit::getTokenEndPosition(unsigned index, unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesEnd(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsEnd(), line, column, fileName); }
void TranslationUnit::getPosition(unsigned tokenOffset,
void TranslationUnit::getPosition(unsigned utf16charOffset,
unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
@@ -405,20 +405,20 @@ void TranslationUnit::getPosition(unsigned tokenOffset,
// If this token is expanded we already have the information directly from the expansion
// section header. Otherwise, we need to calculate it.
TokenLineColumn::const_iterator it = _expandedLineColumn.find(tokenOffset);
TokenLineColumn::const_iterator it = _expandedLineColumn.find(utf16charOffset);
if (it != _expandedLineColumn.end()) {
lineNumber = it->second.first;
columnNumber = it->second.second + 1;
file = _fileId;
} else {
// Identify line within the entire translation unit.
lineNumber = findLineNumber(tokenOffset);
lineNumber = findLineNumber(utf16charOffset);
// Identify column.
columnNumber = findColumnNumber(tokenOffset, lineNumber);
columnNumber = findColumnNumber(utf16charOffset, lineNumber);
// Adjust the line in regards to the preprocessing markers.
const PPLine ppLine = findPreprocessorLine(tokenOffset);
const PPLine ppLine = findPreprocessorLine(utf16charOffset);
lineNumber -= findLineNumber(ppLine.offset) + 1;
lineNumber += ppLine.line;
@@ -508,7 +508,7 @@ void TranslationUnit::fatal(unsigned index, const char *format, ...)
unsigned TranslationUnit::findPreviousLineOffset(unsigned tokenIndex) const
{
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).bytesBegin())];
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).utf16charsBegin())];
return lineOffset;
}
@@ -522,13 +522,16 @@ bool TranslationUnit::maybeSplitGreaterGreaterToken(unsigned tokenIndex)
tok.f.kind = T_GREATER;
tok.f.bytes = 1;
tok.f.utf16chars = 1;
Token newGreater;
newGreater.f.kind = T_GREATER;
newGreater.f.expanded = tok.expanded();
newGreater.f.generated = tok.generated();
newGreater.f.bytes = 1;
newGreater.f.utf16chars = 1;
newGreater.byteOffset = tok.byteOffset + 1;
newGreater.utf16charOffset = tok.utf16charOffset + 1;
_tokens->insert(_tokens->begin() + tokenIndex + 1, newGreater);
@@ -551,7 +554,7 @@ void TranslationUnit::releaseTokensAndComments()
void TranslationUnit::showErrorLine(unsigned index, unsigned column, FILE *out)
{
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).bytesBegin())];
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).utf16charsBegin())];
for (const char *cp = _firstSourceChar + lineOffset + 1; *cp && *cp != '\n'; ++cp) {
fputc(*cp, out);
}

6
src/libs/3rdparty/cplusplus/TranslationUnit.h vendored
View File

@@ -127,7 +127,7 @@ public:
unsigned *column = 0,
const StringLiteral **fileName = 0) const;
void getPosition(unsigned offset,
void getPosition(unsigned utf16charOffset,
unsigned *line,
unsigned *column = 0,
const StringLiteral **fileName = 0) const;
@@ -172,8 +172,8 @@ private:
};
void releaseTokensAndComments();
unsigned findLineNumber(unsigned offset) const;
unsigned findColumnNumber(unsigned offset, unsigned lineNumber) const;
unsigned findLineNumber(unsigned utf16charOffset) const;
unsigned findColumnNumber(unsigned utf16CharOffset, unsigned lineNumber) const;
PPLine findPreprocessorLine(unsigned offset) const;
void showErrorLine(unsigned index, unsigned column, FILE *out);