feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

C++: TranslationUnit::getPosition takes utf16char offsets

Browse Source 
...and not byte offsets anymore. This is necessary in order to calculate
the line and column numbers correctly with respect to unicode code
points.

Change-Id: I5d79857b3eaefeb8d563b4f1e3938a64debc5e08
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>
This commit is contained in:
Nikolai Kosjar
2014-05-06 14:48:24 -04:00
parent 70122b3061
commit 587eb49c4e
11 changed files with 359 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/libs/3rdparty/cplusplus/ASTVisitor.cpp vendored
View File

@@ -86,7 +86,7 @@ void ASTVisitor::getTokenPosition(unsigned index,
{ translationUnit()->getTokenPosition(index, line, column, fileName); }
void ASTVisitor::getTokenStartPosition(unsigned index, unsigned *line, unsigned *column) const
{ getPosition(tokenAt(index).bytesBegin(), line, column); }
{ getPosition(tokenAt(index).utf16charsBegin(), line, column); }
void ASTVisitor::getTokenEndPosition(unsigned index, unsigned *line, unsigned *column) const
{ getPosition(tokenAt(index).bytesEnd(), line, column); }
{ getPosition(tokenAt(index).utf16charsEnd(), line, column); }

94
src/libs/3rdparty/cplusplus/Bind.cpp vendored
View File

@@ -1094,8 +1094,8 @@ void Bind::lambdaDeclarator(LambdaDeclaratorAST *ast)
Function *fun = control()->newFunction(0, 0);
fun->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
fun->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->trailing_return_type)
_type = this->trailingReturnType(ast->trailing_return_type, _type);
fun->setReturnType(_type);
@@ -1192,8 +1192,8 @@ bool Bind::visit(CompoundStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
unsigned startScopeToken = ast->lbrace_token ? ast->lbrace_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ast->symbol = block;
_scope->addMember(block);
Scope *previousScope = switchScope(block);
@@ -1235,8 +1235,8 @@ bool Bind::visit(ForeachStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1285,8 +1285,8 @@ bool Bind::visit(RangeBasedForStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1334,8 +1334,8 @@ bool Bind::visit(ForStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1354,8 +1354,8 @@ bool Bind::visit(IfStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1410,8 +1410,8 @@ bool Bind::visit(SwitchStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1436,8 +1436,8 @@ bool Bind::visit(CatchClauseAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1453,8 +1453,8 @@ bool Bind::visit(WhileStatementAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -1469,8 +1469,8 @@ bool Bind::visit(ObjCFastEnumerationAST *ast)
{
Block *block = control()->newBlock(ast->firstToken());
const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
_scope->addMember(block);
ast->symbol = block;
@@ -2139,7 +2139,7 @@ bool Bind::visit(FunctionDefinitionAST *ast)
if (fun) {
setDeclSpecifiers(fun, declSpecifiers);
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (_scope->isClass()) {
fun->setVisibility(_visibility);
@@ -2201,8 +2201,8 @@ bool Bind::visit(NamespaceAST *ast)
}
Namespace *ns = control()->newNamespace(sourceLocation, namespaceName);
ns->setStartOffset(tokenAt(sourceLocation).bytesEnd()); // the scope starts after the namespace or the identifier token.
ns->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
ns->setStartOffset(tokenAt(sourceLocation).utf16charsEnd()); // the scope starts after the namespace or the identifier token.
ns->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ns->setInline(ast->inline_token != 0);
ast->symbol = ns;
_scope->addMember(ns);
@@ -2261,8 +2261,8 @@ bool Bind::visit(ParameterDeclarationAST *ast)
bool Bind::visit(TemplateDeclarationAST *ast)
{
Template *templ = control()->newTemplate(ast->firstToken(), 0);
templ->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
templ->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
templ->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
templ->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ast->symbol = templ;
Scope *previousScope = switchScope(templ);
@@ -2375,34 +2375,34 @@ unsigned Bind::calculateScopeStart(ObjCClassDeclarationAST *ast) const
{
if (ast->inst_vars_decl)
if (unsigned pos = ast->inst_vars_decl->lbrace_token)
return tokenAt(pos).bytesEnd();
return tokenAt(pos).utf16charsEnd();
if (ast->protocol_refs)
if (unsigned pos = ast->protocol_refs->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->superclass)
if (unsigned pos = ast->superclass->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->colon_token)
return tokenAt(ast->colon_token).bytesEnd();
return tokenAt(ast->colon_token).utf16charsEnd();
if (ast->rparen_token)
return tokenAt(ast->rparen_token).bytesEnd();
return tokenAt(ast->rparen_token).utf16charsEnd();
if (ast->category_name)
if (unsigned pos = ast->category_name->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->lparen_token)
return tokenAt(ast->lparen_token).bytesEnd();
return tokenAt(ast->lparen_token).utf16charsEnd();
if (ast->class_name)
if (unsigned pos = ast->class_name->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
return tokenAt(ast->firstToken()).bytesBegin();
return tokenAt(ast->firstToken()).utf16charsBegin();
}
bool Bind::visit(ObjCClassDeclarationAST *ast)
@@ -2420,7 +2420,7 @@ bool Bind::visit(ObjCClassDeclarationAST *ast)
_scope->addMember(klass);
klass->setStartOffset(calculateScopeStart(ast));
klass->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
klass->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->interface_token)
klass->setInterface(true);
@@ -2479,12 +2479,12 @@ unsigned Bind::calculateScopeStart(ObjCProtocolDeclarationAST *ast) const
{
if (ast->protocol_refs)
if (unsigned pos = ast->protocol_refs->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
if (ast->name)
if (unsigned pos = ast->name->lastToken())
return tokenAt(pos - 1).bytesEnd();
return tokenAt(pos - 1).utf16charsEnd();
return tokenAt(ast->firstToken()).bytesBegin();
return tokenAt(ast->firstToken()).utf16charsBegin();
}
bool Bind::visit(ObjCProtocolDeclarationAST *ast)
@@ -2499,7 +2499,7 @@ bool Bind::visit(ObjCProtocolDeclarationAST *ast)
const unsigned sourceLocation = location(ast->name, ast->firstToken());
ObjCProtocol *protocol = control()->newObjCProtocol(sourceLocation, name);
protocol->setStartOffset(calculateScopeStart(ast));
protocol->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
protocol->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
ast->symbol = protocol;
_scope->addMember(protocol);
@@ -2920,7 +2920,7 @@ bool Bind::visit(ClassSpecifierAST *ast)
{
// unsigned classkey_token = ast->classkey_token;
unsigned sourceLocation = ast->firstToken();
unsigned startScopeOffset = tokenAt(sourceLocation).bytesEnd(); // at the end of the class key
unsigned startScopeOffset = tokenAt(sourceLocation).utf16charsEnd(); // at the end of the class key
for (SpecifierListAST *it = ast->attribute_list; it; it = it->next) {
_type = this->specifier(it->value, _type);
@@ -2930,12 +2930,12 @@ bool Bind::visit(ClassSpecifierAST *ast)
if (ast->name && ! ast->name->asAnonymousName()) {
sourceLocation = location(ast->name, sourceLocation);
startScopeOffset = tokenAt(sourceLocation).bytesEnd(); // at the end of the class name
startScopeOffset = tokenAt(sourceLocation).utf16charsEnd(); // at the end of the class name
if (QualifiedNameAST *q = ast->name->asQualifiedName()) {
if (q->unqualified_name) {
sourceLocation = q->unqualified_name->firstToken();
startScopeOffset = tokenAt(q->unqualified_name->lastToken() - 1).bytesEnd(); // at the end of the unqualified name
startScopeOffset = tokenAt(q->unqualified_name->lastToken() - 1).utf16charsEnd(); // at the end of the unqualified name
}
}
@@ -2944,7 +2944,7 @@ bool Bind::visit(ClassSpecifierAST *ast)
Class *klass = control()->newClass(sourceLocation, className);
klass->setStartOffset(startScopeOffset);
klass->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
klass->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
_scope->addMember(klass);
if (_scope->isClass())
@@ -3003,8 +3003,8 @@ bool Bind::visit(EnumSpecifierAST *ast)
const Name *enumName = this->name(ast->name);
Enum *e = control()->newEnum(sourceLocation, enumName);
e->setStartOffset(tokenAt(sourceLocation).bytesEnd()); // at the end of the enum or identifier token.
e->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
e->setStartOffset(tokenAt(sourceLocation).utf16charsEnd()); // at the end of the enum or identifier token.
e->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->key_token)
e->setScoped(true);
ast->symbol = e;
@@ -3126,8 +3126,8 @@ bool Bind::visit(NestedDeclaratorAST *ast)
bool Bind::visit(FunctionDeclaratorAST *ast)
{
Function *fun = control()->newFunction(0, 0);
fun->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
fun->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
if (ast->trailing_return_type)
_type = this->trailingReturnType(ast->trailing_return_type, _type);
fun->setReturnType(_type);

2
src/libs/3rdparty/cplusplus/Lexer.cpp vendored
View File

@@ -109,7 +109,7 @@ void Lexer::pushLineStartOffset()
++_currentLine;
if (_translationUnit)
_translationUnit->pushLineOffset(_currentChar - _firstChar);
_translationUnit->pushLineOffset(_currentCharUtf16);
}
void Lexer::scan(Token *tok)

2
src/libs/3rdparty/cplusplus/Symbol.cpp vendored
View File

@@ -166,7 +166,7 @@ void Symbol::setSourceLocation(unsigned sourceLocation, TranslationUnit *transla
if (translationUnit) {
const Token &tk = translationUnit->tokenAt(sourceLocation);
_isGenerated = tk.generated();
translationUnit->getPosition(tk.bytesBegin(), &_line, &_column, &_fileId);
translationUnit->getPosition(tk.utf16charsBegin(), &_line, &_column, &_fileId);
} else {
_isGenerated = false;
_line = 0;

33
src/libs/3rdparty/cplusplus/TranslationUnit.cpp vendored
View File

@@ -348,10 +348,10 @@ void TranslationUnit::pushPreprocessorLine(unsigned offset,
const StringLiteral *fileName)
{ _ppLines.push_back(PPLine(offset, line, fileName)); }
unsigned TranslationUnit::findLineNumber(unsigned offset) const
unsigned TranslationUnit::findLineNumber(unsigned utf16charOffset) const
{
std::vector<unsigned>::const_iterator it =
std::lower_bound(_lineOffsets.begin(), _lineOffsets.end(), offset);
std::lower_bound(_lineOffsets.begin(), _lineOffsets.end(), utf16charOffset);
if (it != _lineOffsets.begin())
--it;
@@ -370,31 +370,31 @@ TranslationUnit::PPLine TranslationUnit::findPreprocessorLine(unsigned offset) c
return *it;
}
unsigned TranslationUnit::findColumnNumber(unsigned offset, unsigned lineNumber) const
unsigned TranslationUnit::findColumnNumber(unsigned utf16CharOffset, unsigned lineNumber) const
{
if (! offset)
if (! utf16CharOffset)
return 0;
return offset - _lineOffsets[lineNumber];
return utf16CharOffset - _lineOffsets[lineNumber];
}
void TranslationUnit::getTokenPosition(unsigned index,
unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsBegin(), line, column, fileName); }
void TranslationUnit::getTokenStartPosition(unsigned index, unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsBegin(), line, column, fileName); }
void TranslationUnit::getTokenEndPosition(unsigned index, unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
{ return getPosition(tokenAt(index).bytesEnd(), line, column, fileName); }
{ return getPosition(tokenAt(index).utf16charsEnd(), line, column, fileName); }
void TranslationUnit::getPosition(unsigned tokenOffset,
void TranslationUnit::getPosition(unsigned utf16charOffset,
unsigned *line,
unsigned *column,
const StringLiteral **fileName) const
@@ -405,20 +405,20 @@ void TranslationUnit::getPosition(unsigned tokenOffset,
// If this token is expanded we already have the information directly from the expansion
// section header. Otherwise, we need to calculate it.
TokenLineColumn::const_iterator it = _expandedLineColumn.find(tokenOffset);
TokenLineColumn::const_iterator it = _expandedLineColumn.find(utf16charOffset);
if (it != _expandedLineColumn.end()) {
lineNumber = it->second.first;
columnNumber = it->second.second + 1;
file = _fileId;
} else {
// Identify line within the entire translation unit.
lineNumber = findLineNumber(tokenOffset);
lineNumber = findLineNumber(utf16charOffset);
// Identify column.
columnNumber = findColumnNumber(tokenOffset, lineNumber);
columnNumber = findColumnNumber(utf16charOffset, lineNumber);
// Adjust the line in regards to the preprocessing markers.
const PPLine ppLine = findPreprocessorLine(tokenOffset);
const PPLine ppLine = findPreprocessorLine(utf16charOffset);
lineNumber -= findLineNumber(ppLine.offset) + 1;
lineNumber += ppLine.line;
@@ -508,7 +508,7 @@ void TranslationUnit::fatal(unsigned index, const char *format, ...)
unsigned TranslationUnit::findPreviousLineOffset(unsigned tokenIndex) const
{
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).bytesBegin())];
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).utf16charsBegin())];
return lineOffset;
}
@@ -522,13 +522,16 @@ bool TranslationUnit::maybeSplitGreaterGreaterToken(unsigned tokenIndex)
tok.f.kind = T_GREATER;
tok.f.bytes = 1;
tok.f.utf16chars = 1;
Token newGreater;
newGreater.f.kind = T_GREATER;
newGreater.f.expanded = tok.expanded();
newGreater.f.generated = tok.generated();
newGreater.f.bytes = 1;
newGreater.f.utf16chars = 1;
newGreater.byteOffset = tok.byteOffset + 1;
newGreater.utf16charOffset = tok.utf16charOffset + 1;
_tokens->insert(_tokens->begin() + tokenIndex + 1, newGreater);
@@ -551,7 +554,7 @@ void TranslationUnit::releaseTokensAndComments()
void TranslationUnit::showErrorLine(unsigned index, unsigned column, FILE *out)
{
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).bytesBegin())];
unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).utf16charsBegin())];
for (const char *cp = _firstSourceChar + lineOffset + 1; *cp && *cp != '\n'; ++cp) {
fputc(*cp, out);
}

6
src/libs/3rdparty/cplusplus/TranslationUnit.h vendored
View File

@@ -127,7 +127,7 @@ public:
unsigned *column = 0,
const StringLiteral **fileName = 0) const;
void getPosition(unsigned offset,
void getPosition(unsigned utf16charOffset,
unsigned *line,
unsigned *column = 0,
const StringLiteral **fileName = 0) const;
@@ -172,8 +172,8 @@ private:
};
void releaseTokensAndComments();
unsigned findLineNumber(unsigned offset) const;
unsigned findColumnNumber(unsigned offset, unsigned lineNumber) const;
unsigned findLineNumber(unsigned utf16charOffset) const;
unsigned findColumnNumber(unsigned utf16CharOffset, unsigned lineNumber) const;
PPLine findPreprocessorLine(unsigned offset) const;
void showErrorLine(unsigned index, unsigned column, FILE *out);

2
src/plugins/cppeditor/cppcodemodelinspectordialog.cpp
View File

@@ -852,7 +852,7 @@ void TokensModel::configure(CPlusPlus::TranslationUnit *translationUnit)
for (int i = 0, total = translationUnit->tokenCount(); i < total; ++i) {
TokenInfo info;
info.token = translationUnit->tokenAt(i);
translationUnit->getPosition(info.token.bytesBegin(), &info.line, &info.column);
translationUnit->getPosition(info.token.utf16charsBegin(), &info.line, &info.column);
m_tokenInfos.append(info);
}
emit layoutChanged();

4
src/plugins/cppeditor/fileandtokenactions_test.cpp
View File

@@ -238,7 +238,7 @@ TestActionsTestCase::TestActionsTestCase(const Actions &tokenActions, const Acti
} else {
// Position the cursor on the token
unsigned line, column;
translationUnit->getPosition(token.bytesBegin(), &line, &column);
translationUnit->getPosition(token.utf16charsBegin(), &line, &column);
editor->gotoLine(line, column - 1);
QApplication::processEvents();
@@ -293,7 +293,7 @@ void TestActionsTestCase::moveWordCamelCaseToToken(TranslationUnit *translationU
QVERIFY(editorWidget);
unsigned line, column;
translationUnit->getPosition(token.bytesBegin(), &line, &column);
translationUnit->getPosition(token.utf16charsBegin(), &line, &column);
while (editor->currentLine() < (int) line
|| (editor->currentLine() == (int) line

8
src/plugins/cpptools/cpprefactoringchanges.cpp
View File

@@ -199,7 +199,7 @@ ChangeSet::Range CppRefactoringFile::range(unsigned tokenIndex) const
{
const Token &token = tokenAt(tokenIndex);
unsigned line, column;
cppDocument()->translationUnit()->getPosition(token.bytesBegin(), &line, &column);
cppDocument()->translationUnit()->getPosition(token.utf16charsBegin(), &line, &column);
const int start = document()->findBlockByNumber(line - 1).position() + column - 1;
return ChangeSet::Range(start, start + token.bytes());
}
@@ -212,7 +212,7 @@ ChangeSet::Range CppRefactoringFile::range(AST *ast) const
int CppRefactoringFile::startOf(unsigned index) const
{
unsigned line, column;
cppDocument()->translationUnit()->getPosition(tokenAt(index).bytesBegin(), &line, &column);
cppDocument()->translationUnit()->getPosition(tokenAt(index).utf16charsBegin(), &line, &column);
return document()->findBlockByNumber(line - 1).position() + column - 1;
}
@@ -224,7 +224,7 @@ int CppRefactoringFile::startOf(const AST *ast) const
int CppRefactoringFile::endOf(unsigned index) const
{
unsigned line, column;
cppDocument()->translationUnit()->getPosition(tokenAt(index).bytesEnd(), &line, &column);
cppDocument()->translationUnit()->getPosition(tokenAt(index).utf16charsEnd(), &line, &column);
return document()->findBlockByNumber(line - 1).position() + column - 1;
}
@@ -239,7 +239,7 @@ void CppRefactoringFile::startAndEndOf(unsigned index, int *start, int *end) con
{
unsigned line, column;
Token token(tokenAt(index));
cppDocument()->translationUnit()->getPosition(token.bytesBegin(), &line, &column);
cppDocument()->translationUnit()->getPosition(token.utf16charsBegin(), &line, &column);
*start = document()->findBlockByNumber(line - 1).position() + column - 1;
*end = *start + token.bytes();
}

2
src/plugins/todo/cpptodoitemsscanner.cpp
View File

@@ -98,7 +98,7 @@ void CppTodoItemsScanner::processDocument(CPlusPlus::Document::Ptr doc)
const QStringList commentLines =
QString::fromUtf8(source).split(QLatin1Char('\n'), QString::SkipEmptyParts);
unsigned lineNumber = 0;
translationUnit->getPosition(token.bytesBegin(), &lineNumber);
translationUnit->getPosition(token.utf16charsBegin(), &lineNumber);
for (int j = 0; j < commentLines.count(); ++j) {
const QString &commentLine = commentLines.at(j);
processCommentLine(doc->fileName(), commentLine, lineNumber + j, itemList);

279
tests/auto/cplusplus/translationunit/tst_translationunit.cpp
View File

@@ -36,6 +36,15 @@
#include <QtTest>
#include <QDebug>
struct LineColumn
{
LineColumn(unsigned line = 0, unsigned column = 0) : line(line), column(column) {}
unsigned line;
unsigned column;
};
typedef QList<LineColumn> LineColumnList;
Q_DECLARE_METATYPE(LineColumnList)
//TESTED_COMPONENT=src/libs/cplusplus
using namespace CPlusPlus;
@@ -58,6 +67,9 @@ private slots:
void unicodeStringLiteral();
void unicodeStringLiteral_data();
void locationOfUtf16CharOffset();
void locationOfUtf16CharOffset_data();
private:
class Document
{
@@ -132,8 +144,49 @@ private:
}
} m_diagnosticClient;
};
class TokenGetter
{
public:
typedef QSharedPointer<TokenGetter> Ptr;
public:
TokenGetter(TranslationUnit *translationUnit) : m_translationUnit(translationUnit) {}
virtual ~TokenGetter() {}
virtual unsigned tokenCount() { return m_translationUnit->tokenCount(); }
virtual Token tokenAt(unsigned index) { return m_translationUnit->tokenAt(index); }
protected:
TranslationUnit *m_translationUnit;
};
class CommentTokenGetter : public TokenGetter
{
public:
CommentTokenGetter(TranslationUnit *translationUnit) : TokenGetter(translationUnit) {}
unsigned tokenCount() { return m_translationUnit->commentCount(); }
Token tokenAt(unsigned index) { return m_translationUnit->commentAt(index); }
};
static void compareTokenLocations(TranslationUnit *translationUnit,
tst_TranslationUnit::TokenGetter *tokenGetter,
const LineColumnList &expectedLinesColumns);
};
void tst_TranslationUnit::compareTokenLocations(TranslationUnit *translationUnit,
tst_TranslationUnit::TokenGetter *tokenGetter,
const LineColumnList &expectedLinesColumns)
{
QCOMPARE(tokenGetter->tokenCount(), (unsigned) expectedLinesColumns.count());
for (unsigned i = 0, tokenCount = tokenGetter->tokenCount(); i < tokenCount; ++i) {
const LineColumn expected = expectedLinesColumns.at(i);
const unsigned utf16CharOffset = tokenGetter->tokenAt(i).utf16charsBegin();
unsigned line, column;
translationUnit->getPosition(utf16CharOffset, &line, &column);
// qDebug("%d: LineColumn(%u, %u)", i, line, column);
QCOMPARE(line, expected.line);
QCOMPARE(column, expected.column);
}
}
void tst_TranslationUnit::unicodeIdentifier()
{
QFETCH(QByteArray, identifierText);
@@ -221,5 +274,231 @@ void tst_TranslationUnit::unicodeStringLiteral_data()
QTest::newRow("non-latin1 literal 10") << _("U\"\U00010302\u00FC\u4E8C\"");
}
void tst_TranslationUnit::locationOfUtf16CharOffset()
{
QFETCH(QByteArray, source);
QFETCH(LineColumnList, expectedNonCommentLinesColumns);
QFETCH(LineColumnList, expectedCommentLinesColumns);
Document::Ptr document = Document::create(source);
TranslationUnit *translationUnit = document->translationUnit();
const TokenGetter::Ptr nonCommentTokenGetter(new TokenGetter(translationUnit));
compareTokenLocations(translationUnit, nonCommentTokenGetter.data(),
expectedNonCommentLinesColumns);
const TokenGetter::Ptr commentTokenGetter(new CommentTokenGetter(translationUnit));
compareTokenLocations(translationUnit, commentTokenGetter.data(), expectedCommentLinesColumns);
}
void tst_TranslationUnit::locationOfUtf16CharOffset_data()
{
QTest::addColumn<QByteArray>("source");
QTest::addColumn<LineColumnList>("expectedNonCommentLinesColumns");
QTest::addColumn<LineColumnList>("expectedCommentLinesColumns");
typedef QByteArray _;
QTest::newRow("empty")
<< _("")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(0, 0)
)
<< LineColumnList();
// --- Identifiers ---------------------------------------------------------------------------
QTest::newRow("latin1 identifiers")
<< _("int i;")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(1, 1) // int
<< LineColumn(1, 5) // i
<< LineColumn(1, 6) // ;
<< LineColumn(1, 7)
)
<< LineColumnList();
QTest::newRow("latin1 identifiers")
<< _("int i;\n"
"int jj;\n")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(1, 1) // int 1
<< LineColumn(1, 5) // i
<< LineColumn(1, 6) // ;
<< LineColumn(2, 1) // int 2
<< LineColumn(2, 5) // jj
<< LineColumn(2, 7) // ;
<< LineColumn(3, 1)
)
<< LineColumnList();
QTest::newRow("non-latin1 identifier")
<< _("int \u00FC;")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(1, 1) // int
<< LineColumn(1, 5) // non-latin1 identifier
<< LineColumn(1, 6) // ;
<< LineColumn(1, 7)
)
<< LineColumnList();
QTest::newRow("non-latin1 identifiers 1")
<< _("int \u00FC;\n"
"int \u00FC;")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(1, 1) // int 1
<< LineColumn(1, 5) // non-latin1 identifier 1
<< LineColumn(1, 6) // ;
<< LineColumn(2, 1) // int 2
<< LineColumn(2, 5) // non-latin1 identifier 2
<< LineColumn(2, 6) // ;
<< LineColumn(2, 7) // ;
)
<< LineColumnList();
QTest::newRow("non-latin1 identifiers 2")
<< _("int \u00FC\u4E8C\U00010302;\n"
"int v;\n"
"int \U00010302\u4E8C;")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(1, 1) // int 1
<< LineColumn(1, 5) // non-latin1 identifier 1
<< LineColumn(1, 9) // ;
<< LineColumn(2, 1) // int 2
<< LineColumn(2, 5) // non-latin1 identifier 2
<< LineColumn(2, 6) // ;
<< LineColumn(3, 1) // int 3
<< LineColumn(3, 5) // non-latin1 identifier 3
<< LineColumn(3, 8) // ;
<< LineColumn(3, 9)
)
<< LineColumnList();
// --- String literals -----------------------------------------------------------------------
QTest::newRow("latin1 string literal")
<< _("char t[] = \"foo\";")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(1, 1) // char
<< LineColumn(1, 6) // t
<< LineColumn(1, 7) // [
<< LineColumn(1, 8) // ]
<< LineColumn(1, 10) // =
<< LineColumn(1, 12) // latin1 string literal
<< LineColumn(1, 17) // ;
<< LineColumn(1, 18)
)
<< LineColumnList();
QTest::newRow("non-latin1 string literal")
<< _("char t[] = \"i\u00FC\u4E8C\U00010302\";")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(1, 1) // char
<< LineColumn(1, 6) // t
<< LineColumn(1, 7) // [
<< LineColumn(1, 8) // ]
<< LineColumn(1, 10) // =
<< LineColumn(1, 12) // non-latin1 string literal
<< LineColumn(1, 19) // ;
<< LineColumn(1, 20)
)
<< LineColumnList();
QTest::newRow("non-latin1 string literal multiple lines")
<< _("char t[] = \"i\u00FC\u4E8C\U00010302 \\\n"
"\";")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(1, 1) // char
<< LineColumn(1, 6) // t
<< LineColumn(1, 7) // [
<< LineColumn(1, 8) // ]
<< LineColumn(1, 10) // =
<< LineColumn(1, 12) // non-latin1 string literal
<< LineColumn(2, 2) // ;
<< LineColumn(2, 3)
)
<< LineColumnList();
// --- Comments ------------------------------------------------------------------------------
QTest::newRow("latin1 c++ comment line")
<< _("// comment line\n"
"int i;")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(2, 1) // int
<< LineColumn(2, 5) // i
<< LineColumn(2, 6) // ;
<< LineColumn(2, 7)
)
<< (LineColumnList()
<< LineColumn(1, 1) // comment
);
QTest::newRow("latin1 c comment line")
<< _("/* comment line */ int i;")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(1, 20) // int
<< LineColumn(1, 24) // i
<< LineColumn(1, 25) // ;
<< LineColumn(1, 26)
)
<< (LineColumnList()
<< LineColumn(1, 1) // comment
);
QTest::newRow("latin1 c comment lines")
<< _("/* comment line 1\n"
" comment line 2 */ int i;")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(2, 22) // int
<< LineColumn(2, 26) // i
<< LineColumn(2, 27) // ;
<< LineColumn(2, 28)
)
<< (LineColumnList()
<< LineColumn(1, 1) // comment
);
QTest::newRow("non-latin1 c++ comment line")
<< _("// comment line \u00FC\u4E8C\U00010302\n"
"int i;")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(2, 1) // int
<< LineColumn(2, 5) // i
<< LineColumn(2, 6) // ;
<< LineColumn(2, 7)
)
<< (LineColumnList()
<< LineColumn(1, 1) // comment
);
QTest::newRow("non-latin1 c comment lines")
<< _("/* comment line 1\n"
" comment line 2 */ int i;")
<< (LineColumnList()
<< LineColumn(0, 0)
<< LineColumn(2, 22) // int
<< LineColumn(2, 26) // i
<< LineColumn(2, 27) // ;
<< LineColumn(2, 28)
)
<< (LineColumnList()
<< LineColumn(1, 1) // comment
);
}
QTEST_APPLESS_MAIN(tst_TranslationUnit)
#include "tst_translationunit.moc"