C++: Use Token::utf16chars{Begin,End} where appropriate

...especially in CppTools/CppEditor where the offsets are used with a
QString/QTextDocument.

Change-Id: Ic6d18fbc01fb9cc899a9bd2d7424cd2edae487f1
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>

src/libs/cplusplus/BackwardsScanner.cpp

@@ -87,7 +87,7 @@ const Token &BackwardsScanner::fetchToken(int tokenIndex)
             QList<Token> adaptedTokens;
             for (int i = 0; i < _tokens.size(); ++i) {
                 Token t = _tokens.at(i);
-                t.byteOffset += + blockText.length() + 1;
+                t.utf16charOffset += blockText.length() + 1;
                 adaptedTokens.append(t);
             }
 
@@ -112,19 +112,19 @@ QString BackwardsScanner::text() const
 QString BackwardsScanner::mid(int index) const
 {
     const Token &firstToken = _tokens.at(index + _offset);
-    return _text.mid(firstToken.bytesBegin());
+    return _text.mid(firstToken.utf16charsBegin());
 }
 
 QString BackwardsScanner::text(int index) const
 {
     const Token &firstToken = _tokens.at(index + _offset);
-    return _text.mid(firstToken.bytesBegin(), firstToken.bytes());
+    return _text.mid(firstToken.utf16charsBegin(), firstToken.utf16chars());
 }
 
 QStringRef BackwardsScanner::textRef(int index) const
 {
     const Token &firstToken = _tokens.at(index + _offset);
-    return _text.midRef(firstToken.bytesBegin(), firstToken.bytes());
+    return _text.midRef(firstToken.utf16charsBegin(), firstToken.utf16chars());
 }
 
 int BackwardsScanner::size() const
@@ -247,8 +247,8 @@ QString BackwardsScanner::indentationString(int index) const
 {
     const Token tokenAfterNewline = operator[](startOfLine(index + 1));
     const int newlinePos = qMax(0, _text.lastIndexOf(QLatin1Char('\n'),
-                                                     tokenAfterNewline.bytesBegin()));
-    return _text.mid(newlinePos, tokenAfterNewline.bytesBegin() - newlinePos);
+                                                     tokenAfterNewline.utf16charsBegin()));
+    return _text.mid(newlinePos, tokenAfterNewline.utf16charsBegin() - newlinePos);
 }
 
 

src/libs/cplusplus/ExpressionUnderCursor.cpp

@@ -266,7 +266,7 @@ int ExpressionUnderCursor::startOfFunctionCall(const QTextCursor &cursor) const
         if (tk.is(T_EOF_SYMBOL)) {
             break;
         } else if (tk.is(T_LPAREN)) {
-            return scanner.startPosition() + tk.bytesBegin();
+            return scanner.startPosition() + tk.utf16charsBegin();
         } else if (tk.is(T_RPAREN)) {
             int matchingBrace = scanner.startOfMatchingBrace(index);
 

src/libs/cplusplus/FindUsages.cpp

@@ -178,7 +178,7 @@ void FindUsages::reportResult(unsigned tokenIndex)
     if (col)
         --col;  // adjust the column position.
 
-    const int len = tk.bytes();
+    const int len = tk.utf16chars();
 
     const Usage u(_doc->fileName(), lineText, line, col, len);
     _usages.append(u);

src/libs/cplusplus/SimpleLexer.cpp

@@ -112,11 +112,11 @@ QList<Token> SimpleLexer::operator()(const QString &text, int state)
     return tokens;
 }
 
-int SimpleLexer::tokenAt(const QList<Token> &tokens, unsigned offset)
+int SimpleLexer::tokenAt(const QList<Token> &tokens, unsigned utf16charsOffset)
 {
     for (int index = tokens.size() - 1; index >= 0; --index) {
         const Token &tk = tokens.at(index);
-        if (tk.bytesBegin() <= offset && tk.bytesEnd() >= offset)
+        if (tk.utf16charsBegin() <= utf16charsOffset && tk.utf16charsEnd() >= utf16charsOffset)
             return index;
     }
 
@@ -124,7 +124,7 @@ int SimpleLexer::tokenAt(const QList<Token> &tokens, unsigned offset)
 }
 
 Token SimpleLexer::tokenAt(const QString &text,
-                           unsigned offset,
+                           unsigned utf16charsOffset,
                            int state,
                            bool qtMocRunEnabled)
 {
@@ -138,15 +138,15 @@ Token SimpleLexer::tokenAt(const QString &text,
     SimpleLexer tokenize;
     tokenize.setLanguageFeatures(features);
     const QList<Token> tokens = tokenize(text, state);
-    const int tokenIdx = tokenAt(tokens, offset);
+    const int tokenIdx = tokenAt(tokens, utf16charsOffset);
     return (tokenIdx == -1) ? Token() : tokens.at(tokenIdx);
 }
 
-int SimpleLexer::tokenBefore(const QList<Token> &tokens, unsigned offset)
+int SimpleLexer::tokenBefore(const QList<Token> &tokens, unsigned utf16charsOffset)
 {
     for (int index = tokens.size() - 1; index >= 0; --index) {
         const Token &tk = tokens.at(index);
-        if (tk.bytesBegin() <= offset)
+        if (tk.utf16charsBegin() <= utf16charsOffset)
             return index;
     }
 

src/libs/cplusplus/SimpleLexer.h

@@ -59,13 +59,13 @@ public:
     int state() const
     { return _lastState; }
 
-    static int tokenAt(const QList<Token> &tokens, unsigned offset);
+    static int tokenAt(const QList<Token> &tokens, unsigned utf16charsOffset);
     static Token tokenAt(const QString &text,
-                         unsigned offset,
+                         unsigned utf16charsOffset,
                          int state,
                          bool qtMocRunEnabled = false);
 
-    static int tokenBefore(const QList<Token> &tokens, unsigned offset);
+    static int tokenBefore(const QList<Token> &tokens, unsigned utf16charsOffset);
 
 private:
     int _lastState;