Cpp{Tools,Editor}: Respect multi-QChar code points when handling identifiers

* Consolidate code dealing with C++ identifiers into cpptoolsreuse.h
* Handle code points that are represented with two QChars

Change-Id: I4fb4435aa539f65d88598cac0b50629f33f32440
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>

src/plugins/cppeditor/cppeditor.cpp

@@ -176,11 +176,6 @@ struct CanonicalSymbol
         return typeOfExpression.context();
     }
 
-    static inline bool isIdentifierChar(const QChar &ch)
-    {
-        return ch.isLetterOrNumber() || ch == QLatin1Char('_');
-    }
-
     Scope *getScopeAndExpression(const QTextCursor &cursor, QString *code)
     {
         return getScopeAndExpression(editor, info, cursor, code);
@@ -202,11 +197,11 @@ struct CanonicalSymbol
 
         int pos = tc.position();
 
-        if (!isIdentifierChar(document->characterAt(pos)))
+        if (!isValidIdentifierChar(document->characterAt(pos)))
-            if (!(pos > 0 && isIdentifierChar(document->characterAt(pos - 1))))
+            if (!(pos > 0 && isValidIdentifierChar(document->characterAt(pos - 1))))
                 return 0;
 
-        while (isIdentifierChar(document->characterAt(pos)))
+        while (isValidIdentifierChar(document->characterAt(pos)))
             ++pos;
         tc.setPosition(pos);
 

src/plugins/cppeditor/cppfollowsymbolundercursor.cpp

@@ -40,6 +40,7 @@
 #include <cplusplus/TypeOfExpression.h>
 #include <cpptools/cppmodelmanagerinterface.h>
 #include <cpptools/functionutils.h>
+#include <cpptools/cpptoolsreuse.h>
 #include <cpptools/symbolfinder.h>
 #include <texteditor/basetextdocumentlayout.h>
 #include <utils/qtcassert.h>
@@ -434,7 +435,7 @@ BaseTextEditorWidget::Link FollowSymbolUnderCursor::findLink(const QTextCursor &
     QTextCursor tc = cursor;
     QTextDocument *document = m_widget->document();
     QChar ch = document->characterAt(tc.position());
-    while (ch.isLetterOrNumber() || ch == QLatin1Char('_')) {
+    while (CppTools::isValidIdentifierChar(ch)) {
         tc.movePosition(QTextCursor::NextCharacter);
         ch = document->characterAt(tc.position());
     }
@@ -559,7 +560,7 @@ BaseTextEditorWidget::Link FollowSymbolUnderCursor::findLink(const QTextCursor &
         const QTextBlock block = tc.block();
         int pos = cursor.positionInBlock();
         QChar ch = document->characterAt(cursor.position());
-        if (pos > 0 && !(ch.isLetterOrNumber() || ch == QLatin1Char('_')))
+        if (pos > 0 && !isValidIdentifierChar(ch))
             --pos; // positionInBlock points to a delimiter character.
         const Token tk = SimpleLexer::tokenAt(block.text(), pos,
                                               BackwardsScanner::previousBlockState(block), true);

src/plugins/cppeditor/cpphighlighter.cpp

@@ -412,7 +412,7 @@ void CppHighlighter::highlightDoxygenComment(const QString &text, int position,
             ++it;
 
             const QChar *start = it;
-            while (it->isLetterOrNumber() || it->unicode() == '_')
+            while (CppTools::isValidAsciiIdentifierChar(*it))
                 ++it;
 
             int k = CppTools::classifyDoxygenTag(start, it - start);

src/plugins/cppeditor/followsymbol_switchmethoddecldef_test.cpp

@@ -919,6 +919,11 @@ void CppEditorPlugin::test_FollowSymbolUnderCursor_data()
         "\n"
         "void h() { typedef TreeConstIterator<MyBase> const_iterator; }\n"
     );
+
+    QTest::newRow("unicodeIdentifier") << _(
+        "class Foo { void $\u00FC\u4E8C\U00010302(); };\n"
+        "void Foo::@\u00FC\u4E8C\U00010302() {}\n"
+    );
 }
 
 void CppEditorPlugin::test_FollowSymbolUnderCursor()

src/plugins/cpptools/cppcompletionassist.cpp

@@ -35,6 +35,7 @@
 #include "cppsnapshotupdater.h"
 #include "cpptoolsconstants.h"
 #include "cpptoolseditorsupport.h"
+#include "cpptoolsreuse.h"
 
 #include <coreplugin/icore.h>
 #include <cppeditor/cppeditorconstants.h>
@@ -314,8 +315,7 @@ void CppAssistProposalItem::applyContextualContent(TextEditor::BaseTextEditor *e
         while (preserveLength > 0) {
             if (inEditor.startsWith(toInsert.right(preserveLength))
                     && (inEditorLength == preserveLength
-                        || (!inEditor.at(preserveLength).isLetterOrNumber()
+                        || !CppTools::isValidIdentifierChar(inEditor.at(preserveLength)))) {
-                            && inEditor.at(preserveLength) != QLatin1Char('_')))) {
                 break;
             }
             --preserveLength;
@@ -671,11 +671,12 @@ bool CppCompletionAssistProcessor::accepts() const
     } else {
         // Trigger completion after three characters of a name have been typed, when not editing an existing name
         QChar characterUnderCursor = m_interface->characterAt(pos);
-        if (!characterUnderCursor.isLetterOrNumber() && characterUnderCursor != QLatin1Char('_')) {
+
+        if (!isValidIdentifierChar(characterUnderCursor)) {
             const int startOfName = findStartOfName(pos);
             if (pos - startOfName >= 3) {
                 const QChar firstCharacter = m_interface->characterAt(startOfName);
-                if (firstCharacter.isLetter() || firstCharacter == QLatin1Char('_')) {
+                if (isValidFirstIdentifierChar(firstCharacter)) {
                     // Finally check that we're not inside a comment or string (code copied from startOfOperator)
                     QTextCursor tc(m_interface->textDocument());
                     tc.setPosition(pos);
@@ -875,7 +876,7 @@ int CppCompletionAssistProcessor::findStartOfName(int pos) const
     // Skip to the start of a name
     do {
         chr = m_interface->characterAt(--pos);
-    } while (chr.isLetterOrNumber() || chr == QLatin1Char('_'));
+    } while (CppTools::isValidIdentifierChar(chr));
 
     return pos + 1;
 }

src/plugins/cpptools/cppcompletionassistprovider.cpp

@@ -29,6 +29,8 @@
 
 #include "cppcompletionassistprovider.h"
 
+#include "cpptoolsreuse.h"
+
 #include <cppeditor/cppeditorconstants.h>
 
 #include <cplusplus/Token.h>
@@ -59,6 +61,11 @@ bool CppCompletionAssistProvider::isActivationCharSequence(const QString &sequen
     return false;
 }
 
+bool CppCompletionAssistProvider::isContinuationChar(const QChar &c) const
+{
+    return isValidIdentifierChar(c);
+}
+
 int CppCompletionAssistProvider::activationSequenceChar(const QChar &ch,
                                                         const QChar &ch2,
                                                         const QChar &ch3,

src/plugins/cpptools/cppcompletionassistprovider.h

@@ -58,6 +58,7 @@ public:
     bool supportsEditor(const Core::Id &editorId) const QTC_OVERRIDE;
     int activationCharSequenceLength() const QTC_OVERRIDE;
     bool isActivationCharSequence(const QString &sequence) const QTC_OVERRIDE;
+    bool isContinuationChar(const QChar &c) const QTC_OVERRIDE;
 
     virtual TextEditor::IAssistInterface *createAssistInterface(
             ProjectExplorer::Project *project, TextEditor::BaseTextEditor *editor,

src/plugins/cpptools/cpptoolsreuse.cpp

@@ -50,7 +50,7 @@ static void moveCursorToStartOrEndOfIdentifier(QTextCursor *tc,
         return;
 
     QChar ch = doc->characterAt(tc->position() - posDiff);
-    while (ch.isLetterOrNumber() || ch == QLatin1Char('_')) {
+    while (isValidIdentifierChar(ch)) {
         tc->movePosition(op);
         ch = doc->characterAt(tc->position() - posDiff);
     }
@@ -111,16 +111,31 @@ bool isOwnershipRAIIType(CPlusPlus::Symbol *symbol, const LookupContext &context
     return false;
 }
 
+bool isValidAsciiIdentifierChar(const QChar &ch)
+{
+    return ch.isLetterOrNumber() || ch == QLatin1Char(' ');
+}
+
+bool isValidFirstIdentifierChar(const QChar &ch)
+{
+    return ch.isLetter() || ch == QLatin1Char('_') || ch.isHighSurrogate() || ch.isLowSurrogate();
+}
+
+bool isValidIdentifierChar(const QChar &ch)
+{
+    return isValidFirstIdentifierChar(ch) || ch.isNumber();
+}
+
 bool isValidIdentifier(const QString &s)
 {
     const int length = s.length();
     for (int i = 0; i < length; ++i) {
         const QChar &c = s.at(i);
         if (i == 0) {
-            if (!c.isLetter() && c != QLatin1Char('_'))
+            if (!isValidFirstIdentifierChar(c))
                 return false;
         } else {
-            if (!c.isLetterOrNumber() && c != QLatin1Char('_'))
+            if (!isValidIdentifierChar(c))
                 return false;
         }
     }

src/plugins/cpptools/cpptoolsreuse.h

@@ -32,6 +32,7 @@
 
 #include "cpptools_global.h"
 
+QT_FORWARD_DECLARE_CLASS(QChar)
 QT_FORWARD_DECLARE_CLASS(QTextCursor)
 QT_FORWARD_DECLARE_CLASS(QStringRef)
 
@@ -48,6 +49,9 @@ void CPPTOOLS_EXPORT moveCursorToStartOfIdentifier(QTextCursor *tc);
 bool CPPTOOLS_EXPORT isOwnershipRAIIType(CPlusPlus::Symbol *symbol,
                                          const CPlusPlus::LookupContext &context);
 
+bool CPPTOOLS_EXPORT isValidAsciiIdentifierChar(const QChar &ch);
+bool CPPTOOLS_EXPORT isValidFirstIdentifierChar(const QChar &ch);
+bool CPPTOOLS_EXPORT isValidIdentifierChar(const QChar &ch);
 bool CPPTOOLS_EXPORT isValidIdentifier(const QString &s);
 
 bool CPPTOOLS_EXPORT isQtKeyword(const QStringRef &text);

src/plugins/texteditor/codeassist/genericproposalwidget.cpp

@@ -66,8 +66,10 @@ QString cleanText(const QString &original)
     int ignore = 0;
     for (int i = clean.length() - 1; i >= 0; --i, ++ignore) {
         const QChar &c = clean.at(i);
-        if (c.isLetterOrNumber() || c == QLatin1Char('_'))
+        if (c.isLetterOrNumber() || c == QLatin1Char('_')
+                || c.isHighSurrogate() || c.isLowSurrogate()) {
             break;
+        }
     }
     if (ignore)
         clean.chop(ignore);