CppEditor: Fully handle raw string literals in the syntax highlighter

As of a3af941adf, the built-in highlighter
can properly handle multi-line raw string literals, so we don't need to
abuse the semantic highlighter for this anymore.

Fixes: QTCREATORBUG-26693
Fixes: QTCREATORBUG-28284
Change-Id: If644767dfa8a97294e84a541eea44143e8d1bb88
Reviewed-by: <github-actions-qt-creator@cristianadam.eu>
Reviewed-by: David Schulz <david.schulz@qt.io>

src/plugins/clangcodemodel/clangdsemantichighlighting.cpp

@@ -599,10 +599,9 @@ void ExtraHighlightingResultsCollector::collectFromNode(const ClangdAstNode &nod
     if (node.kind().endsWith("Literal")) {
         const bool isKeyword = node.kind() == "CXXBoolLiteral"
                 || node.kind() == "CXXNullPtrLiteral";
-        const bool isStringLike = !isKeyword && (node.kind().startsWith("String")
+        if (!isKeyword && (node.kind().startsWith("String") || node.kind().startsWith("Character")))
-                || node.kind().startsWith("Character"));
+            return;
-        const TextStyle style = isKeyword ? C_KEYWORD : isStringLike ? C_STRING : C_NUMBER;
+        insertResult(node, isKeyword ? C_KEYWORD : C_NUMBER);
-        insertResult(node, style);
         return;
     }
     if (node.role() == "type" && node.kind() == "Builtin") {

src/plugins/clangcodemodel/test/clangdtests.cpp

@@ -693,10 +693,6 @@ void ClangdTestHighlighting::test_data()
     QTest::addColumn<QList<int>>("expectedStyles");
     QTest::addColumn<int>("expectedKind");
 
-    QTest::newRow("string literal") << 1 << 24 << 1 << 34 << QList<int>{C_STRING} << 0;
-    QTest::newRow("UTF-8 string literal") << 2 << 24 << 2 << 36 << QList<int>{C_STRING} << 0;
-    QTest::newRow("raw string literal") << 3 << 24 << 4 << 9 << QList<int>{C_STRING} << 0;
-    QTest::newRow("character literal") << 5 << 24 << 5 << 27 << QList<int>{C_STRING} << 0;
     QTest::newRow("integer literal") << 23 << 24 << 23 << 25 << QList<int>{C_NUMBER} << 0;
     QTest::newRow("float literal") << 24 << 24 << 24 << 28 << QList<int>{C_NUMBER} << 0;
     QTest::newRow("function definition") << 45 << 5 << 45 << 13
@@ -1225,7 +1221,6 @@ void ClangdTestHighlighting::test_data()
     QTest::newRow("triply nested template instantiation with spacing (closing angle bracket 4)")
         << 812 << 3 << 812 << 4
         << QList<int>{C_PUNCTUATION} << int(CppEditor::SemanticHighlighter::AngleBracketClose);
-    QTest::newRow("cyrillic string") << 792 << 24 << 792 << 27 << QList<int>{C_STRING} << 0;
     QTest::newRow("macro in struct") << 795 << 9 << 795 << 14
         << QList<int>{C_MACRO, C_DECLARATION} << 0;
     QTest::newRow("#ifdef'ed out code") << 800 << 1 << 800 << 17
@@ -1248,10 +1243,6 @@ void ClangdTestHighlighting::test_data()
     QTest::newRow("simple return") << 841 << 12 << 841 << 15 << QList<int>{C_LOCAL} << 0;
     QTest::newRow("lambda parameter") << 847 << 49 << 847 << 52
                                       << QList<int>{C_PARAMETER, C_DECLARATION} << 0;
-    QTest::newRow("string literal passed to macro from same file") << 853 << 32 << 853 << 38
-                                      << QList<int>{C_STRING} << 0;
-    QTest::newRow("string literal passed to macro from header file") << 854 << 32 << 854 << 38
-                                      << QList<int>{C_STRING} << 0;
     QTest::newRow("user-defined operator call") << 860 << 7 << 860 << 8
                                       << QList<int>{C_LOCAL} << 0;
     QTest::newRow("const member as function argument") << 868 << 32 << 868 << 43

src/plugins/cppeditor/cppeditor.qrc

@@ -3,5 +3,6 @@
         <file>images/dark_qt_cpp.png</file>
         <file>images/dark_qt_h.png</file>
         <file>images/dark_qt_c.png</file>
+        <file>testcases/highlightingtestcase.cpp</file>
     </qresource>
 </RCC>

src/plugins/cppeditor/cppeditorplugin.cpp

@@ -34,6 +34,7 @@
 #include "cppcompletion_test.h"
 #include "cppdoxygen_test.h"
 #include "cppheadersource_test.h"
+#include "cpphighlighter.h"
 #include "cppincludehierarchy_test.h"
 #include "cppinsertvirtualmethods.h"
 #include "cpplocalsymbols_test.h"
@@ -566,6 +567,7 @@ QVector<QObject *> CppEditorPlugin::createTestObjects() const
         new CodegenTest,
         new CompilerOptionsBuilderTest,
         new CompletionTest,
+        new CppHighlighterTest,
         new FunctionUtilsTest,
         new HeaderPathFilterTest,
         new HeaderSourceTest,

src/plugins/cppeditor/cpphighlighter.cpp

@@ -4,20 +4,27 @@
 #include "cpphighlighter.h"
 
 #include "cppdoxygen.h"
-#include "cppmodelmanager.h"
 #include "cpptoolsreuse.h"
 
 #include <texteditor/textdocumentlayout.h>
+#include <utils/textutils.h>
 
 #include <cplusplus/SimpleLexer.h>
 #include <cplusplus/Lexer.h>
 
+#include <QFile>
 #include <QTextDocument>
+#include <QTextLayout>
+
+#ifdef WITH_TESTS
+#include <QtTest>
+#endif
 
-using namespace CppEditor;
 using namespace TextEditor;
 using namespace CPlusPlus;
 
+namespace CppEditor {
+
 CppHighlighter::CppHighlighter(QTextDocument *document) :
     SyntaxHighlighter(document)
 {
@@ -38,8 +45,11 @@ void CppHighlighter::highlightBlock(const QString &text)
     SimpleLexer tokenize;
     tokenize.setLanguageFeatures(m_languageFeatures);
     const QTextBlock prevBlock = currentBlock().previous();
-    if (prevBlock.isValid())
+    QByteArray inheritedRawStringSuffix;
-        tokenize.setExpectedRawStringSuffix(TextDocumentLayout::expectedRawStringSuffix(prevBlock));
+    if (prevBlock.isValid()) {
+        inheritedRawStringSuffix = TextDocumentLayout::expectedRawStringSuffix(prevBlock);
+        tokenize.setExpectedRawStringSuffix(inheritedRawStringSuffix);
+    }
 
     int initialLexerState = lexerState;
     const Tokens tokens = tokenize(text, initialLexerState);
@@ -84,6 +94,8 @@ void CppHighlighter::highlightBlock(const QString &text)
 
         int previousTokenEnd = 0;
         if (i != 0) {
+            inheritedRawStringSuffix.clear();
+
             // mark the whitespaces
             previousTokenEnd = tokens.at(i - 1).utf16charsBegin() +
                                tokens.at(i - 1).utf16chars();
@@ -148,7 +160,7 @@ void CppHighlighter::highlightBlock(const QString &text)
         } else if (tk.is(T_NUMERIC_LITERAL)) {
             setFormat(tk.utf16charsBegin(), tk.utf16chars(), formatForCategory(C_NUMBER));
         } else if (tk.isStringLiteral() || tk.isCharLiteral()) {
-            if (!highlightRawStringLiteral(text, tk)) {
+            if (!highlightRawStringLiteral(text, tk, QString::fromUtf8(inheritedRawStringSuffix))) {
                 setFormatWithSpaces(text, tk.utf16charsBegin(), tk.utf16chars(),
                                     formatForCategory(C_STRING));
             }
@@ -354,7 +366,8 @@ void CppHighlighter::highlightWord(QStringView word, int position, int length)
     }
 }
 
-bool CppHighlighter::highlightRawStringLiteral(QStringView _text, const Token &tk)
+bool CppHighlighter::highlightRawStringLiteral(QStringView text, const Token &tk,
+                                               const QString &inheritedSuffix)
 {
     // Step one: Does the lexer think this is a raw string literal?
     switch (tk.kind()) {
@@ -368,37 +381,50 @@ bool CppHighlighter::highlightRawStringLiteral(QStringView _text, const Token &t
         return false;
     }
 
-    // TODO: Remove on upgrade to Qt >= 5.14.
+    // Step two: Try to find all the components (prefix/string/suffix). We might be in the middle
-    const QString text = _text.toString();
+    //           of a multi-line literal, though, so prefix and/or suffix might be missing.
+    int delimiterOffset = -1;
+    int stringOffset = 0;
+    int stringLength = tk.utf16chars();
+    int endDelimiterOffset = -1;
+    QString expectedSuffix = inheritedSuffix;
+    [&] {
+        // If the "inherited" suffix is not empty, then this token is a string continuation and
+        // can therefore not start a new raw string literal.
+        // FIXME: The lexer starts the token at the first non-whitespace character, so
+        //        we have to correct for that here.
+        if (!inheritedSuffix.isEmpty()) {
+            stringLength += tk.utf16charOffset;
+            return;
+        }
 
-    // Step two: Find all the components. Bail out if we don't have a complete,
+        // Conversely, since we are in a raw string literal that is not a continuation,
-    //           well-formed raw string literal.
+        // the start sequence must be in here.
-    const int rOffset = text.indexOf(QLatin1String("R\""), tk.utf16charsBegin());
+        const int rOffset = text.indexOf(QLatin1String("R\""), tk.utf16charsBegin());
-    if (rOffset == -1)
+        QTC_ASSERT(rOffset != -1, return);
-        return false;
+        const int tentativeDelimiterOffset = rOffset + 2;
-    const int delimiterOffset = rOffset + 2;
+        const int openParenOffset = text.indexOf('(', tentativeDelimiterOffset);
-    const int openParenOffset = text.indexOf('(', delimiterOffset);
+        QTC_ASSERT(openParenOffset != -1, return);
-    if (openParenOffset == -1)
+        const QStringView delimiter = text.mid(tentativeDelimiterOffset,
-        return false;
+                                               openParenOffset - tentativeDelimiterOffset);
-    const QStringView delimiter = text.mid(delimiterOffset, openParenOffset - delimiterOffset);
+        expectedSuffix = ')' + delimiter + '"';
-    if (text.at(tk.utf16charsEnd() - 1) != '"')
+        delimiterOffset = tentativeDelimiterOffset;
-        return false;
+        stringOffset = delimiterOffset + delimiter.length() + 1;
-    const int endDelimiterOffset = tk.utf16charsEnd() - 1 - delimiter.length();
+        stringLength -= delimiter.length() + 1;
-    if (endDelimiterOffset <= delimiterOffset)
+    }();
-        return false;
+    if (text.mid(tk.utf16charsBegin(), tk.utf16chars()).endsWith(expectedSuffix)) {
-    if (text.mid(endDelimiterOffset, delimiter.length()) != delimiter)
+        endDelimiterOffset = tk.utf16charsBegin() + tk.utf16chars() - expectedSuffix.size();
-        return false;
+        stringLength -= expectedSuffix.size();
-    if (text.at(endDelimiterOffset - 1) != ')')
+    }
-        return false;
 
     // Step three: Do the actual formatting. For clarity, we display only the actual content as
     //             a string, and the rest (including the delimiter) as a keyword.
     const QTextCharFormat delimiterFormat = formatForCategory(C_KEYWORD);
-    const int stringOffset = delimiterOffset + delimiter.length() + 1;
+    if (delimiterOffset != -1)
-    setFormat(tk.utf16charsBegin(), stringOffset, delimiterFormat);
+        setFormat(tk.utf16charsBegin(), stringOffset, delimiterFormat);
-    setFormatWithSpaces(text, stringOffset, endDelimiterOffset - stringOffset - 1,
+    setFormatWithSpaces(text.toString(), stringOffset, stringLength, formatForCategory(C_STRING));
-                        formatForCategory(C_STRING));
+    if (endDelimiterOffset != -1)
-    setFormat(endDelimiterOffset - 1, delimiter.length() + 2, delimiterFormat);
+        setFormat(endDelimiterOffset, expectedSuffix.size(), delimiterFormat);
     return true;
 }
 
@@ -434,3 +460,87 @@ void CppHighlighter::highlightDoxygenComment(const QString &text, int position,
     setFormatWithSpaces(text, initial, it - uc - initial, format);
 }
 
+namespace Internal {
+CppHighlighterTest::CppHighlighterTest()
+{
+    QFile source(":/cppeditor/testcases/highlightingtestcase.cpp");
+    QVERIFY(source.open(QIODevice::ReadOnly));
+
+    m_doc.setPlainText(QString::fromUtf8(source.readAll()));
+    setDocument(&m_doc);
+    rehighlight();
+}
+
+void CppHighlighterTest::test_data()
+{
+    QTest::addColumn<int>("line");
+    QTest::addColumn<int>("column");
+    QTest::addColumn<int>("lastLine");
+    QTest::addColumn<int>("lastColumn");
+    QTest::addColumn<TextStyle>("style");
+
+    QTest::newRow("auto") << 1 << 1 << 1 << 4 << C_KEYWORD;
+    QTest::newRow("opening brace") << 2 << 1 << 2 << 1 << C_PUNCTUATION;
+    QTest::newRow("return") << 3 << 5 << 3 << 10 << C_KEYWORD;
+    QTest::newRow("raw string prefix") << 3 << 12 << 3 << 14 << C_KEYWORD;
+    QTest::newRow("raw string content (multi-line)") << 3 << 15 << 6 << 13 << C_STRING;
+    QTest::newRow("raw string suffix") << 6 << 14 << 6 << 15 << C_KEYWORD;
+    QTest::newRow("raw string prefix 2") << 6 << 17 << 6 << 19 << C_KEYWORD;
+    QTest::newRow("raw string content 2") << 6 << 20 << 6 << 25 << C_STRING;
+    QTest::newRow("raw string suffix 2") << 6 << 26 << 6 << 27 << C_KEYWORD;
+    QTest::newRow("comment") << 6 << 29 << 6 << 41 << C_COMMENT;
+    QTest::newRow("raw string prefix 3") << 6 << 53 << 6 << 45 << C_KEYWORD;
+    QTest::newRow("raw string content 3") << 6 << 46 << 6 << 50 << C_STRING;
+    QTest::newRow("raw string suffix 3") << 6 << 51 << 6 << 52 << C_KEYWORD;
+    QTest::newRow("semicolon") << 6 << 53 << 6 << 53 << C_PUNCTUATION;
+    QTest::newRow("closing brace") << 7 << 1 << 7 << 1 << C_PUNCTUATION;
+}
+
+void CppHighlighterTest::test()
+{
+    QFETCH(int, line);
+    QFETCH(int, column);
+    QFETCH(int, lastLine);
+    QFETCH(int, lastColumn);
+    QFETCH(TextStyle, style);
+
+    const int startPos = Utils::Text::positionInText(&m_doc, line, column);
+    const int lastPos = Utils::Text::positionInText(&m_doc, lastLine, lastColumn);
+    const auto getActualFormat = [&](int pos) -> QTextCharFormat {
+        const QTextBlock block = m_doc.findBlock(pos);
+        if (!block.isValid())
+            return {};
+        const QList<QTextLayout::FormatRange> &ranges = block.layout()->formats();
+        for (const QTextLayout::FormatRange &range : ranges) {
+            const int offset = block.position() + range.start;
+            if (offset > pos)
+                return {};
+            if (offset + range.length <= pos)
+                continue;
+            return range.format;
+        }
+        return {};
+    };
+
+    const QTextCharFormat formatForStyle = formatForCategory(style);
+    for (int pos = startPos; pos <= lastPos; ++pos) {
+        const QChar c = m_doc.characterAt(pos);
+        if (c == QChar::ParagraphSeparator)
+            continue;
+        const QTextCharFormat expectedFormat = c.isSpace()
+                ? whitespacified(formatForStyle) : formatForStyle;
+        const QTextCharFormat actualFormat = getActualFormat(pos);
+        if (actualFormat != expectedFormat) {
+            int posLine;
+            int posCol;
+            Utils::Text::convertPosition(&m_doc, pos, &posLine, &posCol);
+            qDebug() << posLine << posCol << c
+                     << actualFormat.foreground() << expectedFormat.foreground()
+                     << actualFormat.background() << expectedFormat.background();
+        }
+        QCOMPARE(actualFormat, expectedFormat);
+    }
+}
+
+} // namespace Internal
+} // namespace CppEditor

src/plugins/cppeditor/cpphighlighter.h

@@ -25,7 +25,8 @@ public:
 
 private:
     void highlightWord(QStringView word, int position, int length);
-    bool highlightRawStringLiteral(QStringView text, const CPlusPlus::Token &tk);
+    bool highlightRawStringLiteral(QStringView text, const CPlusPlus::Token &tk,
+                                   const QString &inheritedSuffix);
 
     void highlightDoxygenComment(const QString &text, int position,
                                  int length);
@@ -36,4 +37,21 @@ private:
     CPlusPlus::LanguageFeatures m_languageFeatures = CPlusPlus::LanguageFeatures::defaultFeatures();
 };
 
+namespace Internal {
+class CppHighlighterTest : public CppHighlighter
+{
+    Q_OBJECT
+
+public:
+    CppHighlighterTest();
+
+private slots:
+    void test_data();
+    void test();
+
+private:
+    QTextDocument m_doc;
+};
+} // namespace Internal
+
 } // namespace CppEditor

src/plugins/cppeditor/semantichighlighter.cpp

@@ -27,69 +27,6 @@ namespace CppEditor {
 
 static Utils::Id parenSource() { return "CppEditor"; }
 
-static const QList<std::pair<HighlightingResult, QTextBlock>>
-splitRawStringLiteral(const HighlightingResult &result, const QTextBlock &startBlock)
-{
-    if (result.textStyles.mainStyle != C_STRING)
-        return {{result, startBlock}};
-
-    QTextCursor cursor(startBlock);
-    cursor.setPosition(cursor.position() + result.column - 1);
-    cursor.setPosition(cursor.position() + result.length, QTextCursor::KeepAnchor);
-    const QString theString = cursor.selectedText();
-
-    // Find all the components of a raw string literal. If we don't succeed, then it's
-    // something else.
-    if (!theString.endsWith('"'))
-        return {{result, startBlock}};
-    int rOffset = -1;
-    if (theString.startsWith("R\"")) {
-        rOffset = 0;
-    } else if (theString.startsWith("LR\"")
-               || theString.startsWith("uR\"")
-               || theString.startsWith("UR\"")) {
-        rOffset = 1;
-    } else if (theString.startsWith("u8R\"")) {
-        rOffset = 2;
-    }
-    if (rOffset == -1)
-        return {{result, startBlock}};
-    const int delimiterOffset = rOffset + 2;
-    const int openParenOffset = theString.indexOf('(', delimiterOffset);
-    if (openParenOffset == -1)
-        return {{result, startBlock}};
-    const QStringView delimiter = theString.mid(delimiterOffset, openParenOffset - delimiterOffset);
-    const int endDelimiterOffset = theString.length() - 1 - delimiter.length();
-    if (theString.mid(endDelimiterOffset, delimiter.length()) != delimiter)
-        return {{result, startBlock}};
-    if (theString.at(endDelimiterOffset - 1) != ')')
-        return {{result, startBlock}};
-
-    // Now split the result. For clarity, we display only the actual content as a string,
-    // and the rest (including the delimiter) as a keyword.
-    HighlightingResult prefix = result;
-    prefix.textStyles.mainStyle = C_KEYWORD;
-    prefix.textStyles.mixinStyles = {};
-    prefix.length = delimiterOffset + delimiter.length() + 1;
-    cursor.setPosition(startBlock.position() + result.column - 1 + prefix.length);
-    QTextBlock stringBlock = cursor.block();
-    HighlightingResult actualString = result;
-    actualString.line = stringBlock.blockNumber() + 1;
-    actualString.column = cursor.positionInBlock() + 1;
-    actualString.length = endDelimiterOffset - openParenOffset - 2;
-    cursor.setPosition(cursor.position() + actualString.length);
-    QTextBlock suffixBlock = cursor.block();
-    HighlightingResult suffix = result;
-    suffix.textStyles.mainStyle = C_KEYWORD;
-    suffix.textStyles.mixinStyles = {};
-    suffix.line = suffixBlock.blockNumber() + 1;
-    suffix.column = cursor.positionInBlock() + 1;
-    suffix.length = delimiter.length() + 2;
-    QTC_CHECK(prefix.length + actualString.length + suffix.length == result.length);
-
-    return {{prefix, startBlock}, {actualString, stringBlock}, {suffix, suffixBlock}};
-}
-
 SemanticHighlighter::SemanticHighlighter(TextDocument *baseTextDocument)
     : QObject(baseTextDocument)
     , m_baseTextDocument(baseTextDocument)
@@ -155,8 +92,7 @@ void SemanticHighlighter::onHighlighterResultAvailable(int from, int to)
 
     SyntaxHighlighter *highlighter = m_baseTextDocument->syntaxHighlighter();
     QTC_ASSERT(highlighter, return);
-    incrementalApplyExtraAdditionalFormats(highlighter, m_watcher->future(), from, to, m_formatMap,
+    incrementalApplyExtraAdditionalFormats(highlighter, m_watcher->future(), from, to, m_formatMap);
-                                           &splitRawStringLiteral);
 
     // In addition to the paren matching that the syntactic highlighter does
     // (parentheses, braces, brackets, comments), here we inject info from the code model

src/plugins/cppeditor/testcases/highlightingtestcase.cpp

@@ -0,0 +1,7 @@
+auto func()
+{
+    return R"(foo
+    foobar
+        R"notaprefix!(
+    barfoobar)" R"(second)" /* comment */ R"(third)";
+}

src/plugins/texteditor/syntaxhighlighter.cpp

@@ -478,8 +478,7 @@ void SyntaxHighlighter::setFormatWithSpaces(const QString &text, int start, int
                                             const QTextCharFormat &format)
 {
     Q_D(const SyntaxHighlighter);
-    QTextCharFormat visualSpaceFormat = d->whitespaceFormat;
+    const QTextCharFormat visualSpaceFormat = whitespacified(format);
-    visualSpaceFormat.setBackground(format.background());
 
     const int end = std::min(start + count, int(text.length()));
     int index = start;
@@ -809,6 +808,14 @@ QTextCharFormat SyntaxHighlighter::formatForCategory(int category) const
     return d->formats.at(category);
 }
 
+QTextCharFormat SyntaxHighlighter::whitespacified(const QTextCharFormat &fmt)
+{
+    Q_D(SyntaxHighlighter);
+    QTextCharFormat format = d->whitespaceFormat;
+    format.setBackground(fmt.background());
+    return format;
+}
+
 void SyntaxHighlighter::highlightBlock(const QString &text)
 {
     formatSpaces(text);

src/plugins/texteditor/syntaxhighlighter.h

@@ -61,6 +61,7 @@ protected:
     void setDefaultTextFormatCategories();
     void setTextFormatCategories(int count, std::function<TextStyle(int)> formatMapping);
     QTextCharFormat formatForCategory(int categoryIndex) const;
+    QTextCharFormat whitespacified(const QTextCharFormat &fmt);
 
     // implement in subclasses
     // default implementation highlights whitespace