feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

CppEditor: Fully handle raw string literals in the syntax highlighter

Browse Source 
As of a3af941adf, the built-in highlighter
can properly handle multi-line raw string literals, so we don't need to
abuse the semantic highlighter for this anymore.

Fixes: QTCREATORBUG-26693
Fixes: QTCREATORBUG-28284
Change-Id: If644767dfa8a97294e84a541eea44143e8d1bb88
Reviewed-by: <github-actions-qt-creator@cristianadam.eu>
Reviewed-by: David Schulz <david.schulz@qt.io>
This commit is contained in:
Christian Kandeler
2023-01-31 15:08:20 +01:00
parent 6570895c0b
commit 209e3d0e66
10 changed files with 185 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/plugins/clangcodemodel/clangdsemantichighlighting.cpp
View File

@@ -599,10 +599,9 @@ void ExtraHighlightingResultsCollector::collectFromNode(const ClangdAstNode &nod
if (node.kind().endsWith("Literal")) {
const bool isKeyword = node.kind() == "CXXBoolLiteral"
|| node.kind() == "CXXNullPtrLiteral";
const bool isStringLike = !isKeyword && (node.kind().startsWith("String")
|| node.kind().startsWith("Character"));
const TextStyle style = isKeyword ? C_KEYWORD : isStringLike ? C_STRING : C_NUMBER;
insertResult(node, style);
if (!isKeyword && (node.kind().startsWith("String") || node.kind().startsWith("Character")))
return;
insertResult(node, isKeyword ? C_KEYWORD : C_NUMBER);
return;
}
if (node.role() == "type" && node.kind() == "Builtin") {

9
src/plugins/clangcodemodel/test/clangdtests.cpp
View File

@@ -693,10 +693,6 @@ void ClangdTestHighlighting::test_data()
QTest::addColumn<QList<int>>("expectedStyles");
QTest::addColumn<int>("expectedKind");
QTest::newRow("string literal") << 1 << 24 << 1 << 34 << QList<int>{C_STRING} << 0;
QTest::newRow("UTF-8 string literal") << 2 << 24 << 2 << 36 << QList<int>{C_STRING} << 0;
QTest::newRow("raw string literal") << 3 << 24 << 4 << 9 << QList<int>{C_STRING} << 0;
QTest::newRow("character literal") << 5 << 24 << 5 << 27 << QList<int>{C_STRING} << 0;
QTest::newRow("integer literal") << 23 << 24 << 23 << 25 << QList<int>{C_NUMBER} << 0;
QTest::newRow("float literal") << 24 << 24 << 24 << 28 << QList<int>{C_NUMBER} << 0;
QTest::newRow("function definition") << 45 << 5 << 45 << 13
@@ -1225,7 +1221,6 @@ void ClangdTestHighlighting::test_data()
QTest::newRow("triply nested template instantiation with spacing (closing angle bracket 4)")
<< 812 << 3 << 812 << 4
<< QList<int>{C_PUNCTUATION} << int(CppEditor::SemanticHighlighter::AngleBracketClose);
QTest::newRow("cyrillic string") << 792 << 24 << 792 << 27 << QList<int>{C_STRING} << 0;
QTest::newRow("macro in struct") << 795 << 9 << 795 << 14
<< QList<int>{C_MACRO, C_DECLARATION} << 0;
QTest::newRow("#ifdef'ed out code") << 800 << 1 << 800 << 17
@@ -1248,10 +1243,6 @@ void ClangdTestHighlighting::test_data()
QTest::newRow("simple return") << 841 << 12 << 841 << 15 << QList<int>{C_LOCAL} << 0;
QTest::newRow("lambda parameter") << 847 << 49 << 847 << 52
<< QList<int>{C_PARAMETER, C_DECLARATION} << 0;
QTest::newRow("string literal passed to macro from same file") << 853 << 32 << 853 << 38
<< QList<int>{C_STRING} << 0;
QTest::newRow("string literal passed to macro from header file") << 854 << 32 << 854 << 38
<< QList<int>{C_STRING} << 0;
QTest::newRow("user-defined operator call") << 860 << 7 << 860 << 8
<< QList<int>{C_LOCAL} << 0;
QTest::newRow("const member as function argument") << 868 << 32 << 868 << 43

1
src/plugins/cppeditor/cppeditor.qrc
View File

@@ -3,5 +3,6 @@
<file>images/dark_qt_cpp.png</file>
<file>images/dark_qt_h.png</file>
<file>images/dark_qt_c.png</file>
<file>testcases/highlightingtestcase.cpp</file>
</qresource>
</RCC>

2
src/plugins/cppeditor/cppeditorplugin.cpp
View File

@@ -34,6 +34,7 @@
#include "cppcompletion_test.h"
#include "cppdoxygen_test.h"
#include "cppheadersource_test.h"
#include "cpphighlighter.h"
#include "cppincludehierarchy_test.h"
#include "cppinsertvirtualmethods.h"
#include "cpplocalsymbols_test.h"
@@ -566,6 +567,7 @@ QVector<QObject *> CppEditorPlugin::createTestObjects() const
new CodegenTest,
new CompilerOptionsBuilderTest,
new CompletionTest,
new CppHighlighterTest,
new FunctionUtilsTest,
new HeaderPathFilterTest,
new HeaderSourceTest,

174
src/plugins/cppeditor/cpphighlighter.cpp
View File

@@ -4,20 +4,27 @@
#include "cpphighlighter.h"
#include "cppdoxygen.h"
#include "cppmodelmanager.h"
#include "cpptoolsreuse.h"
#include <texteditor/textdocumentlayout.h>
#include <utils/textutils.h>
#include <cplusplus/SimpleLexer.h>
#include <cplusplus/Lexer.h>
#include <QFile>
#include <QTextDocument>
#include <QTextLayout>
#ifdef WITH_TESTS
#include <QtTest>
#endif
using namespace CppEditor;
using namespace TextEditor;
using namespace CPlusPlus;
namespace CppEditor {
CppHighlighter::CppHighlighter(QTextDocument *document) :
SyntaxHighlighter(document)
{
@@ -38,8 +45,11 @@ void CppHighlighter::highlightBlock(const QString &text)
SimpleLexer tokenize;
tokenize.setLanguageFeatures(m_languageFeatures);
const QTextBlock prevBlock = currentBlock().previous();
if (prevBlock.isValid())
tokenize.setExpectedRawStringSuffix(TextDocumentLayout::expectedRawStringSuffix(prevBlock));
QByteArray inheritedRawStringSuffix;
if (prevBlock.isValid()) {
inheritedRawStringSuffix = TextDocumentLayout::expectedRawStringSuffix(prevBlock);
tokenize.setExpectedRawStringSuffix(inheritedRawStringSuffix);
}
int initialLexerState = lexerState;
const Tokens tokens = tokenize(text, initialLexerState);
@@ -84,6 +94,8 @@ void CppHighlighter::highlightBlock(const QString &text)
int previousTokenEnd = 0;
if (i != 0) {
inheritedRawStringSuffix.clear();
// mark the whitespaces
previousTokenEnd = tokens.at(i - 1).utf16charsBegin() +
tokens.at(i - 1).utf16chars();
@@ -148,7 +160,7 @@ void CppHighlighter::highlightBlock(const QString &text)
} else if (tk.is(T_NUMERIC_LITERAL)) {
setFormat(tk.utf16charsBegin(), tk.utf16chars(), formatForCategory(C_NUMBER));
} else if (tk.isStringLiteral() || tk.isCharLiteral()) {
if (!highlightRawStringLiteral(text, tk)) {
if (!highlightRawStringLiteral(text, tk, QString::fromUtf8(inheritedRawStringSuffix))) {
setFormatWithSpaces(text, tk.utf16charsBegin(), tk.utf16chars(),
formatForCategory(C_STRING));
}
@@ -354,7 +366,8 @@ void CppHighlighter::highlightWord(QStringView word, int position, int length)
}
}
bool CppHighlighter::highlightRawStringLiteral(QStringView _text, const Token &tk)
bool CppHighlighter::highlightRawStringLiteral(QStringView text, const Token &tk,
const QString &inheritedSuffix)
{
// Step one: Does the lexer think this is a raw string literal?
switch (tk.kind()) {
@@ -368,37 +381,50 @@ bool CppHighlighter::highlightRawStringLiteral(QStringView _text, const Token &t
return false;
}
// TODO: Remove on upgrade to Qt >= 5.14.
const QString text = _text.toString();
// Step two: Try to find all the components (prefix/string/suffix). We might be in the middle
// of a multi-line literal, though, so prefix and/or suffix might be missing.
int delimiterOffset = -1;
int stringOffset = 0;
int stringLength = tk.utf16chars();
int endDelimiterOffset = -1;
QString expectedSuffix = inheritedSuffix;
[&] {
// If the "inherited" suffix is not empty, then this token is a string continuation and
// can therefore not start a new raw string literal.
// FIXME: The lexer starts the token at the first non-whitespace character, so
// we have to correct for that here.
if (!inheritedSuffix.isEmpty()) {
stringLength += tk.utf16charOffset;
return;
}
// Step two: Find all the components. Bail out if we don't have a complete,
// well-formed raw string literal.
const int rOffset = text.indexOf(QLatin1String("R\""), tk.utf16charsBegin());
if (rOffset == -1)
return false;
const int delimiterOffset = rOffset + 2;
const int openParenOffset = text.indexOf('(', delimiterOffset);
if (openParenOffset == -1)
return false;
const QStringView delimiter = text.mid(delimiterOffset, openParenOffset - delimiterOffset);
if (text.at(tk.utf16charsEnd() - 1) != '"')
return false;
const int endDelimiterOffset = tk.utf16charsEnd() - 1 - delimiter.length();
if (endDelimiterOffset <= delimiterOffset)
return false;
if (text.mid(endDelimiterOffset, delimiter.length()) != delimiter)
return false;
if (text.at(endDelimiterOffset - 1) != ')')
return false;
// Conversely, since we are in a raw string literal that is not a continuation,
// the start sequence must be in here.
const int rOffset = text.indexOf(QLatin1String("R\""), tk.utf16charsBegin());
QTC_ASSERT(rOffset != -1, return);
const int tentativeDelimiterOffset = rOffset + 2;
const int openParenOffset = text.indexOf('(', tentativeDelimiterOffset);
QTC_ASSERT(openParenOffset != -1, return);
const QStringView delimiter = text.mid(tentativeDelimiterOffset,
openParenOffset - tentativeDelimiterOffset);
expectedSuffix = ')' + delimiter + '"';
delimiterOffset = tentativeDelimiterOffset;
stringOffset = delimiterOffset + delimiter.length() + 1;
stringLength -= delimiter.length() + 1;
}();
if (text.mid(tk.utf16charsBegin(), tk.utf16chars()).endsWith(expectedSuffix)) {
endDelimiterOffset = tk.utf16charsBegin() + tk.utf16chars() - expectedSuffix.size();
stringLength -= expectedSuffix.size();
}
// Step three: Do the actual formatting. For clarity, we display only the actual content as
// a string, and the rest (including the delimiter) as a keyword.
const QTextCharFormat delimiterFormat = formatForCategory(C_KEYWORD);
const int stringOffset = delimiterOffset + delimiter.length() + 1;
setFormat(tk.utf16charsBegin(), stringOffset, delimiterFormat);
setFormatWithSpaces(text, stringOffset, endDelimiterOffset - stringOffset - 1,
formatForCategory(C_STRING));
setFormat(endDelimiterOffset - 1, delimiter.length() + 2, delimiterFormat);
if (delimiterOffset != -1)
setFormat(tk.utf16charsBegin(), stringOffset, delimiterFormat);
setFormatWithSpaces(text.toString(), stringOffset, stringLength, formatForCategory(C_STRING));
if (endDelimiterOffset != -1)
setFormat(endDelimiterOffset, expectedSuffix.size(), delimiterFormat);
return true;
}
@@ -434,3 +460,87 @@ void CppHighlighter::highlightDoxygenComment(const QString &text, int position,
setFormatWithSpaces(text, initial, it - uc - initial, format);
}
namespace Internal {
CppHighlighterTest::CppHighlighterTest()
{
QFile source(":/cppeditor/testcases/highlightingtestcase.cpp");
QVERIFY(source.open(QIODevice::ReadOnly));
m_doc.setPlainText(QString::fromUtf8(source.readAll()));
setDocument(&m_doc);
rehighlight();
}
void CppHighlighterTest::test_data()
{
QTest::addColumn<int>("line");
QTest::addColumn<int>("column");
QTest::addColumn<int>("lastLine");
QTest::addColumn<int>("lastColumn");
QTest::addColumn<TextStyle>("style");
QTest::newRow("auto") << 1 << 1 << 1 << 4 << C_KEYWORD;
QTest::newRow("opening brace") << 2 << 1 << 2 << 1 << C_PUNCTUATION;
QTest::newRow("return") << 3 << 5 << 3 << 10 << C_KEYWORD;
QTest::newRow("raw string prefix") << 3 << 12 << 3 << 14 << C_KEYWORD;
QTest::newRow("raw string content (multi-line)") << 3 << 15 << 6 << 13 << C_STRING;
QTest::newRow("raw string suffix") << 6 << 14 << 6 << 15 << C_KEYWORD;
QTest::newRow("raw string prefix 2") << 6 << 17 << 6 << 19 << C_KEYWORD;
QTest::newRow("raw string content 2") << 6 << 20 << 6 << 25 << C_STRING;
QTest::newRow("raw string suffix 2") << 6 << 26 << 6 << 27 << C_KEYWORD;
QTest::newRow("comment") << 6 << 29 << 6 << 41 << C_COMMENT;
QTest::newRow("raw string prefix 3") << 6 << 53 << 6 << 45 << C_KEYWORD;
QTest::newRow("raw string content 3") << 6 << 46 << 6 << 50 << C_STRING;
QTest::newRow("raw string suffix 3") << 6 << 51 << 6 << 52 << C_KEYWORD;
QTest::newRow("semicolon") << 6 << 53 << 6 << 53 << C_PUNCTUATION;
QTest::newRow("closing brace") << 7 << 1 << 7 << 1 << C_PUNCTUATION;
}
void CppHighlighterTest::test()
{
QFETCH(int, line);
QFETCH(int, column);
QFETCH(int, lastLine);
QFETCH(int, lastColumn);
QFETCH(TextStyle, style);
const int startPos = Utils::Text::positionInText(&m_doc, line, column);
const int lastPos = Utils::Text::positionInText(&m_doc, lastLine, lastColumn);
const auto getActualFormat = [&](int pos) -> QTextCharFormat {
const QTextBlock block = m_doc.findBlock(pos);
if (!block.isValid())
return {};
const QList<QTextLayout::FormatRange> &ranges = block.layout()->formats();
for (const QTextLayout::FormatRange &range : ranges) {
const int offset = block.position() + range.start;
if (offset > pos)
return {};
if (offset + range.length <= pos)
continue;
return range.format;
}
return {};
};
const QTextCharFormat formatForStyle = formatForCategory(style);
for (int pos = startPos; pos <= lastPos; ++pos) {
const QChar c = m_doc.characterAt(pos);
if (c == QChar::ParagraphSeparator)
continue;
const QTextCharFormat expectedFormat = c.isSpace()
? whitespacified(formatForStyle) : formatForStyle;
const QTextCharFormat actualFormat = getActualFormat(pos);
if (actualFormat != expectedFormat) {
int posLine;
int posCol;
Utils::Text::convertPosition(&m_doc, pos, &posLine, &posCol);
qDebug() << posLine << posCol << c
<< actualFormat.foreground() << expectedFormat.foreground()
<< actualFormat.background() << expectedFormat.background();
}
QCOMPARE(actualFormat, expectedFormat);
}
}
} // namespace Internal
} // namespace CppEditor

20
src/plugins/cppeditor/cpphighlighter.h
View File

@@ -25,7 +25,8 @@ public:
private:
void highlightWord(QStringView word, int position, int length);
bool highlightRawStringLiteral(QStringView text, const CPlusPlus::Token &tk);
bool highlightRawStringLiteral(QStringView text, const CPlusPlus::Token &tk,
const QString &inheritedSuffix);
void highlightDoxygenComment(const QString &text, int position,
int length);
@@ -36,4 +37,21 @@ private:
CPlusPlus::LanguageFeatures m_languageFeatures = CPlusPlus::LanguageFeatures::defaultFeatures();
};
namespace Internal {
class CppHighlighterTest : public CppHighlighter
{
Q_OBJECT
public:
CppHighlighterTest();
private slots:
void test_data();
void test();
private:
QTextDocument m_doc;
};
} // namespace Internal
} // namespace CppEditor

66
src/plugins/cppeditor/semantichighlighter.cpp
View File

@@ -27,69 +27,6 @@ namespace CppEditor {
static Utils::Id parenSource() { return "CppEditor"; }
static const QList<std::pair<HighlightingResult, QTextBlock>>
splitRawStringLiteral(const HighlightingResult &result, const QTextBlock &startBlock)
{
if (result.textStyles.mainStyle != C_STRING)
return {{result, startBlock}};
QTextCursor cursor(startBlock);
cursor.setPosition(cursor.position() + result.column - 1);
cursor.setPosition(cursor.position() + result.length, QTextCursor::KeepAnchor);
const QString theString = cursor.selectedText();
// Find all the components of a raw string literal. If we don't succeed, then it's
// something else.
if (!theString.endsWith('"'))
return {{result, startBlock}};
int rOffset = -1;
if (theString.startsWith("R\"")) {
rOffset = 0;
} else if (theString.startsWith("LR\"")
|| theString.startsWith("uR\"")
|| theString.startsWith("UR\"")) {
rOffset = 1;
} else if (theString.startsWith("u8R\"")) {
rOffset = 2;
}
if (rOffset == -1)
return {{result, startBlock}};
const int delimiterOffset = rOffset + 2;
const int openParenOffset = theString.indexOf('(', delimiterOffset);
if (openParenOffset == -1)
return {{result, startBlock}};
const QStringView delimiter = theString.mid(delimiterOffset, openParenOffset - delimiterOffset);
const int endDelimiterOffset = theString.length() - 1 - delimiter.length();
if (theString.mid(endDelimiterOffset, delimiter.length()) != delimiter)
return {{result, startBlock}};
if (theString.at(endDelimiterOffset - 1) != ')')
return {{result, startBlock}};
// Now split the result. For clarity, we display only the actual content as a string,
// and the rest (including the delimiter) as a keyword.
HighlightingResult prefix = result;
prefix.textStyles.mainStyle = C_KEYWORD;
prefix.textStyles.mixinStyles = {};
prefix.length = delimiterOffset + delimiter.length() + 1;
cursor.setPosition(startBlock.position() + result.column - 1 + prefix.length);
QTextBlock stringBlock = cursor.block();
HighlightingResult actualString = result;
actualString.line = stringBlock.blockNumber() + 1;
actualString.column = cursor.positionInBlock() + 1;
actualString.length = endDelimiterOffset - openParenOffset - 2;
cursor.setPosition(cursor.position() + actualString.length);
QTextBlock suffixBlock = cursor.block();
HighlightingResult suffix = result;
suffix.textStyles.mainStyle = C_KEYWORD;
suffix.textStyles.mixinStyles = {};
suffix.line = suffixBlock.blockNumber() + 1;
suffix.column = cursor.positionInBlock() + 1;
suffix.length = delimiter.length() + 2;
QTC_CHECK(prefix.length + actualString.length + suffix.length == result.length);
return {{prefix, startBlock}, {actualString, stringBlock}, {suffix, suffixBlock}};
}
SemanticHighlighter::SemanticHighlighter(TextDocument *baseTextDocument)
: QObject(baseTextDocument)
, m_baseTextDocument(baseTextDocument)
@@ -155,8 +92,7 @@ void SemanticHighlighter::onHighlighterResultAvailable(int from, int to)
SyntaxHighlighter *highlighter = m_baseTextDocument->syntaxHighlighter();
QTC_ASSERT(highlighter, return);
incrementalApplyExtraAdditionalFormats(highlighter, m_watcher->future(), from, to, m_formatMap,
&splitRawStringLiteral);
incrementalApplyExtraAdditionalFormats(highlighter, m_watcher->future(), from, to, m_formatMap);
// In addition to the paren matching that the syntactic highlighter does
// (parentheses, braces, brackets, comments), here we inject info from the code model

7
src/plugins/cppeditor/testcases/highlightingtestcase.cpp Normal file
View File

@@ -0,0 +1,7 @@
auto func()
{
return R"(foo
foobar
R"notaprefix!(
barfoobar)" R"(second)" /* comment */ R"(third)";
}

11
src/plugins/texteditor/syntaxhighlighter.cpp
View File

@@ -478,8 +478,7 @@ void SyntaxHighlighter::setFormatWithSpaces(const QString &text, int start, int
const QTextCharFormat &format)
{
Q_D(const SyntaxHighlighter);
QTextCharFormat visualSpaceFormat = d->whitespaceFormat;
visualSpaceFormat.setBackground(format.background());
const QTextCharFormat visualSpaceFormat = whitespacified(format);
const int end = std::min(start + count, int(text.length()));
int index = start;
@@ -809,6 +808,14 @@ QTextCharFormat SyntaxHighlighter::formatForCategory(int category) const
return d->formats.at(category);
}
QTextCharFormat SyntaxHighlighter::whitespacified(const QTextCharFormat &fmt)
{
Q_D(SyntaxHighlighter);
QTextCharFormat format = d->whitespaceFormat;
format.setBackground(fmt.background());
return format;
}
void SyntaxHighlighter::highlightBlock(const QString &text)
{
formatSpaces(text);

1
src/plugins/texteditor/syntaxhighlighter.h
View File

@@ -61,6 +61,7 @@ protected:
void setDefaultTextFormatCategories();
void setTextFormatCategories(int count, std::function<TextStyle(int)> formatMapping);
QTextCharFormat formatForCategory(int categoryIndex) const;
QTextCharFormat whitespacified(const QTextCharFormat &fmt);
// implement in subclasses
// default implementation highlights whitespace