diff --git a/src/tools/clangbackend/source/tokeninfo.cpp b/src/tools/clangbackend/source/tokeninfo.cpp index 09413145a8e..c33dd008c94 100644 --- a/src/tools/clangbackend/source/tokeninfo.cpp +++ b/src/tools/clangbackend/source/tokeninfo.cpp @@ -49,7 +49,9 @@ TokenInfo::TokenInfo(const Cursor &cursor, m_line = start.line(); m_column = start.column(); m_offset = start.offset(); - m_length = end.offset() - start.offset(); + m_length = token->spelling().operator Utf8String().toString().size(); + if (m_length == 0) + m_length = end.offset() - start.offset(); // Just for safety. } bool TokenInfo::hasInvalidMainType() const diff --git a/tests/unit/unittest/data/highlightingmarks.cpp b/tests/unit/unittest/data/highlightingmarks.cpp index f4c9aa74bbb..fc3f792b85e 100644 --- a/tests/unit/unittest/data/highlightingmarks.cpp +++ b/tests/unit/unittest/data/highlightingmarks.cpp @@ -788,3 +788,5 @@ static inline constexpr vecn operator<(vecn a, vecn b) } return x; } + +const char *cyrillic = "б"; diff --git a/tests/unit/unittest/tokenprocessor-test.cpp b/tests/unit/unittest/tokenprocessor-test.cpp index d076f815016..38755dce663 100644 --- a/tests/unit/unittest/tokenprocessor-test.cpp +++ b/tests/unit/unittest/tokenprocessor-test.cpp @@ -1807,6 +1807,12 @@ TEST_F(TokenProcessor, OperatorInTemplate) ASSERT_THAT(infos[9], HasOnlyType(HighlightingType::Punctuation)); } +TEST_F(TokenProcessor, CyrillicString) +{ + const auto infos = translationUnit.tokenInfosInRange(sourceRange(792, 28)); + ASSERT_THAT(infos[5], IsHighlightingMark(792u, 24u, 3u, HighlightingType::StringLiteral)); +} + Data *TokenProcessor::d; void TokenProcessor::SetUpTestCase()