Clang: fix utf8 related column numbers

Use new clang_getFileContents to efficiently convert
utf8 byte offsets from line start to column numbers.
Also provide simplier backwards convertion to pass
resulting utf8 offset to clang.

Task-number: QTCREATORBUG-16941
Change-Id: If0e58fe01ad3e281b7e952e972b9e86f6e75aadb
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@qt.io>

src/plugins/clangcodemodel/clangbackendcommunicator.cpp

@@ -368,26 +368,23 @@ QFuture<CppTools::CursorInfo> BackendCommunicator::requestReferences(
         const FileContainer &fileContainer,
         quint32 line,
         quint32 column,
-        QTextDocument *textDocument,
         const CppTools::SemanticInfo::LocalUseMap &localUses)
 {
     const RequestReferencesMessage message(fileContainer, line, column);
     m_sender->requestReferences(message);
 
-    return m_receiver.addExpectedReferencesMessage(message.ticketNumber(), textDocument,
+    return m_receiver.addExpectedReferencesMessage(message.ticketNumber(), localUses);
-                                                   localUses);
 }
 
 QFuture<CppTools::CursorInfo> BackendCommunicator::requestLocalReferences(
         const FileContainer &fileContainer,
         quint32 line,
-        quint32 column,
+        quint32 column)
-        QTextDocument *textDocument)
 {
     const RequestReferencesMessage message(fileContainer, line, column, true);
     m_sender->requestReferences(message);
 
-    return m_receiver.addExpectedReferencesMessage(message.ticketNumber(), textDocument);
+    return m_receiver.addExpectedReferencesMessage(message.ticketNumber());
 }
 
 QFuture<CppTools::ToolTipInfo> BackendCommunicator::requestToolTip(

src/plugins/clangcodemodel/clangbackendcommunicator.h

@@ -75,13 +75,11 @@ public:
             const FileContainer &fileContainer,
             quint32 line,
             quint32 column,
-            QTextDocument *textDocument,
             const LocalUseMap &localUses);
     QFuture<CppTools::CursorInfo> requestLocalReferences(
             const FileContainer &fileContainer,
             quint32 line,
-            quint32 column,
+            quint32 column);
-            QTextDocument *textDocument);
     QFuture<CppTools::ToolTipInfo> requestToolTip(const FileContainer &fileContainer,
                                                   quint32 line,
                                                   quint32 column);

src/plugins/clangcodemodel/clangbackendreceiver.cpp

@@ -101,16 +101,14 @@ void BackendReceiver::deleteProcessorsOfEditorWidget(TextEditor::TextEditorWidge
 
 QFuture<CppTools::CursorInfo> BackendReceiver::addExpectedReferencesMessage(
         quint64 ticket,
-        QTextDocument *textDocument,
         const CppTools::SemanticInfo::LocalUseMap &localUses)
 {
-    QTC_CHECK(textDocument);
     QTC_CHECK(!m_referencesTable.contains(ticket));
 
     QFutureInterface<CppTools::CursorInfo> futureInterface;
     futureInterface.reportStarted();
 
-    const ReferencesEntry entry{futureInterface, textDocument, localUses};
+    const ReferencesEntry entry{futureInterface, localUses};
     m_referencesTable.insert(ticket, entry);
 
     return futureInterface.future();
@@ -221,24 +219,17 @@ void BackendReceiver::documentAnnotationsChanged(const DocumentAnnotationsChange
 }
 
 static
-CppTools::CursorInfo::Range toCursorInfoRange(const QTextDocument &textDocument,
+CppTools::CursorInfo::Range toCursorInfoRange(const SourceRangeContainer &sourceRange)
-                                              const SourceRangeContainer &sourceRange)
 {
     const SourceLocationContainer start = sourceRange.start();
     const SourceLocationContainer end = sourceRange.end();
     const unsigned length = end.column() - start.column();
 
-    const QTextBlock block = textDocument.findBlockByNumber(static_cast<int>(start.line()) - 1);
+    return CppTools::CursorInfo::Range(start.line(), start.column(), length);
-    const int shift = ClangCodeModel::Utils::extraUtf8CharsShift(block.text(),
-                                                                 static_cast<int>(start.column()));
-    const uint column = start.column() - static_cast<uint>(shift);
-
-    return CppTools::CursorInfo::Range(start.line(), column, length);
 }
 
 static
-CppTools::CursorInfo toCursorInfo(const QTextDocument &textDocument,
+CppTools::CursorInfo toCursorInfo(const CppTools::SemanticInfo::LocalUseMap &localUses,
-                                  const CppTools::SemanticInfo::LocalUseMap &localUses,
                                   const ReferencesMessage &message)
 {
     CppTools::CursorInfo result;
@@ -246,7 +237,7 @@ CppTools::CursorInfo toCursorInfo(const QTextDocument &textDocument,
 
     result.areUseRangesForLocalVariable = message.isLocalVariable();
     for (const SourceRangeContainer &reference : references)
-        result.useRanges.append(toCursorInfoRange(textDocument, reference));
+        result.useRanges.append(toCursorInfoRange(reference));
 
     result.useRanges.reserve(references.size());
     result.localUses = localUses;
@@ -284,8 +275,7 @@ void BackendReceiver::references(const ReferencesMessage &message)
     if (futureInterface.isCanceled())
         return; // Editor document closed or a new request was issued making this result outdated.
 
-    QTC_ASSERT(entry.textDocument, return);
+    futureInterface.reportResult(toCursorInfo(entry.localUses, message));
-    futureInterface.reportResult(toCursorInfo(*entry.textDocument, entry.localUses, message));
     futureInterface.reportFinished();
 }
 

src/plugins/clangcodemodel/clangbackendreceiver.h

@@ -56,7 +56,6 @@ public:
 
     QFuture<CppTools::CursorInfo>
     addExpectedReferencesMessage(quint64 ticket,
-                                 QTextDocument *textDocument,
                                  const CppTools::SemanticInfo::LocalUseMap &localUses
                                      = CppTools::SemanticInfo::LocalUseMap());
     QFuture<CppTools::SymbolInfo> addExpectedRequestFollowSymbolMessage(quint64 ticket);
@@ -82,13 +81,10 @@ private:
     struct ReferencesEntry {
         ReferencesEntry() = default;
         ReferencesEntry(QFutureInterface<CppTools::CursorInfo> futureInterface,
-                        QTextDocument *textDocument,
                         const CppTools::SemanticInfo::LocalUseMap &localUses)
             : futureInterface(futureInterface)
-            , textDocument(textDocument)
             , localUses(localUses) {}
         QFutureInterface<CppTools::CursorInfo> futureInterface;
-        QPointer<QTextDocument> textDocument;
         CppTools::SemanticInfo::LocalUseMap localUses;
     };
     QHash<quint64, ReferencesEntry> m_referencesTable;

src/plugins/clangcodemodel/clangcompletionassistprocessor.cpp

@@ -559,7 +559,8 @@ ClangCompletionAssistProcessor::extractLineColumn(int position)
     int line = -1, column = -1;
     ::Utils::Text::convertPosition(m_interface->textDocument(), position, &line, &column);
     const QTextBlock block = m_interface->textDocument()->findBlock(position);
-    column += ClangCodeModel::Utils::extraUtf8CharsShift(block.text(), column) + 1;
+    const QString stringOnTheLeft = block.text().left(column);
+    column = stringOnTheLeft.toUtf8().size() + 1; // '+ 1' is for 1-based columns
     return {line, column};
 }
 

src/plugins/clangcodemodel/clangdiagnosticmanager.cpp

@@ -64,8 +64,7 @@ int positionInText(QTextDocument *textDocument,
 {
     auto textBlock = textDocument->findBlockByNumber(
                 static_cast<int>(sourceLocationContainer.line()) - 1);
-    int column = static_cast<int>(sourceLocationContainer.column()) - 1;
+    const int column = static_cast<int>(sourceLocationContainer.column()) - 1;
-    column -= ClangCodeModel::Utils::extraUtf8CharsShift(textBlock.text(), column);
     return textBlock.position() + column;
 }
 

src/plugins/clangcodemodel/clangeditordocumentprocessor.cpp

@@ -331,20 +331,19 @@ ClangEditorDocumentProcessor::cursorInfo(const CppTools::CursorInfoParams &param
 {
     int line, column;
     convertPosition(params.textCursor, &line, &column);
-    ++column; // for 1-based columns
 
     if (!isCursorOnIdentifier(params.textCursor))
         return defaultCursorInfoFuture();
 
     const QTextBlock block = params.textCursor.document()->findBlockByNumber(line - 1);
-    column += ClangCodeModel::Utils::extraUtf8CharsShift(block.text(), column);
+    const QString stringOnTheLeft = block.text().left(column);
+    column = stringOnTheLeft.toUtf8().size() + 1; // '+ 1' is for 1-based columns
     const CppTools::SemanticInfo::LocalUseMap localUses
         = CppTools::BuiltinCursorInfo::findLocalUses(params.semanticInfo.doc, line, column);
 
     return m_communicator.requestReferences(simpleFileContainer(),
                                             static_cast<quint32>(line),
                                             static_cast<quint32>(column),
-                                            textDocument(),
                                             localUses);
 }
 
@@ -361,8 +360,7 @@ QFuture<CppTools::CursorInfo> ClangEditorDocumentProcessor::requestLocalReferenc
 
     return m_communicator.requestLocalReferences(simpleFileContainer(),
                                                  static_cast<quint32>(line),
-                                                 static_cast<quint32>(column),
+                                                 static_cast<quint32>(column));
-                                                 textDocument());
 }
 
 QFuture<CppTools::SymbolInfo>

src/plugins/clangcodemodel/clangutils.cpp

@@ -190,31 +190,5 @@ void setLastSentDocumentRevision(const QString &filePath, uint revision)
         document->sendTracker().setLastSentRevision(int(revision));
 }
 
-// CLANG-UPGRADE-CHECK: Workaround still needed?
-// Remove once clang reports correct columns for lines with multi-byte utf8.
-int extraUtf8CharsShift(const QString &str, int column)
-{
-    int shift = 0;
-    const QByteArray byteArray = str.toUtf8();
-    for (int i = 0; i < qMin(str.length(), column); ++i) {
-        const uchar firstByte = static_cast<uchar>(byteArray.at(i));
-        // Skip different amount of bytes depending on value
-        if (firstByte < 0xC0) {
-            continue;
-        } else if (firstByte < 0xE0) {
-            ++shift;
-            ++i;
-        } else if (firstByte < 0xF0) {
-            shift += 2;
-            i += 2;
-        } else {
-            shift += 3;
-            i += 3;
-        }
-    }
-    return shift;
-}
-
-
 } // namespace Utils
 } // namespace Clang

src/plugins/clangcodemodel/clangutils.h

@@ -47,7 +47,5 @@ CppTools::ProjectPart::Ptr projectPartForFileBasedOnProcessor(const QString &fil
 bool isProjectPartLoaded(const CppTools::ProjectPart::Ptr projectPart);
 QString projectPartIdForFile(const QString &filePath);
 
-int extraUtf8CharsShift(const QString &str, int column);
-
 } // namespace Utils
 } // namespace Clang

src/tools/clangbackend/source/clangfollowsymbol.cpp

@@ -107,7 +107,7 @@ static SourceRange getOperatorRange(const CXTranslationUnit tu,
         ++operatorIndex;
     }
     const CXSourceLocation end = clang_getTokenLocation(tu, tokens.data[operatorIndex]);
-    return SourceRange(clang_getRange(start, end));
+    return SourceRange(tu, clang_getRange(start, end));
 }
 
 static SourceRangeContainer extractMatchingTokenRange(const Cursor &cursor,
@@ -128,7 +128,7 @@ static SourceRangeContainer extractMatchingTokenRange(const Cursor &cursor,
                 continue;
             }
         }
-        return SourceRange(clang_getTokenExtent(tu, tokens.data[i]));
+        return SourceRange(tu, clang_getTokenExtent(tu, tokens.data[i]));
     }
     return SourceRangeContainer();
 }
@@ -137,7 +137,7 @@ static int getTokenIndex(CXTranslationUnit tu, const Tokens &tokens, uint line,
 {
     int tokenIndex = -1;
     for (int i = static_cast<int>(tokens.tokenCount - 1); i >= 0; --i) {
-        const SourceRange range = clang_getTokenExtent(tu, tokens.data[i]);
+        const SourceRange range(tu, clang_getTokenExtent(tu, tokens.data[i]));
         if (range.contains(line, column)) {
             tokenIndex = i;
             break;

src/tools/clangbackend/source/clangreferencescollector.cpp

@@ -155,7 +155,7 @@ ReferencesCollector::~ReferencesCollector()
 
 bool ReferencesCollector::isWithinTokenRange(CXToken token, uint line, uint column) const
 {
-    const SourceRange range = clang_getTokenExtent(m_cxTranslationUnit, token);
+    const SourceRange range {m_cxTranslationUnit, clang_getTokenExtent(m_cxTranslationUnit, token)};
     return range.contains(line, column);
 }
 
@@ -229,7 +229,8 @@ ReferencesResult ReferencesCollector::collect(uint line, uint column, bool local
     const Utf8String identifier = ClangString(clang_getTokenSpelling(m_cxTranslationUnit, token));
     for (uint i = 0; i < m_cxTokenCount; ++i) {
         if (checkToken(i, identifier, usr)) {
-            const SourceRange range = clang_getTokenExtent(m_cxTranslationUnit, m_cxTokens[i]);
+            const SourceRange range {m_cxTranslationUnit,
+                                     clang_getTokenExtent(m_cxTranslationUnit, m_cxTokens[i])};
             result.references.append(range);
         }
     }

src/tools/clangbackend/source/clangtranslationunit.cpp

@@ -161,7 +161,7 @@ ReferencesResult TranslationUnit::references(uint line, uint column, bool localR
 
 DiagnosticSet TranslationUnit::diagnostics() const
 {
-    return DiagnosticSet(clang_getDiagnosticSetFromTU(m_cxTranslationUnit));
+    return DiagnosticSet(m_cxTranslationUnit, clang_getDiagnosticSetFromTU(m_cxTranslationUnit));
 }
 
 SourceLocation TranslationUnit::sourceLocationAt(uint line,uint column) const

src/tools/clangbackend/source/cursor.cpp

@@ -283,7 +283,7 @@ CXFile Cursor::includedFile() const
 
 SourceLocation Cursor::sourceLocation() const
 {
-    return clang_getCursorLocation(cxCursor);
+    return {cxTranslationUnit(), clang_getCursorLocation(cxCursor)};
 }
 
 CXSourceLocation Cursor::cxSourceLocation() const
@@ -293,7 +293,7 @@ CXSourceLocation Cursor::cxSourceLocation() const
 
 SourceRange Cursor::sourceRange() const
 {
-    return clang_getCursorExtent(cxCursor);
+    return {cxTranslationUnit(), clang_getCursorExtent(cxCursor)};
 }
 
 CXSourceRange Cursor::cxSourceRange() const
@@ -308,7 +308,7 @@ CXTranslationUnit Cursor::cxTranslationUnit() const
 
 SourceRange Cursor::commentRange() const
 {
-    return clang_Cursor_getCommentRange(cxCursor);
+    return {cxTranslationUnit(), clang_Cursor_getCommentRange(cxCursor)};
 }
 
 bool Cursor::hasSameSourceLocationAs(const Cursor &other) const

src/tools/clangbackend/source/diagnostic.cpp

@@ -36,8 +36,9 @@
 
 namespace ClangBackEnd {
 
-Diagnostic::Diagnostic(CXDiagnostic cxDiagnostic)
+Diagnostic::Diagnostic(CXTranslationUnit translationUnit, CXDiagnostic cxDiagnostic)
-    : cxDiagnostic(cxDiagnostic)
+    : cxDiagnostic(cxDiagnostic),
+      cxTranslationUnit(translationUnit)
 {
 }
 
@@ -47,9 +48,11 @@ Diagnostic::~Diagnostic()
 }
 
 Diagnostic::Diagnostic(Diagnostic &&other)
-    : cxDiagnostic(std::move(other.cxDiagnostic))
+    : cxDiagnostic(std::move(other.cxDiagnostic)),
+      cxTranslationUnit(std::move(other.cxTranslationUnit))
 {
     other.cxDiagnostic = nullptr;
+    other.cxTranslationUnit = nullptr;
 }
 
 Diagnostic &Diagnostic::operator=(Diagnostic &&other)
@@ -57,7 +60,9 @@ Diagnostic &Diagnostic::operator=(Diagnostic &&other)
     if (this != &other) {
         clang_disposeDiagnostic(cxDiagnostic);
         cxDiagnostic = std::move(other.cxDiagnostic);
+        cxTranslationUnit = std::move(other.cxTranslationUnit);
         other.cxDiagnostic = nullptr;
+        other.cxTranslationUnit = nullptr;
     }
 
     return *this;
@@ -90,7 +95,7 @@ std::pair<Utf8String, Utf8String> Diagnostic::options() const
 
 SourceLocation Diagnostic::location() const
 {
-    return SourceLocation(clang_getDiagnosticLocation(cxDiagnostic));
+    return {cxTranslationUnit, clang_getDiagnosticLocation(cxDiagnostic)};
 }
 
 DiagnosticSeverity Diagnostic::severity() const
@@ -105,7 +110,8 @@ std::vector<SourceRange> Diagnostic::ranges() const
     ranges.reserve(rangesCount);
 
     for (uint index = 0; index < rangesCount; ++index) {
-        const SourceRange sourceRange(clang_getDiagnosticRange(cxDiagnostic, index));
+        const SourceRange sourceRange {cxTranslationUnit,
+                                       clang_getDiagnosticRange(cxDiagnostic, index)};
 
         if (sourceRange.isValid())
             ranges.push_back(std::move(sourceRange));
@@ -123,14 +129,14 @@ std::vector<FixIt> Diagnostic::fixIts() const
     fixIts.reserve(fixItsCount);
 
     for (uint index = 0; index < fixItsCount; ++index)
-        fixIts.push_back(FixIt(cxDiagnostic, index));
+        fixIts.push_back(FixIt(cxTranslationUnit, cxDiagnostic, index));
 
     return fixIts;
 }
 
 DiagnosticSet Diagnostic::childDiagnostics() const
 {
-    return DiagnosticSet(clang_getChildDiagnostics(cxDiagnostic));
+    return DiagnosticSet(cxTranslationUnit, clang_getChildDiagnostics(cxDiagnostic));
 }
 
 DiagnosticContainer Diagnostic::toDiagnosticContainer() const

src/tools/clangbackend/source/diagnostic.h

@@ -73,12 +73,13 @@ public:
     DiagnosticContainer toDiagnosticContainer() const;
 
 private:
-    Diagnostic(CXDiagnostic cxDiagnostic);
+    Diagnostic(CXTranslationUnit translationUnit, CXDiagnostic cxDiagnostic);
     QVector<SourceRangeContainer> getSourceRangeContainers() const;
     QVector<FixItContainer> getFixItContainers() const;
 
 private:
     CXDiagnostic cxDiagnostic;
+    CXTranslationUnit cxTranslationUnit;
 };
 
 inline bool operator==(Diagnostic first, Diagnostic second)

src/tools/clangbackend/source/diagnosticset.cpp

@@ -33,8 +33,9 @@
 
 namespace ClangBackEnd {
 
-DiagnosticSet::DiagnosticSet(CXDiagnosticSet cxDiagnosticSet)
+DiagnosticSet::DiagnosticSet(CXTranslationUnit translationUnit, CXDiagnosticSet cxDiagnosticSet)
-    : cxDiagnosticSet(cxDiagnosticSet)
+    : cxDiagnosticSet(cxDiagnosticSet),
+      cxTranslationUnit(translationUnit)
 {
 }
 
@@ -44,7 +45,8 @@ DiagnosticSet::~DiagnosticSet()
 }
 
 DiagnosticSet::DiagnosticSet(DiagnosticSet &&other)
-    : cxDiagnosticSet(std::move(other.cxDiagnosticSet))
+    : cxDiagnosticSet(std::move(other.cxDiagnosticSet)),
+      cxTranslationUnit(std::move(other.cxTranslationUnit))
 {
     other.cxDiagnosticSet = nullptr;
 }
@@ -54,7 +56,9 @@ DiagnosticSet &DiagnosticSet::operator=(DiagnosticSet &&other)
     if (this != &other) {
         clang_disposeDiagnosticSet(cxDiagnosticSet);
         cxDiagnosticSet = std::move(other.cxDiagnosticSet);
+        cxTranslationUnit = std::move(other.cxTranslationUnit);
         other.cxDiagnosticSet = nullptr;
+        other.cxTranslationUnit = nullptr;
     }
 
     return *this;
@@ -62,22 +66,22 @@ DiagnosticSet &DiagnosticSet::operator=(DiagnosticSet &&other)
 
 Diagnostic DiagnosticSet::front() const
 {
-    return Diagnostic(clang_getDiagnosticInSet(cxDiagnosticSet, 0));
+    return Diagnostic(cxTranslationUnit, clang_getDiagnosticInSet(cxDiagnosticSet, 0));
 }
 
 Diagnostic DiagnosticSet::back() const
 {
-    return Diagnostic(clang_getDiagnosticInSet(cxDiagnosticSet, size() - 1));
+    return Diagnostic(cxTranslationUnit, clang_getDiagnosticInSet(cxDiagnosticSet, size() - 1));
 }
 
 DiagnosticSet::ConstIterator DiagnosticSet::begin() const
 {
-    return DiagnosticSetIterator(cxDiagnosticSet, 0);
+    return DiagnosticSetIterator(cxTranslationUnit, cxDiagnosticSet, 0);
 }
 
 DiagnosticSet::ConstIterator DiagnosticSet::end() const
 {
-    return DiagnosticSetIterator(cxDiagnosticSet, size());
+    return DiagnosticSetIterator(cxTranslationUnit, cxDiagnosticSet, size());
 }
 
 QVector<DiagnosticContainer> DiagnosticSet::toDiagnosticContainers() const
@@ -113,7 +117,7 @@ bool DiagnosticSet::isNull() const
 
 Diagnostic DiagnosticSet::at(uint index) const
 {
-    return Diagnostic(clang_getDiagnosticInSet(cxDiagnosticSet, index));
+    return Diagnostic(cxTranslationUnit, clang_getDiagnosticInSet(cxDiagnosticSet, index));
 }
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/diagnosticset.h

@@ -72,10 +72,11 @@ public:
             const IsAcceptedDiagnostic &isAcceptedDiagnostic) const;
 
 private:
-    DiagnosticSet(CXDiagnosticSet cxDiagnosticSet);
+    DiagnosticSet(CXTranslationUnit translationUnit, CXDiagnosticSet cxDiagnosticSet);
 
 private:
     CXDiagnosticSet cxDiagnosticSet;
+    CXTranslationUnit cxTranslationUnit;
 };
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/diagnosticsetiterator.h

@@ -25,6 +25,8 @@
 
 #pragma once
 
+#include "diagnostic.h"
+
 #include <iterator>
 
 #include <clang-c/Index.h>
@@ -34,13 +36,15 @@ namespace ClangBackEnd {
 using uint = unsigned int;
 
 class DiagnosticSet;
-class Diagnostic;
 
 class DiagnosticSetIterator : public std::iterator<std::random_access_iterator_tag, Diagnostic, uint>
 {
 public:
-    DiagnosticSetIterator(CXDiagnosticSet cxDiagnosticSet, uint index)
+    DiagnosticSetIterator(CXTranslationUnit translationUnit,
+                          CXDiagnosticSet cxDiagnosticSet,
+                          uint index)
         : cxDiagnosticSet(cxDiagnosticSet),
+          cxTranslationUnit(translationUnit),
           index(index)
     {}
 
@@ -58,7 +62,7 @@ public:
     DiagnosticSetIterator operator++(int)
     {
         uint oldIndex = index++;
-        return DiagnosticSetIterator(cxDiagnosticSet, oldIndex);
+        return DiagnosticSetIterator(cxTranslationUnit, cxDiagnosticSet, oldIndex);
     }
 
     bool operator==(const DiagnosticSetIterator &other)
@@ -73,11 +77,12 @@ public:
 
     Diagnostic operator*()
     {
-        return Diagnostic(clang_getDiagnosticInSet(cxDiagnosticSet, index));
+        return Diagnostic(cxTranslationUnit, clang_getDiagnosticInSet(cxDiagnosticSet, index));
     }
 
 private:
     CXDiagnosticSet cxDiagnosticSet;
+    CXTranslationUnit cxTranslationUnit;
     uint index;
 };
 

src/tools/clangbackend/source/fixit.cpp

@@ -46,12 +46,12 @@ FixItContainer FixIt::toFixItContainer() const
     return FixItContainer(text_, sourceRange.toSourceRangeContainer());
 }
 
-FixIt::FixIt(CXDiagnostic cxDiagnostic, uint index)
+FixIt::FixIt(CXTranslationUnit translationUnit, CXDiagnostic cxDiagnostic, uint index)
 {
     CXSourceRange cxSourceRange;
 
     text_ = ClangString(clang_getDiagnosticFixIt(cxDiagnostic, index, &cxSourceRange));
-    sourceRange = SourceRange(cxSourceRange);
+    sourceRange = SourceRange(translationUnit, cxSourceRange);
 }
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/fixit.h

@@ -47,7 +47,7 @@ public:
     FixItContainer toFixItContainer() const;
 
 private:
-    FixIt(CXDiagnostic cxDiagnostic, uint index);
+    FixIt(CXTranslationUnit translationUnit, CXDiagnostic cxDiagnostic, uint index);
 
 private:
     SourceRange sourceRange;

src/tools/clangbackend/source/skippedsourceranges.cpp

@@ -79,7 +79,7 @@ std::vector<SourceRange> SkippedSourceRanges::sourceRanges() const
     sourceRanges.reserve(sourceRangeCount);
 
     for (uint i = 0; i < cxSkippedSourceRanges->count; ++i) {
-        const SourceRange range = cxSkippedSourceRanges->ranges[i];
+        const SourceRange range {cxTranslationUnit, cxSkippedSourceRanges->ranges[i]};
         const SourceRange adaptedRange = adaptedSourceRange(cxTranslationUnit, range);
 
         sourceRanges.push_back(adaptedRange);

src/tools/clangbackend/source/sourcelocation.cpp

@@ -29,10 +29,13 @@
 #include "clangfilepath.h"
 #include "clangstring.h"
 
-#include <utf8string.h>
+#include <clangsupport/sourcelocationcontainer.h>
+
+#include <sqlite/utf8string.h>
+
+#include <utils/textutils.h>
 
 #include <ostream>
-#include <sourcelocationcontainer.h>
 
 namespace ClangBackEnd {
 
@@ -72,8 +75,10 @@ SourceLocationContainer SourceLocation::toSourceLocationContainer() const
     return SourceLocationContainer(filePath(), line_, column_);
 }
 
-SourceLocation::SourceLocation(CXSourceLocation cxSourceLocation)
+SourceLocation::SourceLocation(CXTranslationUnit cxTranslationUnit,
+                               CXSourceLocation cxSourceLocation)
     : cxSourceLocation(cxSourceLocation)
+    , cxTranslationUnit(cxTranslationUnit)
 {
     CXFile cxFile;
 
@@ -83,8 +88,22 @@ SourceLocation::SourceLocation(CXSourceLocation cxSourceLocation)
                           &column_,
                           &offset_);
 
-    filePath_ = ClangString(clang_getFileName(cxFile));
     isFilePathNormalized_ = false;
+    if (!cxFile)
+        return;
+
+    filePath_ = ClangString(clang_getFileName(cxFile));
+// CLANG-UPGRADE-CHECK: Remove HAS_GETFILECONTENTS_BACKPORTED check once we require clang >= 6.0
+#if defined(CINDEX_VERSION_HAS_GETFILECONTENTS_BACKPORTED) || CINDEX_VERSION_MINOR >= 44
+    if (column_ > 1) {
+        const uint lineStart = offset_ + 1 - column_;
+        const char *contents = clang_getFileContents(cxTranslationUnit, cxFile, nullptr);
+        if (!contents)
+            return;
+        column_ = static_cast<uint>(QString::fromUtf8(&contents[lineStart],
+                                                      static_cast<int>(column_)).size());
+    }
+#endif
 }
 
 SourceLocation::SourceLocation(CXTranslationUnit cxTranslationUnit,
@@ -96,6 +115,7 @@ SourceLocation::SourceLocation(CXTranslationUnit cxTranslationUnit,
                                                        filePath.constData()),
                                          line,
                                          column)),
+      cxTranslationUnit(cxTranslationUnit),
       filePath_(filePath),
       line_(line),
       column_(column),

src/tools/clangbackend/source/sourcelocation.h

@@ -57,12 +57,14 @@ public:
     SourceLocationContainer toSourceLocationContainer() const;
 
 private:
-    SourceLocation(CXSourceLocation cxSourceLocation);
+    SourceLocation(CXTranslationUnit cxTranslationUnit,
+                   CXSourceLocation cxSourceLocation);
 
     operator CXSourceLocation() const;
 
 private:
    CXSourceLocation cxSourceLocation;
+   CXTranslationUnit cxTranslationUnit;
    mutable Utf8String filePath_;
    uint line_ = 0;
    uint column_ = 0;

src/tools/clangbackend/source/sourcerange.cpp

@@ -39,7 +39,8 @@ SourceRange::SourceRange()
 }
 
 SourceRange::SourceRange(const SourceLocation &start, const SourceLocation &end)
-    : cxSourceRange(clang_getRange(start, end))
+    : cxSourceRange(clang_getRange(start, end)),
+      cxTranslationUnit(start.cxTranslationUnit)
 {
 }
 
@@ -55,12 +56,12 @@ bool SourceRange::isValid() const
 
 SourceLocation SourceRange::start() const
 {
-    return SourceLocation(clang_getRangeStart(cxSourceRange));
+    return {cxTranslationUnit, clang_getRangeStart(cxSourceRange)};
 }
 
 SourceLocation SourceRange::end() const
 {
-    return SourceLocation(clang_getRangeEnd(cxSourceRange));
+    return {cxTranslationUnit, clang_getRangeEnd(cxSourceRange)};
 }
 
 bool SourceRange::contains(unsigned line, unsigned column) const
@@ -90,8 +91,9 @@ ClangBackEnd::SourceRange::operator CXSourceRange() const
     return cxSourceRange;
 }
 
-SourceRange::SourceRange(CXSourceRange cxSourceRange)
+SourceRange::SourceRange(CXTranslationUnit translationUnit, CXSourceRange cxSourceRange)
-    : cxSourceRange(cxSourceRange)
+    : cxSourceRange(cxSourceRange),
+      cxTranslationUnit(translationUnit)
 {
 }
 

src/tools/clangbackend/source/sourcerange.h

@@ -40,7 +40,7 @@ class SourceRange
 
 public:
     SourceRange();
-    SourceRange(CXSourceRange cxSourceRange);
+    SourceRange(CXTranslationUnit cxTranslationUnit, CXSourceRange cxSourceRange);
     SourceRange(const SourceLocation &start, const SourceLocation &end);
 
     bool isNull() const;
@@ -58,6 +58,7 @@ public:
 
 private:
     CXSourceRange cxSourceRange;
+    CXTranslationUnit cxTranslationUnit = nullptr;
 };
 
 bool operator==(const SourceRange &first, const SourceRange &second);

src/tools/clangbackend/source/tokeninfo.cpp

@@ -43,7 +43,8 @@ TokenInfo::TokenInfo(const CXCursor &cxCursor,
     : m_currentOutputArgumentRanges(&currentOutputArgumentRanges),
       m_originalCursor(cxCursor)
 {
-    const SourceRange sourceRange = clang_getTokenExtent(cxTranslationUnit, *cxToken);
+    const SourceRange sourceRange {cxTranslationUnit,
+                                   clang_getTokenExtent(cxTranslationUnit, *cxToken)};
     const auto start = sourceRange.start();
     const auto end = sourceRange.end();