Clang: fix utf8 related column numbers

Use new clang_getFileContents to efficiently convert
utf8 byte offsets from line start to column numbers.
Also provide simplier backwards convertion to pass
resulting utf8 offset to clang.

Task-number: QTCREATORBUG-16941
Change-Id: If0e58fe01ad3e281b7e952e972b9e86f6e75aadb
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@qt.io>

src/tools/clangbackend/source/clangfollowsymbol.cpp

@@ -107,7 +107,7 @@ static SourceRange getOperatorRange(const CXTranslationUnit tu,
         ++operatorIndex;
     }
     const CXSourceLocation end = clang_getTokenLocation(tu, tokens.data[operatorIndex]);
-    return SourceRange(clang_getRange(start, end));
+    return SourceRange(tu, clang_getRange(start, end));
 }
 
 static SourceRangeContainer extractMatchingTokenRange(const Cursor &cursor,
@@ -128,7 +128,7 @@ static SourceRangeContainer extractMatchingTokenRange(const Cursor &cursor,
                 continue;
             }
         }
-        return SourceRange(clang_getTokenExtent(tu, tokens.data[i]));
+        return SourceRange(tu, clang_getTokenExtent(tu, tokens.data[i]));
     }
     return SourceRangeContainer();
 }
@@ -137,7 +137,7 @@ static int getTokenIndex(CXTranslationUnit tu, const Tokens &tokens, uint line,
 {
     int tokenIndex = -1;
     for (int i = static_cast<int>(tokens.tokenCount - 1); i >= 0; --i) {
-        const SourceRange range = clang_getTokenExtent(tu, tokens.data[i]);
+        const SourceRange range(tu, clang_getTokenExtent(tu, tokens.data[i]));
         if (range.contains(line, column)) {
             tokenIndex = i;
             break;

src/tools/clangbackend/source/clangreferencescollector.cpp

@@ -155,7 +155,7 @@ ReferencesCollector::~ReferencesCollector()
 
 bool ReferencesCollector::isWithinTokenRange(CXToken token, uint line, uint column) const
 {
-    const SourceRange range = clang_getTokenExtent(m_cxTranslationUnit, token);
+    const SourceRange range {m_cxTranslationUnit, clang_getTokenExtent(m_cxTranslationUnit, token)};
     return range.contains(line, column);
 }
 
@@ -229,7 +229,8 @@ ReferencesResult ReferencesCollector::collect(uint line, uint column, bool local
     const Utf8String identifier = ClangString(clang_getTokenSpelling(m_cxTranslationUnit, token));
     for (uint i = 0; i < m_cxTokenCount; ++i) {
         if (checkToken(i, identifier, usr)) {
-            const SourceRange range = clang_getTokenExtent(m_cxTranslationUnit, m_cxTokens[i]);
+            const SourceRange range {m_cxTranslationUnit,
+                                     clang_getTokenExtent(m_cxTranslationUnit, m_cxTokens[i])};
             result.references.append(range);
         }
     }

src/tools/clangbackend/source/clangtranslationunit.cpp

@@ -161,7 +161,7 @@ ReferencesResult TranslationUnit::references(uint line, uint column, bool localR
 
 DiagnosticSet TranslationUnit::diagnostics() const
 {
-    return DiagnosticSet(clang_getDiagnosticSetFromTU(m_cxTranslationUnit));
+    return DiagnosticSet(m_cxTranslationUnit, clang_getDiagnosticSetFromTU(m_cxTranslationUnit));
 }
 
 SourceLocation TranslationUnit::sourceLocationAt(uint line,uint column) const

src/tools/clangbackend/source/cursor.cpp

@@ -283,7 +283,7 @@ CXFile Cursor::includedFile() const
 
 SourceLocation Cursor::sourceLocation() const
 {
-    return clang_getCursorLocation(cxCursor);
+    return {cxTranslationUnit(), clang_getCursorLocation(cxCursor)};
 }
 
 CXSourceLocation Cursor::cxSourceLocation() const
@@ -293,7 +293,7 @@ CXSourceLocation Cursor::cxSourceLocation() const
 
 SourceRange Cursor::sourceRange() const
 {
-    return clang_getCursorExtent(cxCursor);
+    return {cxTranslationUnit(), clang_getCursorExtent(cxCursor)};
 }
 
 CXSourceRange Cursor::cxSourceRange() const
@@ -308,7 +308,7 @@ CXTranslationUnit Cursor::cxTranslationUnit() const
 
 SourceRange Cursor::commentRange() const
 {
-    return clang_Cursor_getCommentRange(cxCursor);
+    return {cxTranslationUnit(), clang_Cursor_getCommentRange(cxCursor)};
 }
 
 bool Cursor::hasSameSourceLocationAs(const Cursor &other) const

src/tools/clangbackend/source/diagnostic.cpp

@@ -36,8 +36,9 @@
 
 namespace ClangBackEnd {
 
-Diagnostic::Diagnostic(CXDiagnostic cxDiagnostic)
-    : cxDiagnostic(cxDiagnostic)
+Diagnostic::Diagnostic(CXTranslationUnit translationUnit, CXDiagnostic cxDiagnostic)
+    : cxDiagnostic(cxDiagnostic),
+      cxTranslationUnit(translationUnit)
 {
 }
 
@@ -47,9 +48,11 @@ Diagnostic::~Diagnostic()
 }
 
 Diagnostic::Diagnostic(Diagnostic &&other)
-    : cxDiagnostic(std::move(other.cxDiagnostic))
+    : cxDiagnostic(std::move(other.cxDiagnostic)),
+      cxTranslationUnit(std::move(other.cxTranslationUnit))
 {
     other.cxDiagnostic = nullptr;
+    other.cxTranslationUnit = nullptr;
 }
 
 Diagnostic &Diagnostic::operator=(Diagnostic &&other)
@@ -57,7 +60,9 @@ Diagnostic &Diagnostic::operator=(Diagnostic &&other)
     if (this != &other) {
         clang_disposeDiagnostic(cxDiagnostic);
         cxDiagnostic = std::move(other.cxDiagnostic);
+        cxTranslationUnit = std::move(other.cxTranslationUnit);
         other.cxDiagnostic = nullptr;
+        other.cxTranslationUnit = nullptr;
     }
 
     return *this;
@@ -90,7 +95,7 @@ std::pair<Utf8String, Utf8String> Diagnostic::options() const
 
 SourceLocation Diagnostic::location() const
 {
-    return SourceLocation(clang_getDiagnosticLocation(cxDiagnostic));
+    return {cxTranslationUnit, clang_getDiagnosticLocation(cxDiagnostic)};
 }
 
 DiagnosticSeverity Diagnostic::severity() const
@@ -105,7 +110,8 @@ std::vector<SourceRange> Diagnostic::ranges() const
     ranges.reserve(rangesCount);
 
     for (uint index = 0; index < rangesCount; ++index) {
-        const SourceRange sourceRange(clang_getDiagnosticRange(cxDiagnostic, index));
+        const SourceRange sourceRange {cxTranslationUnit,
+                                       clang_getDiagnosticRange(cxDiagnostic, index)};
 
         if (sourceRange.isValid())
             ranges.push_back(std::move(sourceRange));
@@ -123,14 +129,14 @@ std::vector<FixIt> Diagnostic::fixIts() const
     fixIts.reserve(fixItsCount);
 
     for (uint index = 0; index < fixItsCount; ++index)
-        fixIts.push_back(FixIt(cxDiagnostic, index));
+        fixIts.push_back(FixIt(cxTranslationUnit, cxDiagnostic, index));
 
     return fixIts;
 }
 
 DiagnosticSet Diagnostic::childDiagnostics() const
 {
-    return DiagnosticSet(clang_getChildDiagnostics(cxDiagnostic));
+    return DiagnosticSet(cxTranslationUnit, clang_getChildDiagnostics(cxDiagnostic));
 }
 
 DiagnosticContainer Diagnostic::toDiagnosticContainer() const

src/tools/clangbackend/source/diagnostic.h

@@ -73,12 +73,13 @@ public:
     DiagnosticContainer toDiagnosticContainer() const;
 
 private:
-    Diagnostic(CXDiagnostic cxDiagnostic);
+    Diagnostic(CXTranslationUnit translationUnit, CXDiagnostic cxDiagnostic);
     QVector<SourceRangeContainer> getSourceRangeContainers() const;
     QVector<FixItContainer> getFixItContainers() const;
 
 private:
     CXDiagnostic cxDiagnostic;
+    CXTranslationUnit cxTranslationUnit;
 };
 
 inline bool operator==(Diagnostic first, Diagnostic second)

src/tools/clangbackend/source/diagnosticset.cpp

@@ -33,8 +33,9 @@
 
 namespace ClangBackEnd {
 
-DiagnosticSet::DiagnosticSet(CXDiagnosticSet cxDiagnosticSet)
-    : cxDiagnosticSet(cxDiagnosticSet)
+DiagnosticSet::DiagnosticSet(CXTranslationUnit translationUnit, CXDiagnosticSet cxDiagnosticSet)
+    : cxDiagnosticSet(cxDiagnosticSet),
+      cxTranslationUnit(translationUnit)
 {
 }
 
@@ -44,7 +45,8 @@ DiagnosticSet::~DiagnosticSet()
 }
 
 DiagnosticSet::DiagnosticSet(DiagnosticSet &&other)
-    : cxDiagnosticSet(std::move(other.cxDiagnosticSet))
+    : cxDiagnosticSet(std::move(other.cxDiagnosticSet)),
+      cxTranslationUnit(std::move(other.cxTranslationUnit))
 {
     other.cxDiagnosticSet = nullptr;
 }
@@ -54,7 +56,9 @@ DiagnosticSet &DiagnosticSet::operator=(DiagnosticSet &&other)
     if (this != &other) {
         clang_disposeDiagnosticSet(cxDiagnosticSet);
         cxDiagnosticSet = std::move(other.cxDiagnosticSet);
+        cxTranslationUnit = std::move(other.cxTranslationUnit);
         other.cxDiagnosticSet = nullptr;
+        other.cxTranslationUnit = nullptr;
     }
 
     return *this;
@@ -62,22 +66,22 @@ DiagnosticSet &DiagnosticSet::operator=(DiagnosticSet &&other)
 
 Diagnostic DiagnosticSet::front() const
 {
-    return Diagnostic(clang_getDiagnosticInSet(cxDiagnosticSet, 0));
+    return Diagnostic(cxTranslationUnit, clang_getDiagnosticInSet(cxDiagnosticSet, 0));
 }
 
 Diagnostic DiagnosticSet::back() const
 {
-    return Diagnostic(clang_getDiagnosticInSet(cxDiagnosticSet, size() - 1));
+    return Diagnostic(cxTranslationUnit, clang_getDiagnosticInSet(cxDiagnosticSet, size() - 1));
 }
 
 DiagnosticSet::ConstIterator DiagnosticSet::begin() const
 {
-    return DiagnosticSetIterator(cxDiagnosticSet, 0);
+    return DiagnosticSetIterator(cxTranslationUnit, cxDiagnosticSet, 0);
 }
 
 DiagnosticSet::ConstIterator DiagnosticSet::end() const
 {
-    return DiagnosticSetIterator(cxDiagnosticSet, size());
+    return DiagnosticSetIterator(cxTranslationUnit, cxDiagnosticSet, size());
 }
 
 QVector<DiagnosticContainer> DiagnosticSet::toDiagnosticContainers() const
@@ -113,7 +117,7 @@ bool DiagnosticSet::isNull() const
 
 Diagnostic DiagnosticSet::at(uint index) const
 {
-    return Diagnostic(clang_getDiagnosticInSet(cxDiagnosticSet, index));
+    return Diagnostic(cxTranslationUnit, clang_getDiagnosticInSet(cxDiagnosticSet, index));
 }
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/diagnosticset.h

@@ -72,10 +72,11 @@ public:
             const IsAcceptedDiagnostic &isAcceptedDiagnostic) const;
 
 private:
-    DiagnosticSet(CXDiagnosticSet cxDiagnosticSet);
+    DiagnosticSet(CXTranslationUnit translationUnit, CXDiagnosticSet cxDiagnosticSet);
 
 private:
     CXDiagnosticSet cxDiagnosticSet;
+    CXTranslationUnit cxTranslationUnit;
 };
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/diagnosticsetiterator.h

@@ -25,6 +25,8 @@
 
 #pragma once
 
+#include "diagnostic.h"
+
 #include <iterator>
 
 #include <clang-c/Index.h>
@@ -34,13 +36,15 @@ namespace ClangBackEnd {
 using uint = unsigned int;
 
 class DiagnosticSet;
-class Diagnostic;
 
 class DiagnosticSetIterator : public std::iterator<std::random_access_iterator_tag, Diagnostic, uint>
 {
 public:
-    DiagnosticSetIterator(CXDiagnosticSet cxDiagnosticSet, uint index)
+    DiagnosticSetIterator(CXTranslationUnit translationUnit,
+                          CXDiagnosticSet cxDiagnosticSet,
+                          uint index)
         : cxDiagnosticSet(cxDiagnosticSet),
+          cxTranslationUnit(translationUnit),
           index(index)
     {}
 
@@ -58,7 +62,7 @@ public:
     DiagnosticSetIterator operator++(int)
     {
         uint oldIndex = index++;
-        return DiagnosticSetIterator(cxDiagnosticSet, oldIndex);
+        return DiagnosticSetIterator(cxTranslationUnit, cxDiagnosticSet, oldIndex);
     }
 
     bool operator==(const DiagnosticSetIterator &other)
@@ -73,11 +77,12 @@ public:
 
     Diagnostic operator*()
     {
-        return Diagnostic(clang_getDiagnosticInSet(cxDiagnosticSet, index));
+        return Diagnostic(cxTranslationUnit, clang_getDiagnosticInSet(cxDiagnosticSet, index));
     }
 
 private:
     CXDiagnosticSet cxDiagnosticSet;
+    CXTranslationUnit cxTranslationUnit;
     uint index;
 };
 

src/tools/clangbackend/source/fixit.cpp

@@ -46,12 +46,12 @@ FixItContainer FixIt::toFixItContainer() const
     return FixItContainer(text_, sourceRange.toSourceRangeContainer());
 }
 
-FixIt::FixIt(CXDiagnostic cxDiagnostic, uint index)
+FixIt::FixIt(CXTranslationUnit translationUnit, CXDiagnostic cxDiagnostic, uint index)
 {
     CXSourceRange cxSourceRange;
 
     text_ = ClangString(clang_getDiagnosticFixIt(cxDiagnostic, index, &cxSourceRange));
-    sourceRange = SourceRange(cxSourceRange);
+    sourceRange = SourceRange(translationUnit, cxSourceRange);
 }
 
 } // namespace ClangBackEnd

src/tools/clangbackend/source/fixit.h

@@ -47,7 +47,7 @@ public:
     FixItContainer toFixItContainer() const;
 
 private:
-    FixIt(CXDiagnostic cxDiagnostic, uint index);
+    FixIt(CXTranslationUnit translationUnit, CXDiagnostic cxDiagnostic, uint index);
 
 private:
     SourceRange sourceRange;

src/tools/clangbackend/source/skippedsourceranges.cpp

@@ -79,7 +79,7 @@ std::vector<SourceRange> SkippedSourceRanges::sourceRanges() const
     sourceRanges.reserve(sourceRangeCount);
 
     for (uint i = 0; i < cxSkippedSourceRanges->count; ++i) {
-        const SourceRange range = cxSkippedSourceRanges->ranges[i];
+        const SourceRange range {cxTranslationUnit, cxSkippedSourceRanges->ranges[i]};
         const SourceRange adaptedRange = adaptedSourceRange(cxTranslationUnit, range);
 
         sourceRanges.push_back(adaptedRange);

src/tools/clangbackend/source/sourcelocation.cpp

@@ -29,10 +29,13 @@
 #include "clangfilepath.h"
 #include "clangstring.h"
 
-#include <utf8string.h>
+#include <clangsupport/sourcelocationcontainer.h>
+
+#include <sqlite/utf8string.h>
+
+#include <utils/textutils.h>
 
 #include <ostream>
-#include <sourcelocationcontainer.h>
 
 namespace ClangBackEnd {
 
@@ -72,8 +75,10 @@ SourceLocationContainer SourceLocation::toSourceLocationContainer() const
     return SourceLocationContainer(filePath(), line_, column_);
 }
 
-SourceLocation::SourceLocation(CXSourceLocation cxSourceLocation)
+SourceLocation::SourceLocation(CXTranslationUnit cxTranslationUnit,
+                               CXSourceLocation cxSourceLocation)
     : cxSourceLocation(cxSourceLocation)
+    , cxTranslationUnit(cxTranslationUnit)
 {
     CXFile cxFile;
 
@@ -83,8 +88,22 @@ SourceLocation::SourceLocation(CXSourceLocation cxSourceLocation)
                           &column_,
                           &offset_);
 
-    filePath_ = ClangString(clang_getFileName(cxFile));
     isFilePathNormalized_ = false;
+    if (!cxFile)
+        return;
+
+    filePath_ = ClangString(clang_getFileName(cxFile));
+// CLANG-UPGRADE-CHECK: Remove HAS_GETFILECONTENTS_BACKPORTED check once we require clang >= 6.0
+#if defined(CINDEX_VERSION_HAS_GETFILECONTENTS_BACKPORTED) || CINDEX_VERSION_MINOR >= 44
+    if (column_ > 1) {
+        const uint lineStart = offset_ + 1 - column_;
+        const char *contents = clang_getFileContents(cxTranslationUnit, cxFile, nullptr);
+        if (!contents)
+            return;
+        column_ = static_cast<uint>(QString::fromUtf8(&contents[lineStart],
+                                                      static_cast<int>(column_)).size());
+    }
+#endif
 }
 
 SourceLocation::SourceLocation(CXTranslationUnit cxTranslationUnit,
@@ -96,6 +115,7 @@ SourceLocation::SourceLocation(CXTranslationUnit cxTranslationUnit,
                                                        filePath.constData()),
                                          line,
                                          column)),
+      cxTranslationUnit(cxTranslationUnit),
       filePath_(filePath),
       line_(line),
       column_(column),

src/tools/clangbackend/source/sourcelocation.h

@@ -57,12 +57,14 @@ public:
     SourceLocationContainer toSourceLocationContainer() const;
 
 private:
-    SourceLocation(CXSourceLocation cxSourceLocation);
+    SourceLocation(CXTranslationUnit cxTranslationUnit,
+                   CXSourceLocation cxSourceLocation);
 
     operator CXSourceLocation() const;
 
 private:
    CXSourceLocation cxSourceLocation;
+   CXTranslationUnit cxTranslationUnit;
    mutable Utf8String filePath_;
    uint line_ = 0;
    uint column_ = 0;

src/tools/clangbackend/source/sourcerange.cpp

@@ -39,7 +39,8 @@ SourceRange::SourceRange()
 }
 
 SourceRange::SourceRange(const SourceLocation &start, const SourceLocation &end)
-    : cxSourceRange(clang_getRange(start, end))
+    : cxSourceRange(clang_getRange(start, end)),
+      cxTranslationUnit(start.cxTranslationUnit)
 {
 }
 
@@ -55,12 +56,12 @@ bool SourceRange::isValid() const
 
 SourceLocation SourceRange::start() const
 {
-    return SourceLocation(clang_getRangeStart(cxSourceRange));
+    return {cxTranslationUnit, clang_getRangeStart(cxSourceRange)};
 }
 
 SourceLocation SourceRange::end() const
 {
-    return SourceLocation(clang_getRangeEnd(cxSourceRange));
+    return {cxTranslationUnit, clang_getRangeEnd(cxSourceRange)};
 }
 
 bool SourceRange::contains(unsigned line, unsigned column) const
@@ -90,8 +91,9 @@ ClangBackEnd::SourceRange::operator CXSourceRange() const
     return cxSourceRange;
 }
 
-SourceRange::SourceRange(CXSourceRange cxSourceRange)
-    : cxSourceRange(cxSourceRange)
+SourceRange::SourceRange(CXTranslationUnit translationUnit, CXSourceRange cxSourceRange)
+    : cxSourceRange(cxSourceRange),
+      cxTranslationUnit(translationUnit)
 {
 }
 

src/tools/clangbackend/source/sourcerange.h

@@ -40,7 +40,7 @@ class SourceRange
 
 public:
     SourceRange();
-    SourceRange(CXSourceRange cxSourceRange);
+    SourceRange(CXTranslationUnit cxTranslationUnit, CXSourceRange cxSourceRange);
     SourceRange(const SourceLocation &start, const SourceLocation &end);
 
     bool isNull() const;
@@ -58,6 +58,7 @@ public:
 
 private:
     CXSourceRange cxSourceRange;
+    CXTranslationUnit cxTranslationUnit = nullptr;
 };
 
 bool operator==(const SourceRange &first, const SourceRange &second);

src/tools/clangbackend/source/tokeninfo.cpp

@@ -43,7 +43,8 @@ TokenInfo::TokenInfo(const CXCursor &cxCursor,
     : m_currentOutputArgumentRanges(&currentOutputArgumentRanges),
       m_originalCursor(cxCursor)
 {
-    const SourceRange sourceRange = clang_getTokenExtent(cxTranslationUnit, *cxToken);
+    const SourceRange sourceRange {cxTranslationUnit,
+                                   clang_getTokenExtent(cxTranslationUnit, *cxToken)};
     const auto start = sourceRange.start();
     const auto end = sourceRange.end();