Clang: implement followSymbol in TranslationUnit

Follow symbol in current TU or dependent files
Current algorithm tries to do the same as built-in
follow symbol but better.
Currently clang-based follow symbol has some limitations:
- following function usage may return the declaration
instead of definition because we don't have header
dependencies in backend
- overrides are not searched because of the same reason
and the amount of dependent files (parsing 250 files
takes a while)
- some includes are not handled correctly, in that case
we return failure and ask built-in code model to follow
(example: <QtGui> or other qt includes)

Change-Id: If35028ee0b5e818fdba29363c9520c5cca996348
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@qt.io>

src/tools/clangbackend/ipcsource/clangbackendclangipc-source.pri

@@ -16,6 +16,8 @@ HEADERS += \
     $$PWD/clangexceptions.h \
     $$PWD/clangfilepath.h \
     $$PWD/clangfilesystemwatcher.h \
+    $$PWD/clangfollowsymboljob.h \
+    $$PWD/clangfollowsymbol.h \
     $$PWD/clangiasyncjob.h \
     $$PWD/clangjobcontext.h \
     $$PWD/clangjobqueue.h \
@@ -55,8 +57,7 @@ HEADERS += \
     $$PWD/sourcerange.h \
     $$PWD/unsavedfile.h \
     $$PWD/unsavedfiles.h \
-    $$PWD/utf8positionfromlinecolumn.h \
+    $$PWD/utf8positionfromlinecolumn.h
-    $$PWD/clangfollowsymboljob.h
 
 SOURCES += \
     $$PWD/clangcodecompleteresults.cpp \
@@ -71,6 +72,8 @@ SOURCES += \
     $$PWD/clangexceptions.cpp \
     $$PWD/clangfilepath.cpp \
     $$PWD/clangfilesystemwatcher.cpp \
+    $$PWD/clangfollowsymboljob.cpp \
+    $$PWD/clangfollowsymbol.cpp \
     $$PWD/clangiasyncjob.cpp \
     $$PWD/clangjobcontext.cpp \
     $$PWD/clangjobqueue.cpp \
@@ -107,5 +110,4 @@ SOURCES += \
     $$PWD/sourcerange.cpp \
     $$PWD/unsavedfile.cpp \
     $$PWD/unsavedfiles.cpp \
-    $$PWD/utf8positionfromlinecolumn.cpp \
+    $$PWD/utf8positionfromlinecolumn.cpp
-    $$PWD/clangfollowsymboljob.cpp

src/tools/clangbackend/ipcsource/clangfollowsymbol.cpp

@@ -0,0 +1,316 @@
+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "clangfollowsymbol.h"
+#include "clangfollowsymboljob.h"
+#include "commandlinearguments.h"
+#include "cursor.h"
+#include "clangstring.h"
+#include "sourcerange.h"
+#include "clangbackendipcdebugutils.h"
+
+#include <utils/qtcassert.h>
+
+#include <future>
+
+namespace ClangBackEnd {
+
+namespace {
+
+struct Tokens
+{
+    Tokens(const Cursor &cursor) {
+        tu = cursor.cxTranslationUnit();
+        clang_tokenize(tu, cursor.cxSourceRange(), &data, &tokenCount);
+    }
+    ~Tokens() {
+        clang_disposeTokens(tu, data, tokenCount);
+    }
+
+    CXToken *data = nullptr;
+    uint tokenCount = 0;
+private:
+    CXTranslationUnit tu;
+};
+
+class FollowSymbolData {
+public:
+    FollowSymbolData() = delete;
+    FollowSymbolData(const Utf8String &usr, const Utf8String &tokenSpelling, bool isFunctionLike,
+                     std::atomic<bool> &ready)
+        : m_usr(usr)
+        , m_spelling(tokenSpelling)
+        , m_isFunctionLike(isFunctionLike)
+        , m_ready(ready)
+    {}
+    FollowSymbolData(const FollowSymbolData &other)
+        : m_usr(other.m_usr)
+        , m_spelling(other.m_spelling)
+        , m_isFunctionLike(other.m_isFunctionLike)
+        , m_ready(other.m_ready)
+    {}
+
+    const Utf8String &usr() const { return m_usr; }
+    const Utf8String &spelling() const { return m_spelling; }
+    bool isFunctionLike() const { return m_isFunctionLike; }
+    bool ready() const { return m_ready; }
+    const SourceRangeContainer &result() const { return m_result; }
+
+    void setReady(bool ready = true) { m_ready = ready; }
+    void setResult(const SourceRangeContainer &result) { m_result = result; }
+private:
+    const Utf8String &m_usr;
+    const Utf8String &m_spelling;
+    SourceRangeContainer m_result;
+    bool m_isFunctionLike;
+    std::atomic<bool> &m_ready;
+};
+
+} // anonymous namespace
+
+static SourceRangeContainer extractMatchingTokenRange(const Cursor &cursor,
+                                                      const Utf8String &tokenStr)
+{
+    Tokens tokens(cursor);
+    const CXTranslationUnit tu = cursor.cxTranslationUnit();
+    for (uint i = 0; i < tokens.tokenCount; ++i) {
+        if (!(tokenStr == ClangString(clang_getTokenSpelling(tu, tokens.data[i]))))
+            continue;
+
+        if (cursor.isFunctionLike()
+                && (i+1 > tokens.tokenCount
+                    || !(ClangString(clang_getTokenSpelling(tu, tokens.data[i+1])) == "("))) {
+            continue;
+        }
+        return SourceRange(clang_getTokenExtent(tu, tokens.data[i]));
+    }
+    return SourceRangeContainer();
+}
+
+static void handleDeclaration(CXClientData client_data, const CXIdxDeclInfo *declInfo)
+{
+    if (!declInfo || !declInfo->isDefinition)
+        return;
+
+    const Cursor currentCursor(declInfo->cursor);
+    auto* data = reinterpret_cast<FollowSymbolData*>(client_data);
+    if (data->ready())
+        return;
+
+    if (data->usr() != currentCursor.canonical().unifiedSymbolResolution())
+        return;
+
+    QString str = Utf8String(currentCursor.displayName());
+    if (currentCursor.isFunctionLike() || currentCursor.isConstructorOrDestructor()) {
+        if (!data->isFunctionLike())
+            return;
+        str = str.mid(0, str.indexOf('('));
+    } else if (data->isFunctionLike()) {
+        return;
+    }
+    if (!str.endsWith(data->spelling()))
+        return;
+    const CXTranslationUnit tu = clang_Cursor_getTranslationUnit(declInfo->cursor);
+    Tokens tokens(currentCursor);
+
+    for (uint i = 0; i < tokens.tokenCount; ++i) {
+        Utf8String curSpelling = ClangString(clang_getTokenSpelling(tu, tokens.data[i]));
+        if (data->spelling() == curSpelling) {
+            if (data->isFunctionLike()
+                    && (i+1 >= tokens.tokenCount
+                        || !(ClangString(clang_getTokenSpelling(tu, tokens.data[i+1])) == "("))) {
+                continue;
+            }
+            data->setResult(SourceRange(clang_getTokenExtent(tu, tokens.data[i])));
+            data->setReady();
+            return;
+        }
+    }
+}
+
+static int getTokenIndex(CXTranslationUnit tu, const Tokens &tokens, uint line, uint column)
+{
+    int tokenIndex = -1;
+    for (int i = static_cast<int>(tokens.tokenCount - 1); i >= 0; --i) {
+        const SourceRange range = clang_getTokenExtent(tu, tokens.data[i]);
+        if (range.contains(line, column)) {
+            tokenIndex = i;
+            break;
+        }
+    }
+    return tokenIndex;
+}
+
+static IndexerCallbacks createIndexerCallbacks()
+{
+    return {
+        [](CXClientData client_data, void *) {
+            auto* data = reinterpret_cast<FollowSymbolData*>(client_data);
+            return data->ready() ? 1 : 0;
+        },
+        [](CXClientData, CXDiagnosticSet, void *) {},
+        [](CXClientData, CXFile, void *) { return CXIdxClientFile(); },
+        [](CXClientData, const CXIdxIncludedFileInfo *) { return CXIdxClientFile(); },
+        [](CXClientData, const CXIdxImportedASTFileInfo *) { return CXIdxClientASTFile(); },
+        [](CXClientData, void *) { return CXIdxClientContainer(); },
+        handleDeclaration,
+        [](CXClientData, const CXIdxEntityRefInfo *) {}
+    };
+}
+
+static FollowSymbolResult followSymbolInDependentFiles(CXIndex index,
+                                                       const Cursor &cursor,
+                                                       const Utf8String &tokenSpelling,
+                                                       const QVector<Utf8String> &dependentFiles,
+                                                       const CommandLineArguments &currentArgs)
+{
+    int argsCount = 0;
+    if (currentArgs.data())
+        argsCount = currentArgs.count() - 1;
+
+    const Utf8String usr = cursor.canonical().unifiedSymbolResolution();
+
+    // ready is shared for all data in vector
+    std::atomic<bool> ready {false};
+    std::vector<FollowSymbolData> dataVector(
+                dependentFiles.size(),
+                FollowSymbolData(usr, tokenSpelling,
+                                 cursor.isFunctionLike() || cursor.isConstructorOrDestructor(),
+                                 ready));
+
+    std::vector<std::future<void>> indexFutures;
+
+    for (int i = 0; i < dependentFiles.size(); ++i) {
+        if (i > 0 && ready)
+            break;
+        indexFutures.emplace_back(std::async([&, i]() {
+            TIME_SCOPE_DURATION("Dependent file " + dependentFiles.at(i) + " indexer runner");
+
+            const CXIndexAction indexAction = clang_IndexAction_create(index);
+            IndexerCallbacks callbacks = createIndexerCallbacks();
+            clang_indexSourceFile(indexAction,
+                                  &dataVector[i],
+                                  &callbacks,
+                                  sizeof(callbacks),
+                                  CXIndexOpt_SkipParsedBodiesInSession
+                                      | CXIndexOpt_SuppressRedundantRefs
+                                      | CXIndexOpt_SuppressWarnings,
+                                  dependentFiles.at(i).constData(),
+                                  currentArgs.data(),
+                                  argsCount,
+                                  nullptr,
+                                  0,
+                                  nullptr,
+                                  CXTranslationUnit_SkipFunctionBodies
+                                      | CXTranslationUnit_KeepGoing);
+            clang_IndexAction_dispose(indexAction);
+        }));
+    }
+
+    for (const std::future<void> &future: indexFutures)
+        future.wait();
+
+    FollowSymbolResult result;
+    for (const FollowSymbolData &data: dataVector) {
+        if (!data.result().start().filePath().isEmpty()) {
+            result.range = data.result();
+            break;
+        }
+    }
+    return result;
+}
+
+FollowSymbolResult FollowSymbol::followSymbol(CXIndex index,
+                                              const Cursor &fullCursor,
+                                              uint line,
+                                              uint column,
+                                              const QVector<Utf8String> &dependentFiles,
+                                              const CommandLineArguments &currentArgs)
+{
+    FollowSymbolResult result;
+    Tokens tokens(fullCursor);
+    if (!tokens.tokenCount) {
+        result.failedToFollow = true;
+        return result;
+    }
+
+    const CXTranslationUnit tu = fullCursor.cxTranslationUnit();
+
+    QVector<CXCursor> cursors(static_cast<int>(tokens.tokenCount));
+    clang_annotateTokens(tu, tokens.data, tokens.tokenCount, cursors.data());
+    int tokenIndex = getTokenIndex(tu, tokens, line, column);
+    QTC_ASSERT(tokenIndex >= 0, return result);
+
+    const Utf8String tokenSpelling = ClangString(clang_getTokenSpelling(tu, tokens.data[tokenIndex]));
+    if (tokenSpelling.isEmpty())
+        return result;
+
+    Cursor cursor{cursors[tokenIndex]};
+    if (cursor.kind() == CXCursor_InclusionDirective) {
+        CXFile file = clang_getIncludedFile(cursors[tokenIndex]);
+        const ClangString filename(clang_getFileName(file));
+        const SourceLocation loc(tu, filename, 1, 1);
+        result.range = SourceRange(loc, loc);
+        return result;
+    }
+
+    if (cursor.isDefinition()) {
+        // For definitions we can always find a declaration in current TU
+        result.range = extractMatchingTokenRange(cursor.canonical(), tokenSpelling);
+        return result;
+    }
+
+    if (!cursor.isDeclaration()) {
+        // This is the symbol usage
+        // We want to return definition or at least declaration of this symbol
+        const Cursor referencedCursor = cursor.referenced();
+        if (referencedCursor.isNull() || referencedCursor == cursor)
+            return result;
+        result.range = extractMatchingTokenRange(referencedCursor, tokenSpelling);
+
+        // We've already found what we need
+        if (referencedCursor.isDefinition())
+            return result;
+        cursor = referencedCursor;
+    }
+
+    const Cursor definitionCursor = cursor.definition();
+    if (!definitionCursor.isNull() && definitionCursor != cursor) {
+        // If we are able to find a definition in current TU
+        result.range = extractMatchingTokenRange(definitionCursor, tokenSpelling);
+        return result;
+    }
+
+    // Search for the definition in the dependent files
+    FollowSymbolResult dependentFilesResult = followSymbolInDependentFiles(index,
+                                                                           cursor,
+                                                                           tokenSpelling,
+                                                                           dependentFiles,
+                                                                           currentArgs);
+    return dependentFilesResult.range.start().filePath().isEmpty() ?
+                result : dependentFilesResult;
+}
+
+} // namespace ClangBackEnd

src/tools/clangbackend/ipcsource/clangfollowsymbol.h

@@ -0,0 +1,51 @@
+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <QVector>
+
+#include <clang-c/Index.h>
+
+class Utf8String;
+
+namespace ClangBackEnd {
+
+class Cursor;
+class FollowSymbolResult;
+class CommandLineArguments;
+
+class FollowSymbol
+{
+public:
+    static FollowSymbolResult followSymbol(CXIndex index,
+                                           const Cursor &fullCursor,
+                                           uint line,
+                                           uint column,
+                                           const QVector<Utf8String> &dependentFiles,
+                                           const CommandLineArguments &currentArgs);
+};
+
+} // namespace ClangBackEnd

src/tools/clangbackend/ipcsource/clangfollowsymboljob.cpp

@@ -37,11 +37,11 @@ static FollowSymbolJob::AsyncResult runAsyncHelperFollow(const TranslationUnit &
                                                          quint32 line,
                                                          quint32 column,
                                                          const QVector<Utf8String> &dependentFiles,
-                                                         bool resolveTarget)
+                                                         const CommandLineArguments &currentArgs)
 {
     TIME_SCOPE_DURATION("FollowSymbolJobRunner");
 
-    return FollowSymbolResult();
+    return translationUnit.followSymbol(line, column, dependentFiles, currentArgs);
 }
 
 IAsyncJob::AsyncPrepareResult FollowSymbolJob::prepareAsyncRun()
@@ -49,6 +49,8 @@ IAsyncJob::AsyncPrepareResult FollowSymbolJob::prepareAsyncRun()
     const JobRequest jobRequest = context().jobRequest;
     QTC_ASSERT(jobRequest.type == JobRequest::Type::FollowSymbol,
                return AsyncPrepareResult());
+    // Is too slow because of IPC timings, no implementation for now
+    QTC_ASSERT(jobRequest.resolveTarget, return AsyncPrepareResult());
 
     try {
         m_pinnedDocument = context().documentForJobRequest();
@@ -56,12 +58,18 @@ IAsyncJob::AsyncPrepareResult FollowSymbolJob::prepareAsyncRun()
 
         const TranslationUnit translationUnit
                 = m_pinnedDocument.translationUnit(jobRequest.preferredTranslationUnit);
+
+        const TranslationUnitUpdateInput updateInput = m_pinnedDocument.createUpdateInput();
+        const CommandLineArguments currentArgs(updateInput.filePath.constData(),
+                                               updateInput.projectArguments,
+                                               updateInput.fileArguments,
+                                               false);
+
         const quint32 line = jobRequest.line;
         const quint32 column = jobRequest.column;
         const QVector<Utf8String> &dependentFiles = jobRequest.dependentFiles;
-        const bool resolveTarget = jobRequest.resolveTarget;
+        setRunner([translationUnit, line, column, dependentFiles, currentArgs]() {
-        setRunner([translationUnit, line, column, dependentFiles, resolveTarget]() {
+            return runAsyncHelperFollow(translationUnit, line, column, dependentFiles, currentArgs);
-            return runAsyncHelperFollow(translationUnit, line, column, dependentFiles, resolveTarget);
         });
         return AsyncPrepareResult{translationUnit.id()};
 
@@ -77,8 +85,8 @@ void FollowSymbolJob::finalizeAsyncRun()
         const AsyncResult result = asyncResult();
 
         const FollowSymbolMessage message(m_pinnedFileContainer,
-                                          result.m_range,
+                                          result.range,
-                                          result.m_failedToFollow,
+                                          result.failedToFollow,
                                           context().jobRequest.ticketNumber);
         context().client->followSymbol(message);
     }

src/tools/clangbackend/ipcsource/clangfollowsymboljob.h

@@ -36,13 +36,13 @@ class FollowSymbolResult
 {
 public:
     FollowSymbolResult() = default;
-    FollowSymbolResult(SourceRangeContainer &range, bool failedToFollow = false)
+    FollowSymbolResult(const SourceRangeContainer &range, bool failedToFollow = false)
-        : m_range(range)
+        : range(range)
-        , m_failedToFollow(failedToFollow)
+        , failedToFollow(failedToFollow)
     {}
 
-    SourceRangeContainer m_range;
+    SourceRangeContainer range;
-    bool m_failedToFollow = false;
+    bool failedToFollow = false;
 };
 
 class FollowSymbolJob : public AsyncJob<FollowSymbolResult>

src/tools/clangbackend/ipcsource/clangtranslationunit.cpp

@@ -28,6 +28,8 @@
 #include "clangbackend_global.h"
 #include "clangreferencescollector.h"
 #include "clangtranslationunitupdater.h"
+#include "clangfollowsymbol.h"
+#include "clangfollowsymboljob.h"
 
 #include <codecompleter.h>
 #include <cursor.h>
@@ -38,6 +40,7 @@
 #include <skippedsourceranges.h>
 #include <sourcelocation.h>
 #include <sourcerange.h>
+#include <commandlinearguments.h>
 
 #include <utils/qtcassert.h>
 
@@ -236,4 +239,13 @@ void TranslationUnit::extractDiagnostics(DiagnosticContainer &firstHeaderErrorDi
     }
 }
 
+FollowSymbolResult TranslationUnit::followSymbol(uint line,
+                                                 uint column,
+                                                 const QVector<Utf8String> &dependentFiles,
+                                                 const CommandLineArguments &currentArgs) const
+{
+    return FollowSymbol::followSymbol(m_cxIndex, cursorAt(line, column), line, column,
+                                      dependentFiles, currentArgs);
+}
+
 } // namespace ClangBackEnd

src/tools/clangbackend/ipcsource/clangtranslationunit.h

@@ -27,12 +27,8 @@
 
 #include <clangbackendipc/codecompletion.h>
 
-#include <utf8string.h>
-
 #include <clang-c/Index.h>
 
-class Utf8String;
-
 namespace ClangBackEnd {
 
 class Cursor;
@@ -41,6 +37,7 @@ class DiagnosticSet;
 class HighlightingMarkContainer;
 class HighlightingMarks;
 class ReferencesResult;
+class FollowSymbolResult;
 class SkippedSourceRanges;
 class SourceLocation;
 class SourceRange;
@@ -48,6 +45,7 @@ class SourceRangeContainer;
 class TranslationUnitUpdateInput;
 class TranslationUnitUpdateResult;
 class UnsavedFiles;
+class CommandLineArguments;
 
 class TranslationUnit
 {
@@ -102,6 +100,10 @@ public:
     HighlightingMarks highlightingMarksInRange(const SourceRange &range) const;
 
     SkippedSourceRanges skippedSourceRanges() const;
+    FollowSymbolResult followSymbol(uint line,
+                                    uint column,
+                                    const QVector<Utf8String> &dependentFiles,
+                                    const CommandLineArguments &currentArgs) const;
 
 private:
     const Utf8String m_id;

src/tools/clangbackend/ipcsource/cursor.cpp

@@ -273,6 +273,11 @@ CXSourceRange Cursor::cxSourceRange() const
     return clang_getCursorExtent(cxCursor);
 }
 
+CXTranslationUnit Cursor::cxTranslationUnit() const
+{
+    return clang_Cursor_getTranslationUnit(cxCursor);
+}
+
 SourceRange Cursor::commentRange() const
 {
     return clang_Cursor_getCommentRange(cxCursor);

src/tools/clangbackend/ipcsource/cursor.h

@@ -86,6 +86,7 @@ public:
     CXSourceLocation cxSourceLocation() const;
     SourceRange sourceRange() const;
     CXSourceRange cxSourceRange() const;
+    CXTranslationUnit cxTranslationUnit() const;
     SourceRange commentRange() const;
     bool hasSameSourceLocationAs(const Cursor &other) const;