Clang: Extend clang query

It's a first step to introduce clang query.

Change-Id: I4d001a8883f56066765ce6bc561fa3f49611c0a4
Reviewed-by: Tim Jenssen <tim.jenssen@qt.io>

src/tools/clangrefactoringbackend/source/clangquery.cpp

@@ -26,6 +26,7 @@
 #include "clangquery.h"
 
 #include "sourcelocationsutils.h"
+#include "sourcerangeextractor.h"
 
 #include <sourcerangescontainer.h>
 
@@ -76,16 +77,7 @@ void ClangQuery::findLocations()
 
     std::vector<std::unique_ptr<clang::ASTUnit>> asts;
 
-    {
-        QTime timer;
-        timer.start();
-        tool.buildASTs(asts);
-
-        qWarning() << "ASTs are built: " << timer.elapsed();
-    }
-
-    std::vector<clang::SourceRange> sourceRanges;
-    sourceRanges.reserve(100);
+    tool.buildASTs(asts);
 
     std::for_each (std::make_move_iterator(asts.begin()),
                    std::make_move_iterator(asts.end()),
@@ -95,7 +87,7 @@ void ClangQuery::findLocations()
                                                               nullptr,
                                                               &diagnostics);
         parseDiagnostics(diagnostics);
-        matchLocation(optionalMatcher, std::move(ast), sourceRanges);
+        matchLocation(optionalMatcher, std::move(ast));
     });
 }
 
@@ -189,10 +181,28 @@ void ClangQuery::parseDiagnostics(const clang::ast_matchers::dynamic::Diagnostic
     }
 }
 
+namespace {
+std::vector<clang::SourceRange> generateSourceRangesFromMatches(const std::vector<BoundNodes> &matches)
+{
+    std::vector<clang::SourceRange> sourceRanges;
+    sourceRanges.reserve(matches.size());
+
+    for (const auto boundNodes : matches) {
+        for (const auto &mapEntry : boundNodes.getMap()) {
+            const auto sourceRange = mapEntry.second.getSourceRange();
+            if (sourceRange.isValid())
+                sourceRanges.push_back(sourceRange);
+        }
+
+    }
+
+    return sourceRanges;
+}
+}
+
 void ClangQuery::matchLocation(
         const llvm::Optional< clang::ast_matchers::internal::DynTypedMatcher> &optionalStartMatcher,
-        std::unique_ptr<clang::ASTUnit> ast,
-        std::vector<clang::SourceRange> &sourceRanges)
+        std::unique_ptr<clang::ASTUnit> ast)
 {
     if (optionalStartMatcher) {
         auto matcher = *optionalStartMatcher;
@@ -207,18 +217,13 @@ void ClangQuery::matchLocation(
 
         finder.matchAST(ast->getASTContext());
 
-        for (const auto &boundNodes : matches) {
-            for (const auto &mapEntry : boundNodes.getMap()) {
-                const auto sourceRange = mapEntry.second.getSourceRange();
-                if (sourceRange.isValid())
-                    sourceRanges.push_back(sourceRange);
-            }
+        auto sourceRanges = generateSourceRangesFromMatches(matches);
 
-        }
+        SourceRangeExtractor extractor(ast->getSourceManager(),
+                                       ast->getLangOpts(),
+                                       sourceRangesContainer);
+        extractor.addSourceRanges(sourceRanges);
 
-        appendSourceRangesToSourceRangesContainer(sourceRangesContainer,
-                                                  sourceRanges,
-                                                  ast->getSourceManager());
     }
 }
 

src/tools/clangrefactoringbackend/source/clangquery.h

@@ -61,8 +61,7 @@ public:
 private:
     void parseDiagnostics(const clang::ast_matchers::dynamic::Diagnostics &diagnostics);
     void matchLocation(const llvm::Optional< clang::ast_matchers::internal::DynTypedMatcher> &optionalStartMatcher,
-                       std::unique_ptr<clang::ASTUnit> ast,
-                       std::vector<clang::SourceRange> &sourceRanges);
+                       std::unique_ptr<clang::ASTUnit> ast);
 
 private:
     SourceRangesContainer sourceRangesContainer;

src/tools/clangrefactoringbackend/source/clangrefactoringbackend-source.pri

@@ -5,11 +5,12 @@ SOURCES += \
     $$PWD/symbolfinder.cpp \
     $$PWD/symbollocationfinderaction.cpp \
     $$PWD/refactoringserver.cpp \
-    $$PWD/sourcefilecallbacks.cpp \
     $$PWD/macropreprocessorcallbacks.cpp \
     $$PWD/findusrforcursoraction.cpp \
     $$PWD/clangquery.cpp \
-    $$PWD/clangtool.cpp
+    $$PWD/clangtool.cpp \
+    $$PWD/sourcerangeextractor.cpp \
+    $$PWD/locationsourcefilecallbacks.cpp
 
 HEADERS += \
     $$PWD/refactoringcompilationdatabase.h \
@@ -17,11 +18,12 @@ HEADERS += \
     $$PWD/symbolfinder.h \
     $$PWD/symbollocationfinderaction.h \
     $$PWD/refactoringserver.h \
-    $$PWD/sourcefilecallbacks.h \
     $$PWD/macropreprocessorcallbacks.h \
     $$PWD/sourcelocationsutils.h \
     $$PWD/findcursorusr.h \
     $$PWD/findusrforcursoraction.h \
     $$PWD/findlocationsofusrs.h \
     $$PWD/clangquery.h \
-    $$PWD/clangtool.h
+    $$PWD/clangtool.h \
+    $$PWD/sourcerangeextractor.h \
+    $$PWD/locationsourcefilecallbacks.h

src/tools/clangrefactoringbackend/source/clangtool.cpp

@@ -57,6 +57,24 @@ void ClangTool::addFile(std::string &&directory,
     sourceFilePaths.push_back(fileContent.filePath);
 }
 
+void ClangTool::addFiles(const Utils::SmallStringVector &filePaths,
+                         const Utils::SmallStringVector &arguments)
+{
+    for (const Utils::SmallString &filePath : filePaths) {
+        auto found = std::find(filePath.rbegin(), filePath.rend(), '/');
+
+        auto fileNameBegin = found.base();
+
+        std::vector<std::string> commandLine(arguments.begin(), arguments.end());
+        commandLine.push_back(filePath);
+
+        addFile({filePath.begin(), std::prev(fileNameBegin)},
+                {fileNameBegin, filePath.end()},
+                {},
+                std::move(commandLine));
+    }
+}
+
 clang::tooling::ClangTool ClangTool::createTool() const
 {
     clang::tooling::ClangTool tool(compilationDatabase, sourceFilePaths);

src/tools/clangrefactoringbackend/source/clangtool.h

@@ -74,6 +74,8 @@ public:
                  std::string &&fileName,
                  std::string &&content,
                  std::vector<std::string> &&commandLine);
+    void addFiles(const Utils::SmallStringVector &filePaths,
+                 const Utils::SmallStringVector &arguments);
 
     clang::tooling::ClangTool createTool() const;
 

src/tools/clangrefactoringbackend/source/collectincludesaction.h

@@ -0,0 +1,88 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <collectincludespreprocessorcallbacks.h>
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#endif
+
+#include <clang/Frontend/FrontendActions.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Lex/Preprocessor.h>
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
+namespace ClangBackEnd {
+
+class CollectIncludesAction final : public clang::PreprocessOnlyAction
+{
+public:
+    CollectIncludesAction(Utils::SmallStringVector &includes,
+                          const std::vector<uint> &excludedIncludeUID,
+                          std::vector<uint> &alreadyIncludedFileUIDs)
+        : includes(includes),
+          excludedIncludeUID(excludedIncludeUID),
+          alreadyIncludedFileUIDs(alreadyIncludedFileUIDs)
+    {
+    }
+
+    bool BeginSourceFileAction(clang::CompilerInstance &compilerInstance,
+                               llvm::StringRef filename) override
+    {
+      if (clang::PreprocessOnlyAction::BeginSourceFileAction(compilerInstance, filename)) {
+          auto &preprocessor = compilerInstance.getPreprocessor();
+          auto &headerSearch = preprocessor.getHeaderSearchInfo();
+
+          auto macroPreprocessorCallbacks = new CollectIncludesPreprocessorCallbacks(headerSearch,
+                                                                                     includes,
+                                                                                     excludedIncludeUID,
+                                                                                     alreadyIncludedFileUIDs);
+
+          preprocessor.addPPCallbacks(std::unique_ptr<clang::PPCallbacks>(macroPreprocessorCallbacks));
+
+          return true;
+      }
+
+      return false;
+    }
+
+    void EndSourceFileAction() override
+    {
+      clang::PreprocessOnlyAction::EndSourceFileAction();
+    }
+
+private:
+    Utils::SmallStringVector &includes;
+    const std::vector<uint> &excludedIncludeUID;
+    std::vector<uint> &alreadyIncludedFileUIDs;
+};
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/collectincludespreprocessorcallbacks.h

@@ -0,0 +1,120 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#endif
+
+#include <clang/Basic/SourceManager.h>
+#include <clang/Lex/MacroInfo.h>
+#include <clang/Lex/HeaderSearch.h>
+#include <clang/Lex/PPCallbacks.h>
+#include <clang/Lex/Preprocessor.h>
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
+#include <utils/smallstringvector.h>
+
+#include <algorithm>
+
+#include <QDebug>
+
+namespace ClangBackEnd {
+
+class CollectIncludesPreprocessorCallbacks final : public clang::PPCallbacks
+{
+public:
+    CollectIncludesPreprocessorCallbacks(clang::HeaderSearch &headerSearch,
+                                         Utils::SmallStringVector &includes,
+                                         const std::vector<uint> &excludedIncludeUID,
+                                         std::vector<uint>  &alreadyIncludedFileUIDs)
+        : headerSearch(headerSearch),
+          includes(includes),
+          excludedIncludeUID(excludedIncludeUID),
+          alreadyIncludedFileUIDs(alreadyIncludedFileUIDs)
+    {}
+
+    void InclusionDirective(clang::SourceLocation /*hashLocation*/,
+                            const clang::Token &/*includeToken*/,
+                            llvm::StringRef fileName,
+                            bool /*isAngled*/,
+                            clang::CharSourceRange /*fileNameRange*/,
+                            const clang::FileEntry *file,
+                            llvm::StringRef /*searchPath*/,
+                            llvm::StringRef /*relativePath*/,
+                            const clang::Module */*imported*/) override
+    {
+        auto fileUID = file->getUID();
+
+        flagIncludeAlreadyRead(file);
+
+        if (isNotInExcludedIncludeUID(fileUID)) {
+            auto notAlreadyIncluded = isNotAlreadyIncluded(fileUID);
+            if (notAlreadyIncluded.first) {
+                alreadyIncludedFileUIDs.insert(notAlreadyIncluded.second, file->getUID());
+                includes.emplace_back(fileName.data(), fileName.size());
+            }
+        }
+    }
+
+    bool isNotInExcludedIncludeUID(uint uid) const
+    {
+        return !std::binary_search(excludedIncludeUID.begin(),
+                                   excludedIncludeUID.end(),
+                                   uid);
+    }
+
+
+    std::pair<bool, std::vector<uint>::iterator> isNotAlreadyIncluded(uint uid)
+    {
+        auto range = std::equal_range(alreadyIncludedFileUIDs.begin(),
+                                      alreadyIncludedFileUIDs.end(),
+                                      uid);
+
+        return {range.first == range.second, range.first};
+    }
+
+    void flagIncludeAlreadyRead(const clang::FileEntry *file)
+    {
+        auto &headerFileInfo = headerSearch.getFileInfo(file);
+
+        headerFileInfo.isImport = true;
+        ++headerFileInfo.NumIncludes;
+
+    }
+
+private:
+    clang::HeaderSearch &headerSearch;
+    std::vector<Utils::SmallString>  &includes;
+    const std::vector<uint> &excludedIncludeUID;
+    std::vector<uint>  &alreadyIncludedFileUIDs;
+};
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/collectincludestoolaction.h

@@ -0,0 +1,54 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "collectincludesaction.h"
+
+#include <clang/Tooling/Tooling.h>
+
+namespace ClangBackEnd {
+
+class CollectIncludesToolAction final : public clang::tooling::FrontendActionFactory
+{
+public:
+    CollectIncludesToolAction(Utils::SmallStringVector &includes,
+                              const std::vector<uint> &excludedIncludeUIDs)
+        : includes(includes),
+          excludedIncludeUIDs(excludedIncludeUIDs)
+    {}
+
+    clang::FrontendAction *create()
+    {
+        return new CollectIncludesAction(includes, excludedIncludeUIDs, alreadyIncludedFileUIDs);
+    }
+
+private:
+    Utils::SmallStringVector &includes;
+    const std::vector<uint> &excludedIncludeUIDs;
+    std::vector<uint> alreadyIncludedFileUIDs;
+};
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/includecollector.cpp

@@ -0,0 +1,75 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "includecollector.h"
+
+#include "collectincludestoolaction.h"
+
+namespace ClangBackEnd {
+
+void IncludeCollector::collectIncludes()
+{
+    clang::tooling::ClangTool tool = createTool();
+
+    auto excludedIncludeFileUIDs = generateExcludedIncludeFileUIDs(tool.getFiles());
+
+    auto action = std::unique_ptr<CollectIncludesToolAction>(
+                new CollectIncludesToolAction(includes, excludedIncludeFileUIDs));
+
+    tool.run(action.get());
+}
+
+void IncludeCollector::setExcludedIncludes(Utils::SmallStringVector &&excludedIncludes)
+{
+    this->excludedIncludes = std::move(excludedIncludes);
+}
+
+Utils::SmallStringVector IncludeCollector::takeIncludes()
+{
+    std::sort(includes.begin(), includes.end());
+
+    return std::move(includes);
+}
+
+std::vector<uint> IncludeCollector::generateExcludedIncludeFileUIDs(clang::FileManager &fileManager) const
+{
+    std::vector<uint> fileUIDs;
+    fileUIDs.reserve(excludedIncludes.size());
+
+    auto generateUID = [&] (const Utils::SmallString &filePath) {
+        return fileManager.getFile({filePath.data(), filePath.size()})->getUID();
+    };
+
+    std::transform(excludedIncludes.begin(),
+                   excludedIncludes.end(),
+                   std::back_inserter(fileUIDs),
+                   generateUID);
+
+    std::sort(fileUIDs.begin(), fileUIDs.end());
+
+    return fileUIDs;
+}
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/includecollector.h

@@ -0,0 +1,49 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "clangtool.h"
+
+namespace ClangBackEnd {
+
+class IncludeCollector : public ClangTool
+{
+public:
+    void collectIncludes();
+
+    void setExcludedIncludes(Utils::SmallStringVector &&excludedIncludes);
+
+    Utils::SmallStringVector takeIncludes();
+
+private:
+    std::vector<uint> generateExcludedIncludeFileUIDs(clang::FileManager &fileManager) const;
+
+private:
+    Utils::SmallStringVector excludedIncludes;
+    Utils::SmallStringVector includes;
+};
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/locationsourcefilecallbacks.cpp

@@ -23,7 +23,7 @@
 **
 ****************************************************************************/
 
-#include "sourcefilecallbacks.h"
+#include "locationsourcefilecallbacks.h"
 
 #include "macropreprocessorcallbacks.h"
 
@@ -43,13 +43,13 @@
 
 namespace ClangBackEnd {
 
-SourceFileCallbacks::SourceFileCallbacks(uint line, uint column)
+LocationSourceFileCallbacks::LocationSourceFileCallbacks(uint line, uint column)
     : line(line),
       column(column)
 {
 }
 
-bool SourceFileCallbacks::handleBeginSource(clang::CompilerInstance &compilerInstance, llvm::StringRef /*fileName*/)
+bool LocationSourceFileCallbacks::handleBeginSource(clang::CompilerInstance &compilerInstance, llvm::StringRef /*fileName*/)
 {
     auto &preprocessor = compilerInstance.getPreprocessor();
 
@@ -64,17 +64,17 @@ bool SourceFileCallbacks::handleBeginSource(clang::CompilerInstance &compilerIns
     return true;
 }
 
-SourceLocationsContainer SourceFileCallbacks::takeSourceLocations()
+SourceLocationsContainer LocationSourceFileCallbacks::takeSourceLocations()
 {
     return std::move(sourceLocationsContainer);
 }
 
-Utils::SmallString SourceFileCallbacks::takeSymbolName()
+Utils::SmallString LocationSourceFileCallbacks::takeSymbolName()
 {
     return std::move(symbolName);
 }
 
-bool SourceFileCallbacks::hasSourceLocations() const
+bool LocationSourceFileCallbacks::hasSourceLocations() const
 {
     return sourceLocationsContainer.hasContent();
 }

src/tools/clangrefactoringbackend/source/locationsourcefilecallbacks.h

@@ -51,10 +51,10 @@ namespace ClangBackEnd {
 class MacroPreprocessorCallbacks;
 class SourceLocationsContainer;
 
-class SourceFileCallbacks : public clang::tooling::SourceFileCallbacks
+class LocationSourceFileCallbacks : public clang::tooling::SourceFileCallbacks
 {
 public:
-    SourceFileCallbacks(uint line, uint column);
+    LocationSourceFileCallbacks(uint line, uint column);
 
     bool handleBeginSource(clang::CompilerInstance &compilerInstance,
                            llvm::StringRef fileName) override;

src/tools/clangrefactoringbackend/source/refactoringserver.cpp

@@ -39,11 +39,13 @@
 #include <algorithm>
 #include <chrono>
 #include <future>
+#include <atomic>
 
 namespace ClangBackEnd {
 
 RefactoringServer::RefactoringServer()
 {
+    pollEventLoop = [] () { QCoreApplication::processEvents(); };
 }
 
 void RefactoringServer::end()
@@ -67,19 +69,43 @@ void RefactoringServer::requestSourceLocationsForRenamingMessage(RequestSourceLo
                                                  message.textDocumentRevision()});
 }
 
+void RefactoringServer::requestSourceRangesAndDiagnosticsForQueryMessage(
+        RequestSourceRangesAndDiagnosticsForQueryMessage &&message)
+{
+    gatherSourceRangesAndDiagnosticsForQueryMessage(message.takeFileContainers(), message.takeQuery());
+}
+
+void RefactoringServer::cancel()
+{
+    cancelWork = true;
+}
+
+bool RefactoringServer::isCancelingJobs() const
+{
+    return cancelWork;
+}
+
+void RefactoringServer::supersedePollEventLoop(std::function<void ()> &&pollEventLoop)
+{
+    this->pollEventLoop = std::move(pollEventLoop);
+}
+
 namespace {
 
 SourceRangesAndDiagnosticsForQueryMessage createSourceRangesAndDiagnosticsForQueryMessage(
         V2::FileContainer &&fileContainer,
-        Utils::SmallString &&query) {
+        Utils::SmallString &&query,
+        const std::atomic_bool &cancelWork) {
     ClangQuery clangQuery(std::move(query));
 
-    clangQuery.addFile(fileContainer.filePath().directory(),
-                       fileContainer.filePath().name(),
-                       fileContainer.takeUnsavedFileContent(),
-                       fileContainer.takeCommandLineArguments());
+    if (!cancelWork) {
+        clangQuery.addFile(fileContainer.filePath().directory(),
+                           fileContainer.filePath().name(),
+                           fileContainer.takeUnsavedFileContent(),
+                           fileContainer.takeCommandLineArguments());
 
-    clangQuery.findLocations();
+        clangQuery.findLocations();
+    }
 
     return {clangQuery.takeSourceRanges(), clangQuery.takeDiagnosticContainers()};
 }
@@ -87,12 +113,6 @@ SourceRangesAndDiagnosticsForQueryMessage createSourceRangesAndDiagnosticsForQue
 
 }
 
-void RefactoringServer::requestSourceRangesAndDiagnosticsForQueryMessage(
-        RequestSourceRangesAndDiagnosticsForQueryMessage &&message)
-{
-    gatherSourceRangesAndDiagnosticsForQueryMessage(message.takeFileContainers(), message.takeQuery());
-}
-
 void RefactoringServer::gatherSourceRangesAndDiagnosticsForQueryMessage(
         std::vector<V2::FileContainer> &&fileContainers,
         Utils::SmallString &&query)
@@ -108,7 +128,7 @@ void RefactoringServer::gatherSourceRangesAndDiagnosticsForQueryMessage(
             Future &&future = std::async(std::launch::async,
                                          createSourceRangesAndDiagnosticsForQueryMessage,
                                          std::move(fileContainers.back()),
-                                         query.clone());
+                                         query.clone(), std::ref(cancelWork));
             fileContainers.pop_back();
 
             futures.emplace_back(std::move(future));
@@ -122,6 +142,8 @@ void RefactoringServer::gatherSourceRangesAndDiagnosticsForQueryMessage(
 std::size_t RefactoringServer::waitForNewSourceRangesAndDiagnosticsForQueryMessage(std::vector<Future> &futures)
 {
     while (true) {
+        pollEventLoop();
+
         std::vector<Future> readyFutures;
         readyFutures.reserve(futures.size());
 

src/tools/clangrefactoringbackend/source/refactoringserver.h

@@ -47,11 +47,21 @@ public:
     void end() override;
     void requestSourceLocationsForRenamingMessage(RequestSourceLocationsForRenamingMessage &&message) override;
     void requestSourceRangesAndDiagnosticsForQueryMessage(RequestSourceRangesAndDiagnosticsForQueryMessage &&message) override;
+    void cancel() override;
+
+    bool isCancelingJobs() const;
+
+    void supersedePollEventLoop(std::function<void()> &&pollEventLoop);
 
 private:
     void gatherSourceRangesAndDiagnosticsForQueryMessage(std::vector<V2::FileContainer> &&fileContainers,
                                                          Utils::SmallString &&query);
     std::size_t waitForNewSourceRangesAndDiagnosticsForQueryMessage(std::vector<Future> &futures);
+
+private:
+    std::function<void()> pollEventLoop;
+    std::atomic_bool cancelWork{false};
+
 };
 
 } // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/sourcelocationsutils.h

@@ -34,6 +34,7 @@
 #endif
 
 #include <clang/Basic/SourceManager.h>
+#include <clang/Lex/Lexer.h>
 #include <llvm/Support/FileSystem.h>
 #include <llvm/Support/FileUtilities.h>
 
@@ -41,6 +42,7 @@
 #pragma GCC diagnostic pop
 #endif
 
+#include <iterator>
 #include <cctype>
 
 namespace ClangBackEnd {
@@ -68,75 +70,6 @@ Utils::SmallString fromNativePath(Container container)
     return path;
 }
 
-inline Utils::SmallString getSourceText(const clang::FullSourceLoc &startFullSourceLocation,
-                                        uint startOffset,
-                                        uint endOffset)
-{
-    auto startBuffer = startFullSourceLocation.getBufferData();
-    const auto bufferSize = endOffset - startOffset;
-
-    return Utils::SmallString(startBuffer.data() + startOffset, bufferSize + 1);
-}
-
-inline void makePrintable(Utils::SmallString &text)
-{
-    text.replace("\n", " ");
-    text.replace("\t", " ");
-
-    auto end = std::unique(text.begin(), text.end(), [](char l, char r){
-        return std::isspace(l) && std::isspace(r) && l == r;
-    });
-    text.resize(std::distance(text.begin(), end));
-}
-
-inline void appendSourceRangeToSourceRangesContainer(
-        const clang::SourceRange &sourceRange,
-        ClangBackEnd::SourceRangesContainer &sourceRangesContainer,
-        const clang::SourceManager &sourceManager)
-{
-    clang::FullSourceLoc startFullSourceLocation(sourceRange.getBegin(), sourceManager);
-    clang::FullSourceLoc endFullSourceLocation(sourceRange.getEnd(), sourceManager);
-    if (startFullSourceLocation.isFileID() && endFullSourceLocation.isFileID()) {
-        const auto startDecomposedLoction = startFullSourceLocation.getDecomposedLoc();
-        const auto endDecomposedLoction = endFullSourceLocation.getDecomposedLoc();
-        const auto fileId = startDecomposedLoction.first;
-        const auto startOffset = startDecomposedLoction.second;
-        const auto endOffset = endDecomposedLoction.second;
-        const auto fileEntry = sourceManager.getFileEntryForID(fileId);
-        auto filePath = absolutePath(fileEntry->getName());
-        const auto fileName = llvm::sys::path::filename(filePath);
-        llvm::sys::path::remove_filename(filePath);
-        Utils::SmallString content = getSourceText(startFullSourceLocation,
-                                                   startOffset,
-                                                   endOffset);
-        makePrintable(content);
-
-        sourceRangesContainer.insertFilePath(fileId.getHashValue(),
-                                             fromNativePath(filePath),
-                                             fromNativePath(fileName));
-        sourceRangesContainer.insertSourceRange(fileId.getHashValue(),
-                                                startFullSourceLocation.getSpellingLineNumber(),
-                                                startFullSourceLocation.getSpellingColumnNumber(),
-                                                startOffset,
-                                                endFullSourceLocation.getSpellingLineNumber(),
-                                                endFullSourceLocation.getSpellingColumnNumber(),
-                                                endOffset,
-                                                std::move(content));
-    }
-}
-
-inline
-void appendSourceRangesToSourceRangesContainer(
-        ClangBackEnd::SourceRangesContainer &sourceRangesContainer,
-        const std::vector<clang::SourceRange> &sourceRanges,
-        const clang::SourceManager &sourceManager)
-{
-    sourceRangesContainer.reserve(sourceRanges.size());
-
-    for (const auto &sourceRange : sourceRanges)
-        appendSourceRangeToSourceRangesContainer(sourceRange, sourceRangesContainer, sourceManager);
-}
-
 inline
 void appendSourceLocationsToSourceLocationsContainer(
         ClangBackEnd::SourceLocationsContainer &sourceLocationsContainer,

src/tools/clangrefactoringbackend/source/sourcerangeextractor.cpp

@@ -0,0 +1,178 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "sourcerangeextractor.h"
+
+#include "sourcelocationsutils.h"
+
+#include <sourcerangescontainer.h>
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#endif
+
+#include <clang/Basic/SourceManager.h>
+#include <clang/Lex/Lexer.h>
+#include <llvm/Support/FileSystem.h>
+#include <llvm/Support/FileUtilities.h>
+#include <llvm/ADT/SmallVector.h>
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
+namespace ClangBackEnd {
+
+SourceRangeExtractor::SourceRangeExtractor(const clang::SourceManager &sourceManager,
+                                           const clang::LangOptions &languageOptions,
+                                           SourceRangesContainer &sourceRangesContainer)
+    : sourceManager(sourceManager),
+      languageOptions(languageOptions),
+      sourceRangesContainer(sourceRangesContainer)
+{
+}
+
+const char *SourceRangeExtractor::findStartOfLineInBuffer(llvm::StringRef buffer, uint startOffset)
+{
+    auto beginText = buffer.begin() + startOffset;
+    auto reverseEnd = std::make_reverse_iterator(buffer.begin());
+
+    auto found = std::find_if(std::make_reverse_iterator(beginText),
+                              reverseEnd,
+                              [] (const char character) {
+        return character == '\n' || character == '\r';
+    });
+
+    if (found != reverseEnd)
+        return found.base();
+
+    return buffer.begin();
+}
+
+const char *SourceRangeExtractor::findEndOfLineInBuffer(llvm::StringRef buffer, uint endOffset)
+{
+    auto beginText = buffer.begin() + endOffset;
+
+    auto found = std::find_if(beginText,
+                              buffer.end(),
+                              [] (const char character) {
+        return character == '\n' || character == '\r';
+    });
+
+    if (found != buffer.end())
+        return found;
+
+    return buffer.end();
+}
+
+Utils::SmallString SourceRangeExtractor::getExpandedText(llvm::StringRef buffer,
+                                                    uint startOffset,
+                                                    uint endOffset)
+{
+    auto startBuffer = findStartOfLineInBuffer(buffer, startOffset);
+    auto endBuffer = findEndOfLineInBuffer(buffer, endOffset);
+
+    return Utils::SmallString(startBuffer, endBuffer);
+}
+
+const clang::SourceRange SourceRangeExtractor::extendSourceRangeToLastTokenEnd(const clang::SourceRange sourceRange)
+{
+    auto endLocation = sourceRange.getEnd();
+    uint length = clang::Lexer::MeasureTokenLength(sourceManager.getSpellingLoc(endLocation),
+                                                   sourceManager,
+                                                   languageOptions);
+    endLocation = endLocation.getLocWithOffset(length);
+
+    return {sourceRange.getBegin(), endLocation};
+}
+
+void SourceRangeExtractor::insertSourceRange(uint fileHash,
+                                             Utils::SmallString &&directoryPath,
+                                             Utils::SmallString &&fileName,
+                                             const clang::FullSourceLoc &startLocation,
+                                             uint startOffset,
+                                             const clang::FullSourceLoc &endLocation,
+                                             uint endOffset,
+                                             Utils::SmallString &&lineSnippet)
+{
+    sourceRangesContainer.insertFilePath(fileHash,
+                                         std::move(directoryPath),
+                                         std::move(fileName));
+    sourceRangesContainer.insertSourceRange(fileHash,
+                                            startLocation.getSpellingLineNumber(),
+                                            startLocation.getSpellingColumnNumber(),
+                                            startOffset,
+                                            endLocation.getSpellingLineNumber(),
+                                            endLocation.getSpellingColumnNumber(),
+                                            endOffset,
+                                            std::move(lineSnippet));
+}
+
+void SourceRangeExtractor::addSourceRange(const clang::SourceRange &sourceRange)
+{
+    auto extendedSourceRange = extendSourceRangeToLastTokenEnd(sourceRange);
+
+    clang::FullSourceLoc startSourceLocation(extendedSourceRange.getBegin(), sourceManager);
+    clang::FullSourceLoc endSourceLocation(extendedSourceRange.getEnd(), sourceManager);
+    if (startSourceLocation.isFileID() && endSourceLocation.isFileID()) {
+        const auto startDecomposedLoction = startSourceLocation.getDecomposedLoc();
+        const auto endDecomposedLoction = endSourceLocation.getDecomposedLoc();
+        const auto fileId = startDecomposedLoction.first;
+        const auto startOffset = startDecomposedLoction.second;
+        const auto endOffset = endDecomposedLoction.second;
+        const auto fileEntry = sourceManager.getFileEntryForID(fileId);
+        auto filePath = absolutePath(fileEntry->getName());
+        const auto fileName = llvm::sys::path::filename(filePath);
+        llvm::sys::path::remove_filename(filePath);
+        Utils::SmallString lineSnippet = getExpandedText(startSourceLocation.getBufferData(),
+                                                         startOffset,
+                                                         endOffset);
+        insertSourceRange(fileId.getHashValue(),
+                          fromNativePath(filePath),
+                          {fileName.data(), fileName.size()},
+                          startSourceLocation,
+                          startOffset,
+                          endSourceLocation,
+                          endOffset,
+                          std::move(lineSnippet));
+
+    }
+}
+
+void SourceRangeExtractor::addSourceRanges(const std::vector<clang::SourceRange> &sourceRanges)
+{
+    sourceRangesContainer.reserve(sourceRanges.size() + sourceRangeWithTextContainers().size());
+
+    for (const clang::SourceRange &sourceRange : sourceRanges)
+        addSourceRange(sourceRange);
+}
+
+const std::vector<SourceRangeWithTextContainer> &SourceRangeExtractor::sourceRangeWithTextContainers() const
+{
+    return sourceRangesContainer.sourceRangeWithTextContainers();
+}
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/sourcerangeextractor.h

@@ -0,0 +1,88 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <vector>
+
+using uint = unsigned int;
+
+namespace Utils {
+template <uint Size>
+class BasicSmallString;
+using SmallString = BasicSmallString<31>;
+}
+
+namespace llvm {
+class StringRef;
+}
+
+namespace clang {
+class SourceManager;
+class LangOptions;
+class SourceRange;
+class FullSourceLoc;
+}
+
+namespace ClangBackEnd {
+
+class SourceRangesContainer;
+class SourceRangeWithTextContainer;
+
+class SourceRangeExtractor
+{
+public:
+    SourceRangeExtractor(const clang::SourceManager &sourceManager,
+                         const clang::LangOptions &languageOptions,
+                         SourceRangesContainer &sourceRangesContainer);
+
+    void addSourceRange(const clang::SourceRange &sourceRange);
+    void addSourceRanges(const std::vector<clang::SourceRange> &sourceRanges);
+
+    const std::vector<SourceRangeWithTextContainer> &sourceRangeWithTextContainers() const;
+
+    static const char *findStartOfLineInBuffer(const llvm::StringRef buffer, uint startOffset);
+    static const char *findEndOfLineInBuffer(llvm::StringRef buffer, uint endOffset);
+    static Utils::SmallString getExpandedText(llvm::StringRef buffer, uint startOffset, uint endOffset);
+
+    const clang::SourceRange extendSourceRangeToLastTokenEnd(const clang::SourceRange sourceRange);
+
+private:
+    void insertSourceRange(uint fileHash,
+                           Utils::SmallString &&directoryPath,
+                           Utils::SmallString &&fileName,
+                           const clang::FullSourceLoc &startLocation,
+                           uint startOffset,
+                           const clang::FullSourceLoc &endLocation,
+                           uint endOffset,
+                           Utils::SmallString &&lineSnippet);
+
+private:
+    const clang::SourceManager &sourceManager;
+    const clang::LangOptions &languageOptions;
+    SourceRangesContainer &sourceRangesContainer;
+};
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/symbolfinder.cpp

@@ -25,7 +25,7 @@
 
 #include "symbolfinder.h"
 
-#include "sourcefilecallbacks.h"
+#include "locationsourcefilecallbacks.h"
 #include "symbollocationfinderaction.h"
 
 namespace ClangBackEnd {

src/tools/clangrefactoringbackend/source/symbolfinder.h

@@ -28,7 +28,7 @@
 #include "clangtool.h"
 #include "findusrforcursoraction.h"
 #include "symbollocationfinderaction.h"
-#include "sourcefilecallbacks.h"
+#include "locationsourcefilecallbacks.h"
 
 #include <sourcelocationscontainer.h>
 
@@ -61,7 +61,7 @@ private:
     Utils::SmallString symbolName;
     USRFindingAction usrFindingAction;
     SymbolLocationFinderAction symbolLocationFinderAction;
-    SourceFileCallbacks sourceFileCallbacks;
+    LocationSourceFileCallbacks sourceFileCallbacks;
 
     ClangBackEnd::SourceLocationsContainer sourceLocations_;
 };