feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

Clang: Introduce parallel indexing

Browse Source 
Change-Id: I522cb18e6d24b7dbed5d5dfa3a732e5b3b5113bb
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>
This commit is contained in:
Marco Bubke
2018-08-28 12:08:37 +02:00
parent 86cd29b13c
commit a86867eb8a
32 changed files with 828 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/tools/clangrefactoringbackend/source/clangrefactoringbackend-source.pri
View File

@@ -27,7 +27,10 @@ HEADERS += \
$$PWD/symbolindexertaskqueue.h \
$$PWD/symbolindexertaskscheduler.h \
$$PWD/symbolscollectormanagerinterface.h \
$$PWD/symbolindexertaskqueueinterface.h
$$PWD/symbolindexertaskqueueinterface.h \
$$PWD/symbolindexertaskschedulerinterface.h \
$$PWD/symbolscollectormanager.h \
$$PWD/symbolindexertask.h
!isEmpty(LIBTOOLING_LIBS) {
SOURCES += \
@@ -75,4 +78,5 @@ SOURCES += \
$$PWD/filestatuscache.cpp \
$$PWD/projectpartqueue.cpp \
$$PWD/symbolindexertaskqueue.cpp \
$$PWD/symbolindexertaskscheduler.cpp
$$PWD/symbolindexertaskscheduler.cpp \
$$PWD/symbolscollectormanager.cpp

115
src/tools/clangrefactoringbackend/source/symbolindexer.cpp
View File

@@ -25,15 +25,18 @@
#include "symbolindexer.h"
#include <symbolscollector.h>
#include <symbolindexertaskqueue.h>
namespace ClangBackEnd {
SymbolIndexer::SymbolIndexer(SymbolsCollectorInterface &symbolsCollector,
SymbolIndexer::SymbolIndexer(SymbolIndexerTaskQueueInterface &symbolIndexerTaskQueue,
SymbolStorageInterface &symbolStorage,
ClangPathWatcherInterface &pathWatcher,
FilePathCachingInterface &filePathCache,
FileStatusCache &fileStatusCache,
Sqlite::TransactionInterface &transactionInterface)
: m_symbolsCollector(symbolsCollector),
: m_symbolIndexerTaskQueue(symbolIndexerTaskQueue),
m_symbolStorage(symbolStorage),
m_pathWatcher(pathWatcher),
m_filePathCache(filePathCache),
@@ -52,41 +55,60 @@ void SymbolIndexer::updateProjectParts(V2::ProjectPartContainers &&projectParts,
void SymbolIndexer::updateProjectPart(V2::ProjectPartContainer &&projectPart,
const V2::FileContainers &generatedFiles)
{
m_symbolsCollector.clear();
Sqlite::ImmediateTransaction transaction{m_transactionInterface};
const auto optionalArtefact = m_symbolStorage.fetchProjectPartArtefact(projectPart.projectPartId);
int projectPartId = m_symbolStorage.insertOrUpdateProjectPart(projectPart.projectPartId,
projectPart.arguments,
projectPart.compilerMacros,
projectPart.includeSearchPaths);
transaction.commit();
const Utils::optional<ProjectPartArtefact> optionalArtefact = m_symbolStorage.fetchProjectPartArtefact(
projectPart.projectPartId);
if (optionalArtefact)
projectPartId = optionalArtefact->projectPartId;
FilePathIds sourcePathIds = updatableFilePathIds(projectPart, optionalArtefact);
if (!sourcePathIds.empty()) {
m_symbolsCollector.addFiles(projectPart.sourcePathIds,
compilerArguments(projectPart, optionalArtefact));
if (sourcePathIds.empty())
return;
m_symbolsCollector.addUnsavedFiles(generatedFiles);
Utils::SmallStringVector arguments = compilerArguments(projectPart, optionalArtefact);
m_symbolsCollector.collectSymbols();
std::vector<SymbolIndexerTask> symbolIndexerTask;
symbolIndexerTask.reserve(projectPart.sourcePathIds.size());
Sqlite::ImmediateTransaction transaction{m_transactionInterface};
for (FilePathId sourcePathId : projectPart.sourcePathIds) {
auto indexing = [projectPart, arguments, generatedFiles, sourcePathId]
(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage,
Sqlite::TransactionInterface &transactionInterface) {
symbolsCollector.addFile(sourcePathId, arguments);
m_symbolStorage.addSymbolsAndSourceLocations(m_symbolsCollector.symbols(),
m_symbolsCollector.sourceLocations());
symbolsCollector.addUnsavedFiles(generatedFiles);
m_symbolStorage.insertOrUpdateProjectPart(projectPart.projectPartId,
projectPart.arguments,
projectPart.compilerMacros,
projectPart.includeSearchPaths);
m_symbolStorage.updateProjectPartSources(projectPart.projectPartId,
m_symbolsCollector.sourceFiles());
symbolsCollector.collectSymbols();
m_symbolStorage.insertOrUpdateUsedMacros(m_symbolsCollector.usedMacros());
Sqlite::ImmediateTransaction transaction{transactionInterface};
m_symbolStorage.insertFileStatuses(m_symbolsCollector.fileStatuses());
symbolStorage.addSymbolsAndSourceLocations(symbolsCollector.symbols(),
symbolsCollector.sourceLocations());
m_symbolStorage.insertOrUpdateSourceDependencies(m_symbolsCollector.sourceDependencies());
symbolStorage.updateProjectPartSources(projectPart.projectPartId,
symbolsCollector.sourceFiles());
transaction.commit();
symbolStorage.insertOrUpdateUsedMacros(symbolsCollector.usedMacros());
symbolStorage.insertFileStatuses(symbolsCollector.fileStatuses());
symbolStorage.insertOrUpdateSourceDependencies(symbolsCollector.sourceDependencies());
transaction.commit();
};
symbolIndexerTask.emplace_back(sourcePathId, projectPartId, std::move(indexing));
}
m_symbolIndexerTaskQueue.addOrUpdateTasks(std::move(symbolIndexerTask));
m_symbolIndexerTaskQueue.processTasks();
}
void SymbolIndexer::pathsWithIdsChanged(const Utils::SmallStringVector &)
@@ -95,40 +117,57 @@ void SymbolIndexer::pathsWithIdsChanged(const Utils::SmallStringVector &)
void SymbolIndexer::pathsChanged(const FilePathIds &filePathIds)
{
std::vector<SymbolIndexerTask> symbolIndexerTask;
symbolIndexerTask.reserve(filePathIds.size());
for (FilePathId filePathId : filePathIds)
updateChangedPath(filePathId);
updateChangedPath(filePathId, symbolIndexerTask);
m_symbolIndexerTaskQueue.addOrUpdateTasks(std::move(symbolIndexerTask));
m_symbolIndexerTaskQueue.processTasks();
}
void SymbolIndexer::updateChangedPath(FilePathId filePathId)
void SymbolIndexer::updateChangedPath(FilePathId filePathId,
std::vector<SymbolIndexerTask> &symbolIndexerTask)
{
m_symbolsCollector.clear();
m_fileStatusCache.update(filePathId);
Sqlite::DeferredTransaction transaction{m_transactionInterface};
const Utils::optional<ProjectPartArtefact> optionalArtefact = m_symbolStorage.fetchProjectPartArtefact(filePathId);
transaction.commit();
if (optionalArtefact && !optionalArtefact.value().compilerArguments.empty()) {
const ProjectPartArtefact &artefact = optionalArtefact.value();
m_symbolsCollector.addFiles({filePathId},
compilerArguments(artefact.compilerArguments, artefact.projectPartId));
Utils::SmallStringVector arguments = compilerArguments(artefact.compilerArguments,
artefact.projectPartId);
m_symbolsCollector.collectSymbols();
auto indexing = [projectPartId=artefact.projectPartId, arguments, filePathId]
(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage,
Sqlite::TransactionInterface &transactionInterface) {
symbolsCollector.addFile(filePathId, arguments);
Sqlite::ImmediateTransaction transaction{m_transactionInterface};
symbolsCollector.collectSymbols();
m_symbolStorage.addSymbolsAndSourceLocations(m_symbolsCollector.symbols(),
m_symbolsCollector.sourceLocations());
Sqlite::ImmediateTransaction transaction{transactionInterface};
m_symbolStorage.updateProjectPartSources(artefact.projectPartId,
m_symbolsCollector.sourceFiles());
symbolStorage.addSymbolsAndSourceLocations(symbolsCollector.symbols(),
symbolsCollector.sourceLocations());
m_symbolStorage.insertOrUpdateUsedMacros(m_symbolsCollector.usedMacros());
symbolStorage.updateProjectPartSources(projectPartId, symbolsCollector.sourceFiles());
m_symbolStorage.insertFileStatuses(m_symbolsCollector.fileStatuses());
symbolStorage.insertOrUpdateUsedMacros(symbolsCollector.usedMacros());
m_symbolStorage.insertOrUpdateSourceDependencies(m_symbolsCollector.sourceDependencies());
symbolStorage.insertFileStatuses(symbolsCollector.fileStatuses());
transaction.commit();
symbolStorage.insertOrUpdateSourceDependencies(symbolsCollector.sourceDependencies());
transaction.commit();
};
symbolIndexerTask.emplace_back(filePathId, optionalArtefact->projectPartId, std::move(indexing));
}
}

11
src/tools/clangrefactoringbackend/source/symbolindexer.h
View File

@@ -26,7 +26,7 @@
#pragma once
#include "filestatuscache.h"
#include "symbolscollectorinterface.h"
#include "symbolindexertaskqueueinterface.h"
#include "symbolstorageinterface.h"
#include "clangpathwatcher.h"
@@ -35,10 +35,12 @@
namespace ClangBackEnd {
class SymbolsCollectorInterface;
class SymbolIndexer : public ClangPathWatcherNotifier
{
public:
SymbolIndexer(SymbolsCollectorInterface &symbolsCollector,
SymbolIndexer(SymbolIndexerTaskQueueInterface &symbolIndexerTaskQueue,
SymbolStorageInterface &symbolStorage,
ClangPathWatcherInterface &pathWatcher,
FilePathCachingInterface &filePathCache,
@@ -52,7 +54,8 @@ public:
void pathsWithIdsChanged(const Utils::SmallStringVector &ids) override;
void pathsChanged(const FilePathIds &filePathIds) override;
void updateChangedPath(FilePathId filePath);
void updateChangedPath(FilePathId filePath,
std::vector<SymbolIndexerTask> &symbolIndexerTask);
bool compilerMacrosOrIncludeSearchPathsAreDifferent(
const V2::ProjectPartContainer &projectPart,
@@ -70,7 +73,7 @@ public:
int projectPartId) const;
private:
SymbolsCollectorInterface &m_symbolsCollector;
SymbolIndexerTaskQueueInterface &m_symbolIndexerTaskQueue;
SymbolStorageInterface &m_symbolStorage;
ClangPathWatcherInterface &m_pathWatcher;
FilePathCachingInterface &m_filePathCache;

87
src/tools/clangrefactoringbackend/source/symbolindexertask.h Normal file
View File

@@ -0,0 +1,87 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include <filepathid.h>
#include <functional>
namespace Sqlite {
class TransactionInterface;
}
namespace ClangBackEnd {
class SymbolIndexerTaskSchedulerInterface;
class SymbolsCollectorInterface;
class SymbolStorageInterface;
class SymbolIndexerTask
{
public:
using Callable = std::function<void(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage,
Sqlite::TransactionInterface &transaction)>;
SymbolIndexerTask(FilePathId filePathId,
int projectPartId,
Callable &&callable)
: callable(std::move(callable)),
filePathId(filePathId),
projectPartId(projectPartId)
{
}
SymbolIndexerTask clone() const
{
return *this;
}
friend
bool operator==(const SymbolIndexerTask &first, const SymbolIndexerTask &second)
{
return first.filePathId == second.filePathId && first.projectPartId == second.projectPartId;
}
friend
bool operator<(const SymbolIndexerTask &first, const SymbolIndexerTask &second)
{
return std::tie(first.filePathId, first.projectPartId)
< std::tie(second.filePathId, second.projectPartId);
}
operator Callable&&()
{
return std::move(callable);
}
public:
Callable callable;
FilePathId filePathId;
int projectPartId;
};
} // namespace ClangBackEnd

12
src/tools/clangrefactoringbackend/source/symbolindexertaskqueue.cpp
View File

@@ -25,6 +25,8 @@
#include "symbolindexertaskqueue.h"
#include <symbolindexertaskschedulerinterface.h>
namespace ClangBackEnd {
namespace {
@@ -60,11 +62,6 @@ OutputIt set_union_merge(InputIt1 first1,
}
SymbolIndexerTaskQueue::SymbolIndexerTaskQueue()
{
}
void SymbolIndexerTaskQueue::addOrUpdateTasks(std::vector<SymbolIndexerTask> &&tasks)
{
std::vector<SymbolIndexerTask> mergedTasks;
@@ -133,7 +130,12 @@ std::vector<std::size_t> SymbolIndexerTaskQueue::projectPartNumberIds(const Util
void SymbolIndexerTaskQueue::processTasks()
{
int taskCount = m_symbolIndexerScheduler.freeSlots();
auto newEnd = std::prev(m_tasks.end(), std::min<int>(taskCount, int(m_tasks.size())));
m_symbolIndexerScheduler.addTasks({std::make_move_iterator(newEnd),
std::make_move_iterator(m_tasks.end())});
m_tasks.erase(newEnd, m_tasks.end());
}
} // namespace ClangBackEnd

61
src/tools/clangrefactoringbackend/source/symbolindexertaskqueue.h
View File

@@ -26,6 +26,7 @@
#pragma once
#include "symbolindexertaskqueueinterface.h"
#include "symbolindexertask.h"
#include <filepathid.h>
@@ -34,59 +35,25 @@
#include <functional>
#include <vector>
namespace Sqlite {
class TransactionInterface;
}
namespace ClangBackEnd {
class SymbolIndexerTaskSchedulerInterface;
class SymbolsCollectorInterface;
class SymbolStorageInterface;
class SymbolIndexerTask
class SymbolIndexerTaskQueue final : public SymbolIndexerTaskQueueInterface
{
public:
using Callable = std::function<void(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage)>;
SymbolIndexerTaskQueue(SymbolIndexerTaskSchedulerInterface &symbolIndexerTaskScheduler)
: m_symbolIndexerScheduler(symbolIndexerTaskScheduler)
{}
SymbolIndexerTask(FilePathId filePathId,
std::size_t projectPartId,
Callable &&callable)
: callable(std::move(callable)),
filePathId(filePathId),
projectPartId(projectPartId)
{
}
SymbolIndexerTask clone() const
{
return *this;
}
friend
bool operator==(const SymbolIndexerTask &first, const SymbolIndexerTask &second)
{
return first.filePathId == second.filePathId && first.projectPartId == second.projectPartId;
}
friend
bool operator<(const SymbolIndexerTask &first, const SymbolIndexerTask &second)
{
return std::tie(first.filePathId, first.projectPartId)
< std::tie(second.filePathId, second.projectPartId);
}
public:
Callable callable;
FilePathId filePathId;
std::size_t projectPartId;
};
class SymbolIndexerTaskQueue
{
public:
SymbolIndexerTaskQueue();
void addOrUpdateTasks(std::vector<SymbolIndexerTask> &&tasks)
/* [[expects: std::is_sorted(tasks)]] */;
void removeTasks(const Utils::SmallStringVector &projectPartIds)
/* [[expects: std::is_sorted(projectPartIds)]] */;
void addOrUpdateTasks(std::vector<SymbolIndexerTask> &&tasks);
void removeTasks(const Utils::SmallStringVector &projectPartIds);
const std::vector<SymbolIndexerTask> &tasks() const;
@@ -94,11 +61,13 @@ public:
std::vector<std::size_t> projectPartNumberIds(const Utils::SmallStringVector &projectPartIds)
/* [[ensures result: std::is_sorted(result)]] */;
void processTasks() ;
void processTasks();
void syncTasks();
private:
std::vector<Utils::SmallString> m_projectPartIds;
std::vector<SymbolIndexerTask> m_tasks;
SymbolIndexerTaskSchedulerInterface &m_symbolIndexerScheduler;
};
} // namespace ClangBackEnd

12
src/tools/clangrefactoringbackend/source/symbolindexertaskqueueinterface.h
View File

@@ -25,11 +25,23 @@
#pragma once
#include <utils/smallstringvector.h>
namespace ClangBackEnd {
class SymbolIndexerTask;
class SymbolIndexerTaskQueueInterface
{
public:
SymbolIndexerTaskQueueInterface() = default;
SymbolIndexerTaskQueueInterface(const SymbolIndexerTaskQueueInterface &) = delete;
SymbolIndexerTaskQueueInterface &operator=(const SymbolIndexerTaskQueueInterface &) = delete;
virtual void addOrUpdateTasks(std::vector<SymbolIndexerTask> &&tasks) = 0
/* [[expects: std::is_sorted(tasks)]] */;
virtual void removeTasks(const Utils::SmallStringVector &projectPartIds) = 0
/* [[expects: std::is_sorted(projectPartIds)]] */;
virtual void processTasks() = 0;
protected:

18
src/tools/clangrefactoringbackend/source/symbolindexertaskscheduler.cpp
View File

@@ -87,7 +87,7 @@ void SymbolIndexerTaskScheduler::addTasks(std::vector<Task> &&tasks)
auto callWrapper = [&, task=std::move(task)] (
std::reference_wrapper<SymbolsCollectorInterface> symbolsCollector)
-> SymbolsCollectorInterface& {
task(symbolsCollector.get(), m_symbolStorage);
task(symbolsCollector.get(), m_symbolStorage, m_transactionInterface);
executeInLoop([&] {
m_symbolIndexerTaskQueue.processTasks();
});
@@ -105,11 +105,14 @@ const std::vector<SymbolIndexerTaskScheduler::Future> &SymbolIndexerTaskSchedule
return m_futures;
}
int SymbolIndexerTaskScheduler::freeSlots()
uint SymbolIndexerTaskScheduler::freeSlots()
{
removeFinishedFutures();
return std::max(m_hardware_concurrency - int(m_futures.size()), 0);
if (m_isDisabled)
return 0;
return uint(std::max(int(m_hardware_concurrency) - int(m_futures.size()), 0));
}
void SymbolIndexerTaskScheduler::syncTasks()
@@ -118,6 +121,11 @@ void SymbolIndexerTaskScheduler::syncTasks()
future.wait();
}
void SymbolIndexerTaskScheduler::disable()
{
m_isDisabled = true;
}
void SymbolIndexerTaskScheduler::removeFinishedFutures()
{
auto notReady = [] (Future &future) {
@@ -127,7 +135,9 @@ void SymbolIndexerTaskScheduler::removeFinishedFutures()
auto split = std::partition(m_futures.begin(), m_futures.end(), notReady);
std::for_each(split, m_futures.end(), [] (Future &future) {
future.get().setIsUsed(false);
SymbolsCollectorInterface &symbolCollector = future.get();
symbolCollector.setIsUsed(false);
symbolCollector.clear();
});
m_futures.erase(split, m_futures.end());

24
src/tools/clangrefactoringbackend/source/symbolindexertaskscheduler.h
View File

@@ -25,10 +25,17 @@
#pragma once
#include "symbolindexertaskschedulerinterface.h"
#include "symbolindexertask.h"
#include <functional>
#include <future>
#include <vector>
namespace Sqlite {
class TransactionInterface;
};
namespace ClangBackEnd {
class FilePathCachingInterface;
@@ -37,20 +44,21 @@ class SymbolsCollectorManagerInterface;
class SymbolIndexerTaskQueueInterface;
class SymbolStorageInterface;
class SymbolIndexerTaskScheduler
class SymbolIndexerTaskScheduler final : public SymbolIndexerTaskSchedulerInterface
{
public:
using Task = std::function<void(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage)>;
using Task = SymbolIndexerTask::Callable;
using Future = std::future<SymbolsCollectorInterface&>;
SymbolIndexerTaskScheduler(SymbolsCollectorManagerInterface &symbolsCollectorManager,
SymbolStorageInterface &symbolStorage,
Sqlite::TransactionInterface &transactionInterface,
SymbolIndexerTaskQueueInterface &symbolIndexerTaskQueue,
int hardware_concurrency,
uint hardware_concurrency,
std::launch launchPolicy = std::launch::async)
: m_symbolsCollectorManager(symbolsCollectorManager),
m_symbolStorage(symbolStorage),
m_transactionInterface(transactionInterface),
m_symbolIndexerTaskQueue(symbolIndexerTaskQueue),
m_hardware_concurrency(hardware_concurrency),
m_launchPolicy(launchPolicy)
@@ -60,10 +68,12 @@ public:
const std::vector<Future> &futures() const;
int freeSlots();
uint freeSlots();
void syncTasks();
void disable();
private:
void removeFinishedFutures();
@@ -71,9 +81,11 @@ private:
std::vector<Future> m_futures;
SymbolsCollectorManagerInterface &m_symbolsCollectorManager;
SymbolStorageInterface &m_symbolStorage;
Sqlite::TransactionInterface &m_transactionInterface;
SymbolIndexerTaskQueueInterface &m_symbolIndexerTaskQueue;
int m_hardware_concurrency;
uint m_hardware_concurrency;
std::launch m_launchPolicy;
bool m_isDisabled = false;
};
} // namespace ClangBackEnd

60
src/tools/clangrefactoringbackend/source/symbolindexertaskschedulerinterface.h Normal file
View File

@@ -0,0 +1,60 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include <functional>
#include <vector>
namespace Sqlite {
class TransactionInterface;
}
namespace ClangBackEnd {
class SymbolsCollectorInterface;
class SymbolStorageInterface;
using uint = unsigned int;
class SymbolIndexerTaskSchedulerInterface
{
public:
using Task = std::function<void(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage,
Sqlite::TransactionInterface &transaction)>;
SymbolIndexerTaskSchedulerInterface() = default;
SymbolIndexerTaskSchedulerInterface(const SymbolIndexerTaskSchedulerInterface &) = delete;
SymbolIndexerTaskSchedulerInterface &operator=(const SymbolIndexerTaskSchedulerInterface &) = delete;
virtual void addTasks(std::vector<Task> &&tasks) = 0;
virtual uint freeSlots() = 0;
protected:
~SymbolIndexerTaskSchedulerInterface() = default;
};
} // namespace ClangBackEnd

29
src/tools/clangrefactoringbackend/source/symbolindexing.h
View File

@@ -30,6 +30,9 @@
#include "storagesqlitestatementfactory.h"
#include "symbolindexer.h"
#include "symbolscollector.h"
#include "symbolscollectormanager.h"
#include "symbolindexertaskqueue.h"
#include "symbolindexertaskscheduler.h"
#include "symbolstorage.h"
#include <refactoringdatabaseinitializer.h>
@@ -41,6 +44,8 @@
#include <QFileSystemWatcher>
#include <thread>
namespace ClangBackEnd {
class SymbolIndexing final : public SymbolIndexingInterface
@@ -52,26 +57,44 @@ public:
SymbolIndexing(Sqlite::Database &database,
FilePathCachingInterface &filePathCache)
: m_filePathCache(filePathCache),
m_statementFactory(database)
m_statementFactory(database),
m_collectorManger(database),
m_indexerScheduler(m_collectorManger, m_symbolStorage, database, m_indexerQueue, std::thread::hardware_concurrency())
{
}
~SymbolIndexing()
{
syncTasks();
}
SymbolIndexer &indexer()
{
return m_indexer;
}
void syncTasks()
{
m_indexerScheduler.disable();
while (!m_indexerScheduler.futures().empty()) {
m_indexerScheduler.syncTasks();
m_indexerScheduler.freeSlots();
}
}
void updateProjectParts(V2::ProjectPartContainers &&projectParts,
const V2::FileContainers &generatedFiles) override;
private:
FilePathCachingInterface &m_filePathCache;
SymbolsCollector m_collector{m_filePathCache};
StatementFactory m_statementFactory;
Storage m_symbolStorage{m_statementFactory};
ClangPathWatcher<QFileSystemWatcher, QTimer> m_sourceWatcher{m_filePathCache};
FileStatusCache m_fileStatusCache{m_filePathCache};
SymbolIndexer m_indexer{m_collector,
SymbolsCollectorManager<SymbolsCollector> m_collectorManger;
SymbolIndexerTaskScheduler m_indexerScheduler;
SymbolIndexerTaskQueue m_indexerQueue{m_indexerScheduler};
SymbolIndexer m_indexer{m_indexerQueue,
m_symbolStorage,
m_sourceWatcher,
m_filePathCache,

13
src/tools/clangrefactoringbackend/source/symbolscollector.cpp
View File

@@ -29,11 +29,11 @@
namespace ClangBackEnd {
SymbolsCollector::SymbolsCollector(FilePathCachingInterface &filePathCache)
: m_indexDataConsumer(std::make_shared<IndexDataConsumer>(m_symbolEntries, m_sourceLocationEntries, filePathCache, m_sourcesManager)),
SymbolsCollector::SymbolsCollector(Sqlite::Database &database)
: m_filePathCache(database),
m_indexDataConsumer(std::make_shared<IndexDataConsumer>(m_symbolEntries, m_sourceLocationEntries, m_filePathCache, m_sourcesManager)),
m_collectSymbolsAction(m_indexDataConsumer),
m_collectMacrosSourceFileCallbacks(m_symbolEntries, m_sourceLocationEntries, filePathCache, m_sourcesManager),
m_filePathCache(filePathCache)
m_collectMacrosSourceFileCallbacks(m_symbolEntries, m_sourceLocationEntries, m_filePathCache, m_sourcesManager)
{
}
@@ -44,6 +44,11 @@ void SymbolsCollector::addFiles(const FilePathIds &filePathIds,
m_collectMacrosSourceFileCallbacks.addSourceFiles(filePathIds);
}
void SymbolsCollector::addFile(FilePathId filePathId, const Utils::SmallStringVector &arguments)
{
addFiles({filePathId}, arguments);
}
void SymbolsCollector::addUnsavedFiles(const V2::FileContainers &unsavedFiles)
{
m_clangTool.addUnsavedFiles(unsavedFiles);

13
src/tools/clangrefactoringbackend/source/symbolscollector.h
View File

@@ -31,17 +31,22 @@
#include "sourcesmanager.h"
#include "symbolscollectorinterface.h"
#include <filepathcachingfwd.h>
#include <filepathcaching.h>
namespace Sqlite {
class Database;
}
namespace ClangBackEnd {
class SymbolsCollector final : public SymbolsCollectorInterface
{
public:
SymbolsCollector(FilePathCachingInterface &filePathCache);
SymbolsCollector(Sqlite::Database &database);
void addFiles(const FilePathIds &filePathIds,
const Utils::SmallStringVector &arguments) override;
const Utils::SmallStringVector &arguments);
void addFile(FilePathId filePathId, const Utils::SmallStringVector &arguments) override;
void addUnsavedFiles(const V2::FileContainers &unsavedFiles) override;
@@ -60,6 +65,7 @@ public:
void setIsUsed(bool isUsed) override;
private:
FilePathCaching m_filePathCache;
ClangTool m_clangTool;
SymbolEntries m_symbolEntries;
SourceLocationEntries m_sourceLocationEntries;
@@ -67,7 +73,6 @@ private:
CollectSymbolsAction m_collectSymbolsAction;
CollectMacrosSourceFileCallbacks m_collectMacrosSourceFileCallbacks;
SourcesManager m_sourcesManager;
FilePathCachingInterface &m_filePathCache;
bool m_isUsed = false;
};

7
src/tools/clangrefactoringbackend/source/symbolscollectorinterface.h
View File

@@ -44,11 +44,11 @@ class SymbolsCollectorInterface
{
public:
SymbolsCollectorInterface() = default;
virtual ~SymbolsCollectorInterface() = default;
SymbolsCollectorInterface(const SymbolsCollectorInterface &) = delete;
SymbolsCollectorInterface &operator=(const SymbolsCollectorInterface &) = delete;
virtual void addFiles(const FilePathIds &filePathIds,
const Utils::SmallStringVector &arguments) = 0;
virtual void addFile(FilePathId filePathId, const Utils::SmallStringVector &arguments) = 0;
virtual void addUnsavedFiles(const V2::FileContainers &unsavedFiles) = 0;
@@ -65,9 +65,6 @@ public:
virtual bool isUsed() const = 0;
virtual void setIsUsed(bool isUsed) = 0;
protected:
~SymbolsCollectorInterface() = default;
};
} // namespace ClangBackEnd

35
src/tools/clangrefactoringbackend/source/symbolscollectormanager.cpp Normal file
View File

@@ -0,0 +1,35 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "symbolscollectormanager.h"
#include "symbolscollector.h"
namespace ClangBackEnd {
} // namespace ClangBackEnd

83
src/tools/clangrefactoringbackend/source/symbolscollectormanager.h Normal file
View File

@@ -0,0 +1,83 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "symbolscollectormanagerinterface.h"
#include "symbolscollectorinterface.h"
#include <memory>
namespace Sqlite {
class Database;
}
namespace ClangBackEnd {
class SymbolsCollector;
template<typename SymbolsCollector>
class SymbolsCollectorManager final : public SymbolsCollectorManagerInterface
{
public:
SymbolsCollectorManager(Sqlite::Database &database)
: m_database(database)
{}
SymbolsCollector &unusedSymbolsCollector() override
{
auto split = std::partition(m_collectors.begin(),
m_collectors.end(),
[] (const auto &collector) {
return collector->isUsed();
});
auto freeCollectors = std::distance(split, m_collectors.end());
if (freeCollectors > 0)
return usedCollector(*split->get());
m_collectors.emplace_back(std::make_unique<SymbolsCollector>(m_database));
return usedCollector(*m_collectors.back().get());
}
const std::vector<std::unique_ptr<SymbolsCollector>> &collectors() const
{
return m_collectors;
}
private:
SymbolsCollector &usedCollector(SymbolsCollector &collector)
{
collector.setIsUsed(true);
return collector;
}
private:
std::vector<std::unique_ptr<SymbolsCollector>> m_collectors;
Sqlite::Database &m_database;
};
} // namespace ClangBackEnd

4
src/tools/clangrefactoringbackend/source/symbolscollectormanagerinterface.h
View File

@@ -34,6 +34,10 @@ class SymbolsCollectorInterface;
class SymbolsCollectorManagerInterface
{
public:
SymbolsCollectorManagerInterface() = default;
SymbolsCollectorManagerInterface(const SymbolsCollectorManagerInterface &) = delete;
SymbolsCollectorManagerInterface &operator=(const SymbolsCollectorManagerInterface &) = delete;
virtual SymbolsCollectorInterface &unusedSymbolsCollector() = 0;
protected:

4
src/tools/clangrefactoringbackend/source/symbolstorage.h
View File

@@ -64,7 +64,7 @@ public:
deleteNewLocationsTable();
}
void insertOrUpdateProjectPart(Utils::SmallStringView projectPartName,
int insertOrUpdateProjectPart(Utils::SmallStringView projectPartName,
const Utils::SmallStringVector &commandLineArguments,
const CompilerMacros &compilerMacros,
const Utils::SmallStringVector &includeSearchPaths) override
@@ -88,6 +88,8 @@ public:
includeSearchPathsAsJason,
projectPartName);
}
return int(m_statementFactory.database.lastInsertedRowId());
}
Utils::optional<ProjectPartArtefact> fetchProjectPartArtefact(FilePathId sourceId) const override

8
src/tools/clangrefactoringbackend/source/symbolstorageinterface.h
View File

@@ -49,10 +49,10 @@ public:
virtual void addSymbolsAndSourceLocations(const SymbolEntries &symbolEntries,
const SourceLocationEntries &sourceLocations) = 0;
virtual void insertOrUpdateProjectPart(Utils::SmallStringView projectPartName,
const Utils::SmallStringVector &commandLineArguments,
const CompilerMacros &compilerMacros,
const Utils::SmallStringVector &includeSearchPaths) = 0;
virtual int insertOrUpdateProjectPart(Utils::SmallStringView projectPartName,
const Utils::SmallStringVector &commandLineArguments,
const CompilerMacros &compilerMacros,
const Utils::SmallStringVector &includeSearchPaths) = 0;
virtual void updateProjectPartSources(Utils::SmallStringView projectPartName,
const FilePathIds &sourceFilePathIds) = 0;
virtual void updateProjectPartSources(int projectPartId,