feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

Clang: Introduce parallel indexing

Browse Source 
Change-Id: I522cb18e6d24b7dbed5d5dfa3a732e5b3b5113bb
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>
This commit is contained in:
Marco Bubke
2018-08-28 12:08:37 +02:00
parent 86cd29b13c
commit a86867eb8a
32 changed files with 828 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/tools/clangrefactoringbackend/source/clangrefactoringbackend-source.pri
View File

@@ -27,7 +27,10 @@ HEADERS += \
$$PWD/symbolindexertaskqueue.h \
$$PWD/symbolindexertaskscheduler.h \
$$PWD/symbolscollectormanagerinterface.h \
$$PWD/symbolindexertaskqueueinterface.h
$$PWD/symbolindexertaskqueueinterface.h \
$$PWD/symbolindexertaskschedulerinterface.h \
$$PWD/symbolscollectormanager.h \
$$PWD/symbolindexertask.h
!isEmpty(LIBTOOLING_LIBS) {
SOURCES += \
@@ -75,4 +78,5 @@ SOURCES += \
$$PWD/filestatuscache.cpp \
$$PWD/projectpartqueue.cpp \
$$PWD/symbolindexertaskqueue.cpp \
$$PWD/symbolindexertaskscheduler.cpp
$$PWD/symbolindexertaskscheduler.cpp \
$$PWD/symbolscollectormanager.cpp

115
src/tools/clangrefactoringbackend/source/symbolindexer.cpp
View File

@@ -25,15 +25,18 @@
#include "symbolindexer.h"
#include <symbolscollector.h>
#include <symbolindexertaskqueue.h>
namespace ClangBackEnd {
SymbolIndexer::SymbolIndexer(SymbolsCollectorInterface &symbolsCollector,
SymbolIndexer::SymbolIndexer(SymbolIndexerTaskQueueInterface &symbolIndexerTaskQueue,
SymbolStorageInterface &symbolStorage,
ClangPathWatcherInterface &pathWatcher,
FilePathCachingInterface &filePathCache,
FileStatusCache &fileStatusCache,
Sqlite::TransactionInterface &transactionInterface)
: m_symbolsCollector(symbolsCollector),
: m_symbolIndexerTaskQueue(symbolIndexerTaskQueue),
m_symbolStorage(symbolStorage),
m_pathWatcher(pathWatcher),
m_filePathCache(filePathCache),
@@ -52,41 +55,60 @@ void SymbolIndexer::updateProjectParts(V2::ProjectPartContainers &&projectParts,
void SymbolIndexer::updateProjectPart(V2::ProjectPartContainer &&projectPart,
const V2::FileContainers &generatedFiles)
{
m_symbolsCollector.clear();
Sqlite::ImmediateTransaction transaction{m_transactionInterface};
const auto optionalArtefact = m_symbolStorage.fetchProjectPartArtefact(projectPart.projectPartId);
int projectPartId = m_symbolStorage.insertOrUpdateProjectPart(projectPart.projectPartId,
projectPart.arguments,
projectPart.compilerMacros,
projectPart.includeSearchPaths);
transaction.commit();
const Utils::optional<ProjectPartArtefact> optionalArtefact = m_symbolStorage.fetchProjectPartArtefact(
projectPart.projectPartId);
if (optionalArtefact)
projectPartId = optionalArtefact->projectPartId;
FilePathIds sourcePathIds = updatableFilePathIds(projectPart, optionalArtefact);
if (!sourcePathIds.empty()) {
m_symbolsCollector.addFiles(projectPart.sourcePathIds,
compilerArguments(projectPart, optionalArtefact));
if (sourcePathIds.empty())
return;
m_symbolsCollector.addUnsavedFiles(generatedFiles);
Utils::SmallStringVector arguments = compilerArguments(projectPart, optionalArtefact);
m_symbolsCollector.collectSymbols();
std::vector<SymbolIndexerTask> symbolIndexerTask;
symbolIndexerTask.reserve(projectPart.sourcePathIds.size());
Sqlite::ImmediateTransaction transaction{m_transactionInterface};
for (FilePathId sourcePathId : projectPart.sourcePathIds) {
auto indexing = [projectPart, arguments, generatedFiles, sourcePathId]
(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage,
Sqlite::TransactionInterface &transactionInterface) {
symbolsCollector.addFile(sourcePathId, arguments);
m_symbolStorage.addSymbolsAndSourceLocations(m_symbolsCollector.symbols(),
m_symbolsCollector.sourceLocations());
symbolsCollector.addUnsavedFiles(generatedFiles);
m_symbolStorage.insertOrUpdateProjectPart(projectPart.projectPartId,
projectPart.arguments,
projectPart.compilerMacros,
projectPart.includeSearchPaths);
m_symbolStorage.updateProjectPartSources(projectPart.projectPartId,
m_symbolsCollector.sourceFiles());
symbolsCollector.collectSymbols();
m_symbolStorage.insertOrUpdateUsedMacros(m_symbolsCollector.usedMacros());
Sqlite::ImmediateTransaction transaction{transactionInterface};
m_symbolStorage.insertFileStatuses(m_symbolsCollector.fileStatuses());
symbolStorage.addSymbolsAndSourceLocations(symbolsCollector.symbols(),
symbolsCollector.sourceLocations());
m_symbolStorage.insertOrUpdateSourceDependencies(m_symbolsCollector.sourceDependencies());
symbolStorage.updateProjectPartSources(projectPart.projectPartId,
symbolsCollector.sourceFiles());
transaction.commit();
symbolStorage.insertOrUpdateUsedMacros(symbolsCollector.usedMacros());
symbolStorage.insertFileStatuses(symbolsCollector.fileStatuses());
symbolStorage.insertOrUpdateSourceDependencies(symbolsCollector.sourceDependencies());
transaction.commit();
};
symbolIndexerTask.emplace_back(sourcePathId, projectPartId, std::move(indexing));
}
m_symbolIndexerTaskQueue.addOrUpdateTasks(std::move(symbolIndexerTask));
m_symbolIndexerTaskQueue.processTasks();
}
void SymbolIndexer::pathsWithIdsChanged(const Utils::SmallStringVector &)
@@ -95,40 +117,57 @@ void SymbolIndexer::pathsWithIdsChanged(const Utils::SmallStringVector &)
void SymbolIndexer::pathsChanged(const FilePathIds &filePathIds)
{
std::vector<SymbolIndexerTask> symbolIndexerTask;
symbolIndexerTask.reserve(filePathIds.size());
for (FilePathId filePathId : filePathIds)
updateChangedPath(filePathId);
updateChangedPath(filePathId, symbolIndexerTask);
m_symbolIndexerTaskQueue.addOrUpdateTasks(std::move(symbolIndexerTask));
m_symbolIndexerTaskQueue.processTasks();
}
void SymbolIndexer::updateChangedPath(FilePathId filePathId)
void SymbolIndexer::updateChangedPath(FilePathId filePathId,
std::vector<SymbolIndexerTask> &symbolIndexerTask)
{
m_symbolsCollector.clear();
m_fileStatusCache.update(filePathId);
Sqlite::DeferredTransaction transaction{m_transactionInterface};
const Utils::optional<ProjectPartArtefact> optionalArtefact = m_symbolStorage.fetchProjectPartArtefact(filePathId);
transaction.commit();
if (optionalArtefact && !optionalArtefact.value().compilerArguments.empty()) {
const ProjectPartArtefact &artefact = optionalArtefact.value();
m_symbolsCollector.addFiles({filePathId},
compilerArguments(artefact.compilerArguments, artefact.projectPartId));
Utils::SmallStringVector arguments = compilerArguments(artefact.compilerArguments,
artefact.projectPartId);
m_symbolsCollector.collectSymbols();
auto indexing = [projectPartId=artefact.projectPartId, arguments, filePathId]
(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage,
Sqlite::TransactionInterface &transactionInterface) {
symbolsCollector.addFile(filePathId, arguments);
Sqlite::ImmediateTransaction transaction{m_transactionInterface};
symbolsCollector.collectSymbols();
m_symbolStorage.addSymbolsAndSourceLocations(m_symbolsCollector.symbols(),
m_symbolsCollector.sourceLocations());
Sqlite::ImmediateTransaction transaction{transactionInterface};
m_symbolStorage.updateProjectPartSources(artefact.projectPartId,
m_symbolsCollector.sourceFiles());
symbolStorage.addSymbolsAndSourceLocations(symbolsCollector.symbols(),
symbolsCollector.sourceLocations());
m_symbolStorage.insertOrUpdateUsedMacros(m_symbolsCollector.usedMacros());
symbolStorage.updateProjectPartSources(projectPartId, symbolsCollector.sourceFiles());
m_symbolStorage.insertFileStatuses(m_symbolsCollector.fileStatuses());
symbolStorage.insertOrUpdateUsedMacros(symbolsCollector.usedMacros());
m_symbolStorage.insertOrUpdateSourceDependencies(m_symbolsCollector.sourceDependencies());
symbolStorage.insertFileStatuses(symbolsCollector.fileStatuses());
transaction.commit();
symbolStorage.insertOrUpdateSourceDependencies(symbolsCollector.sourceDependencies());
transaction.commit();
};
symbolIndexerTask.emplace_back(filePathId, optionalArtefact->projectPartId, std::move(indexing));
}
}

11
src/tools/clangrefactoringbackend/source/symbolindexer.h
View File

@@ -26,7 +26,7 @@
#pragma once
#include "filestatuscache.h"
#include "symbolscollectorinterface.h"
#include "symbolindexertaskqueueinterface.h"
#include "symbolstorageinterface.h"
#include "clangpathwatcher.h"
@@ -35,10 +35,12 @@
namespace ClangBackEnd {
class SymbolsCollectorInterface;
class SymbolIndexer : public ClangPathWatcherNotifier
{
public:
SymbolIndexer(SymbolsCollectorInterface &symbolsCollector,
SymbolIndexer(SymbolIndexerTaskQueueInterface &symbolIndexerTaskQueue,
SymbolStorageInterface &symbolStorage,
ClangPathWatcherInterface &pathWatcher,
FilePathCachingInterface &filePathCache,
@@ -52,7 +54,8 @@ public:
void pathsWithIdsChanged(const Utils::SmallStringVector &ids) override;
void pathsChanged(const FilePathIds &filePathIds) override;
void updateChangedPath(FilePathId filePath);
void updateChangedPath(FilePathId filePath,
std::vector<SymbolIndexerTask> &symbolIndexerTask);
bool compilerMacrosOrIncludeSearchPathsAreDifferent(
const V2::ProjectPartContainer &projectPart,
@@ -70,7 +73,7 @@ public:
int projectPartId) const;
private:
SymbolsCollectorInterface &m_symbolsCollector;
SymbolIndexerTaskQueueInterface &m_symbolIndexerTaskQueue;
SymbolStorageInterface &m_symbolStorage;
ClangPathWatcherInterface &m_pathWatcher;
FilePathCachingInterface &m_filePathCache;

87
src/tools/clangrefactoringbackend/source/symbolindexertask.h Normal file
View File

@@ -0,0 +1,87 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include <filepathid.h>
#include <functional>
namespace Sqlite {
class TransactionInterface;
}
namespace ClangBackEnd {
class SymbolIndexerTaskSchedulerInterface;
class SymbolsCollectorInterface;
class SymbolStorageInterface;
class SymbolIndexerTask
{
public:
using Callable = std::function<void(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage,
Sqlite::TransactionInterface &transaction)>;
SymbolIndexerTask(FilePathId filePathId,
int projectPartId,
Callable &&callable)
: callable(std::move(callable)),
filePathId(filePathId),
projectPartId(projectPartId)
{
}
SymbolIndexerTask clone() const
{
return *this;
}
friend
bool operator==(const SymbolIndexerTask &first, const SymbolIndexerTask &second)
{
return first.filePathId == second.filePathId && first.projectPartId == second.projectPartId;
}
friend
bool operator<(const SymbolIndexerTask &first, const SymbolIndexerTask &second)
{
return std::tie(first.filePathId, first.projectPartId)
< std::tie(second.filePathId, second.projectPartId);
}
operator Callable&&()
{
return std::move(callable);
}
public:
Callable callable;
FilePathId filePathId;
int projectPartId;
};
} // namespace ClangBackEnd

12
src/tools/clangrefactoringbackend/source/symbolindexertaskqueue.cpp
View File

@@ -25,6 +25,8 @@
#include "symbolindexertaskqueue.h"
#include <symbolindexertaskschedulerinterface.h>
namespace ClangBackEnd {
namespace {
@@ -60,11 +62,6 @@ OutputIt set_union_merge(InputIt1 first1,
}
SymbolIndexerTaskQueue::SymbolIndexerTaskQueue()
{
}
void SymbolIndexerTaskQueue::addOrUpdateTasks(std::vector<SymbolIndexerTask> &&tasks)
{
std::vector<SymbolIndexerTask> mergedTasks;
@@ -133,7 +130,12 @@ std::vector<std::size_t> SymbolIndexerTaskQueue::projectPartNumberIds(const Util
void SymbolIndexerTaskQueue::processTasks()
{
int taskCount = m_symbolIndexerScheduler.freeSlots();
auto newEnd = std::prev(m_tasks.end(), std::min<int>(taskCount, int(m_tasks.size())));
m_symbolIndexerScheduler.addTasks({std::make_move_iterator(newEnd),
std::make_move_iterator(m_tasks.end())});
m_tasks.erase(newEnd, m_tasks.end());
}
} // namespace ClangBackEnd

61
src/tools/clangrefactoringbackend/source/symbolindexertaskqueue.h
View File

@@ -26,6 +26,7 @@
#pragma once
#include "symbolindexertaskqueueinterface.h"
#include "symbolindexertask.h"
#include <filepathid.h>
@@ -34,59 +35,25 @@
#include <functional>
#include <vector>
namespace Sqlite {
class TransactionInterface;
}
namespace ClangBackEnd {
class SymbolIndexerTaskSchedulerInterface;
class SymbolsCollectorInterface;
class SymbolStorageInterface;
class SymbolIndexerTask
class SymbolIndexerTaskQueue final : public SymbolIndexerTaskQueueInterface
{
public:
using Callable = std::function<void(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage)>;
SymbolIndexerTaskQueue(SymbolIndexerTaskSchedulerInterface &symbolIndexerTaskScheduler)
: m_symbolIndexerScheduler(symbolIndexerTaskScheduler)
{}
SymbolIndexerTask(FilePathId filePathId,
std::size_t projectPartId,
Callable &&callable)
: callable(std::move(callable)),
filePathId(filePathId),
projectPartId(projectPartId)
{
}
SymbolIndexerTask clone() const
{
return *this;
}
friend
bool operator==(const SymbolIndexerTask &first, const SymbolIndexerTask &second)
{
return first.filePathId == second.filePathId && first.projectPartId == second.projectPartId;
}
friend
bool operator<(const SymbolIndexerTask &first, const SymbolIndexerTask &second)
{
return std::tie(first.filePathId, first.projectPartId)
< std::tie(second.filePathId, second.projectPartId);
}
public:
Callable callable;
FilePathId filePathId;
std::size_t projectPartId;
};
class SymbolIndexerTaskQueue
{
public:
SymbolIndexerTaskQueue();
void addOrUpdateTasks(std::vector<SymbolIndexerTask> &&tasks)
/* [[expects: std::is_sorted(tasks)]] */;
void removeTasks(const Utils::SmallStringVector &projectPartIds)
/* [[expects: std::is_sorted(projectPartIds)]] */;
void addOrUpdateTasks(std::vector<SymbolIndexerTask> &&tasks);
void removeTasks(const Utils::SmallStringVector &projectPartIds);
const std::vector<SymbolIndexerTask> &tasks() const;
@@ -94,11 +61,13 @@ public:
std::vector<std::size_t> projectPartNumberIds(const Utils::SmallStringVector &projectPartIds)
/* [[ensures result: std::is_sorted(result)]] */;
void processTasks() ;
void processTasks();
void syncTasks();
private:
std::vector<Utils::SmallString> m_projectPartIds;
std::vector<SymbolIndexerTask> m_tasks;
SymbolIndexerTaskSchedulerInterface &m_symbolIndexerScheduler;
};
} // namespace ClangBackEnd

12
src/tools/clangrefactoringbackend/source/symbolindexertaskqueueinterface.h
View File

@@ -25,11 +25,23 @@
#pragma once
#include <utils/smallstringvector.h>
namespace ClangBackEnd {
class SymbolIndexerTask;
class SymbolIndexerTaskQueueInterface
{
public:
SymbolIndexerTaskQueueInterface() = default;
SymbolIndexerTaskQueueInterface(const SymbolIndexerTaskQueueInterface &) = delete;
SymbolIndexerTaskQueueInterface &operator=(const SymbolIndexerTaskQueueInterface &) = delete;
virtual void addOrUpdateTasks(std::vector<SymbolIndexerTask> &&tasks) = 0
/* [[expects: std::is_sorted(tasks)]] */;
virtual void removeTasks(const Utils::SmallStringVector &projectPartIds) = 0
/* [[expects: std::is_sorted(projectPartIds)]] */;
virtual void processTasks() = 0;
protected:

18
src/tools/clangrefactoringbackend/source/symbolindexertaskscheduler.cpp
View File

@@ -87,7 +87,7 @@ void SymbolIndexerTaskScheduler::addTasks(std::vector<Task> &&tasks)
auto callWrapper = [&, task=std::move(task)] (
std::reference_wrapper<SymbolsCollectorInterface> symbolsCollector)
-> SymbolsCollectorInterface& {
task(symbolsCollector.get(), m_symbolStorage);
task(symbolsCollector.get(), m_symbolStorage, m_transactionInterface);
executeInLoop([&] {
m_symbolIndexerTaskQueue.processTasks();
});
@@ -105,11 +105,14 @@ const std::vector<SymbolIndexerTaskScheduler::Future> &SymbolIndexerTaskSchedule
return m_futures;
}
int SymbolIndexerTaskScheduler::freeSlots()
uint SymbolIndexerTaskScheduler::freeSlots()
{
removeFinishedFutures();
return std::max(m_hardware_concurrency - int(m_futures.size()), 0);
if (m_isDisabled)
return 0;
return uint(std::max(int(m_hardware_concurrency) - int(m_futures.size()), 0));
}
void SymbolIndexerTaskScheduler::syncTasks()
@@ -118,6 +121,11 @@ void SymbolIndexerTaskScheduler::syncTasks()
future.wait();
}
void SymbolIndexerTaskScheduler::disable()
{
m_isDisabled = true;
}
void SymbolIndexerTaskScheduler::removeFinishedFutures()
{
auto notReady = [] (Future &future) {
@@ -127,7 +135,9 @@ void SymbolIndexerTaskScheduler::removeFinishedFutures()
auto split = std::partition(m_futures.begin(), m_futures.end(), notReady);
std::for_each(split, m_futures.end(), [] (Future &future) {
future.get().setIsUsed(false);
SymbolsCollectorInterface &symbolCollector = future.get();
symbolCollector.setIsUsed(false);
symbolCollector.clear();
});
m_futures.erase(split, m_futures.end());

24
src/tools/clangrefactoringbackend/source/symbolindexertaskscheduler.h
View File

@@ -25,10 +25,17 @@
#pragma once
#include "symbolindexertaskschedulerinterface.h"
#include "symbolindexertask.h"
#include <functional>
#include <future>
#include <vector>
namespace Sqlite {
class TransactionInterface;
};
namespace ClangBackEnd {
class FilePathCachingInterface;
@@ -37,20 +44,21 @@ class SymbolsCollectorManagerInterface;
class SymbolIndexerTaskQueueInterface;
class SymbolStorageInterface;
class SymbolIndexerTaskScheduler
class SymbolIndexerTaskScheduler final : public SymbolIndexerTaskSchedulerInterface
{
public:
using Task = std::function<void(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage)>;
using Task = SymbolIndexerTask::Callable;
using Future = std::future<SymbolsCollectorInterface&>;
SymbolIndexerTaskScheduler(SymbolsCollectorManagerInterface &symbolsCollectorManager,
SymbolStorageInterface &symbolStorage,
Sqlite::TransactionInterface &transactionInterface,
SymbolIndexerTaskQueueInterface &symbolIndexerTaskQueue,
int hardware_concurrency,
uint hardware_concurrency,
std::launch launchPolicy = std::launch::async)
: m_symbolsCollectorManager(symbolsCollectorManager),
m_symbolStorage(symbolStorage),
m_transactionInterface(transactionInterface),
m_symbolIndexerTaskQueue(symbolIndexerTaskQueue),
m_hardware_concurrency(hardware_concurrency),
m_launchPolicy(launchPolicy)
@@ -60,10 +68,12 @@ public:
const std::vector<Future> &futures() const;
int freeSlots();
uint freeSlots();
void syncTasks();
void disable();
private:
void removeFinishedFutures();
@@ -71,9 +81,11 @@ private:
std::vector<Future> m_futures;
SymbolsCollectorManagerInterface &m_symbolsCollectorManager;
SymbolStorageInterface &m_symbolStorage;
Sqlite::TransactionInterface &m_transactionInterface;
SymbolIndexerTaskQueueInterface &m_symbolIndexerTaskQueue;
int m_hardware_concurrency;
uint m_hardware_concurrency;
std::launch m_launchPolicy;
bool m_isDisabled = false;
};
} // namespace ClangBackEnd

60
src/tools/clangrefactoringbackend/source/symbolindexertaskschedulerinterface.h Normal file
View File

@@ -0,0 +1,60 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include <functional>
#include <vector>
namespace Sqlite {
class TransactionInterface;
}
namespace ClangBackEnd {
class SymbolsCollectorInterface;
class SymbolStorageInterface;
using uint = unsigned int;
class SymbolIndexerTaskSchedulerInterface
{
public:
using Task = std::function<void(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage,
Sqlite::TransactionInterface &transaction)>;
SymbolIndexerTaskSchedulerInterface() = default;
SymbolIndexerTaskSchedulerInterface(const SymbolIndexerTaskSchedulerInterface &) = delete;
SymbolIndexerTaskSchedulerInterface &operator=(const SymbolIndexerTaskSchedulerInterface &) = delete;
virtual void addTasks(std::vector<Task> &&tasks) = 0;
virtual uint freeSlots() = 0;
protected:
~SymbolIndexerTaskSchedulerInterface() = default;
};
} // namespace ClangBackEnd

29
src/tools/clangrefactoringbackend/source/symbolindexing.h
View File

@@ -30,6 +30,9 @@
#include "storagesqlitestatementfactory.h"
#include "symbolindexer.h"
#include "symbolscollector.h"
#include "symbolscollectormanager.h"
#include "symbolindexertaskqueue.h"
#include "symbolindexertaskscheduler.h"
#include "symbolstorage.h"
#include <refactoringdatabaseinitializer.h>
@@ -41,6 +44,8 @@
#include <QFileSystemWatcher>
#include <thread>
namespace ClangBackEnd {
class SymbolIndexing final : public SymbolIndexingInterface
@@ -52,26 +57,44 @@ public:
SymbolIndexing(Sqlite::Database &database,
FilePathCachingInterface &filePathCache)
: m_filePathCache(filePathCache),
m_statementFactory(database)
m_statementFactory(database),
m_collectorManger(database),
m_indexerScheduler(m_collectorManger, m_symbolStorage, database, m_indexerQueue, std::thread::hardware_concurrency())
{
}
~SymbolIndexing()
{
syncTasks();
}
SymbolIndexer &indexer()
{
return m_indexer;
}
void syncTasks()
{
m_indexerScheduler.disable();
while (!m_indexerScheduler.futures().empty()) {
m_indexerScheduler.syncTasks();
m_indexerScheduler.freeSlots();
}
}
void updateProjectParts(V2::ProjectPartContainers &&projectParts,
const V2::FileContainers &generatedFiles) override;
private:
FilePathCachingInterface &m_filePathCache;
SymbolsCollector m_collector{m_filePathCache};
StatementFactory m_statementFactory;
Storage m_symbolStorage{m_statementFactory};
ClangPathWatcher<QFileSystemWatcher, QTimer> m_sourceWatcher{m_filePathCache};
FileStatusCache m_fileStatusCache{m_filePathCache};
SymbolIndexer m_indexer{m_collector,
SymbolsCollectorManager<SymbolsCollector> m_collectorManger;
SymbolIndexerTaskScheduler m_indexerScheduler;
SymbolIndexerTaskQueue m_indexerQueue{m_indexerScheduler};
SymbolIndexer m_indexer{m_indexerQueue,
m_symbolStorage,
m_sourceWatcher,
m_filePathCache,

13
src/tools/clangrefactoringbackend/source/symbolscollector.cpp
View File

@@ -29,11 +29,11 @@
namespace ClangBackEnd {
SymbolsCollector::SymbolsCollector(FilePathCachingInterface &filePathCache)
: m_indexDataConsumer(std::make_shared<IndexDataConsumer>(m_symbolEntries, m_sourceLocationEntries, filePathCache, m_sourcesManager)),
SymbolsCollector::SymbolsCollector(Sqlite::Database &database)
: m_filePathCache(database),
m_indexDataConsumer(std::make_shared<IndexDataConsumer>(m_symbolEntries, m_sourceLocationEntries, m_filePathCache, m_sourcesManager)),
m_collectSymbolsAction(m_indexDataConsumer),
m_collectMacrosSourceFileCallbacks(m_symbolEntries, m_sourceLocationEntries, filePathCache, m_sourcesManager),
m_filePathCache(filePathCache)
m_collectMacrosSourceFileCallbacks(m_symbolEntries, m_sourceLocationEntries, m_filePathCache, m_sourcesManager)
{
}
@@ -44,6 +44,11 @@ void SymbolsCollector::addFiles(const FilePathIds &filePathIds,
m_collectMacrosSourceFileCallbacks.addSourceFiles(filePathIds);
}
void SymbolsCollector::addFile(FilePathId filePathId, const Utils::SmallStringVector &arguments)
{
addFiles({filePathId}, arguments);
}
void SymbolsCollector::addUnsavedFiles(const V2::FileContainers &unsavedFiles)
{
m_clangTool.addUnsavedFiles(unsavedFiles);

13
src/tools/clangrefactoringbackend/source/symbolscollector.h
View File

@@ -31,17 +31,22 @@
#include "sourcesmanager.h"
#include "symbolscollectorinterface.h"
#include <filepathcachingfwd.h>
#include <filepathcaching.h>
namespace Sqlite {
class Database;
}
namespace ClangBackEnd {
class SymbolsCollector final : public SymbolsCollectorInterface
{
public:
SymbolsCollector(FilePathCachingInterface &filePathCache);
SymbolsCollector(Sqlite::Database &database);
void addFiles(const FilePathIds &filePathIds,
const Utils::SmallStringVector &arguments) override;
const Utils::SmallStringVector &arguments);
void addFile(FilePathId filePathId, const Utils::SmallStringVector &arguments) override;
void addUnsavedFiles(const V2::FileContainers &unsavedFiles) override;
@@ -60,6 +65,7 @@ public:
void setIsUsed(bool isUsed) override;
private:
FilePathCaching m_filePathCache;
ClangTool m_clangTool;
SymbolEntries m_symbolEntries;
SourceLocationEntries m_sourceLocationEntries;
@@ -67,7 +73,6 @@ private:
CollectSymbolsAction m_collectSymbolsAction;
CollectMacrosSourceFileCallbacks m_collectMacrosSourceFileCallbacks;
SourcesManager m_sourcesManager;
FilePathCachingInterface &m_filePathCache;
bool m_isUsed = false;
};

7
src/tools/clangrefactoringbackend/source/symbolscollectorinterface.h
View File

@@ -44,11 +44,11 @@ class SymbolsCollectorInterface
{
public:
SymbolsCollectorInterface() = default;
virtual ~SymbolsCollectorInterface() = default;
SymbolsCollectorInterface(const SymbolsCollectorInterface &) = delete;
SymbolsCollectorInterface &operator=(const SymbolsCollectorInterface &) = delete;
virtual void addFiles(const FilePathIds &filePathIds,
const Utils::SmallStringVector &arguments) = 0;
virtual void addFile(FilePathId filePathId, const Utils::SmallStringVector &arguments) = 0;
virtual void addUnsavedFiles(const V2::FileContainers &unsavedFiles) = 0;
@@ -65,9 +65,6 @@ public:
virtual bool isUsed() const = 0;
virtual void setIsUsed(bool isUsed) = 0;
protected:
~SymbolsCollectorInterface() = default;
};
} // namespace ClangBackEnd

35
src/tools/clangrefactoringbackend/source/symbolscollectormanager.cpp Normal file
View File

@@ -0,0 +1,35 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "symbolscollectormanager.h"
#include "symbolscollector.h"
namespace ClangBackEnd {
} // namespace ClangBackEnd

83
src/tools/clangrefactoringbackend/source/symbolscollectormanager.h Normal file
View File

@@ -0,0 +1,83 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "symbolscollectormanagerinterface.h"
#include "symbolscollectorinterface.h"
#include <memory>
namespace Sqlite {
class Database;
}
namespace ClangBackEnd {
class SymbolsCollector;
template<typename SymbolsCollector>
class SymbolsCollectorManager final : public SymbolsCollectorManagerInterface
{
public:
SymbolsCollectorManager(Sqlite::Database &database)
: m_database(database)
{}
SymbolsCollector &unusedSymbolsCollector() override
{
auto split = std::partition(m_collectors.begin(),
m_collectors.end(),
[] (const auto &collector) {
return collector->isUsed();
});
auto freeCollectors = std::distance(split, m_collectors.end());
if (freeCollectors > 0)
return usedCollector(*split->get());
m_collectors.emplace_back(std::make_unique<SymbolsCollector>(m_database));
return usedCollector(*m_collectors.back().get());
}
const std::vector<std::unique_ptr<SymbolsCollector>> &collectors() const
{
return m_collectors;
}
private:
SymbolsCollector &usedCollector(SymbolsCollector &collector)
{
collector.setIsUsed(true);
return collector;
}
private:
std::vector<std::unique_ptr<SymbolsCollector>> m_collectors;
Sqlite::Database &m_database;
};
} // namespace ClangBackEnd

4
src/tools/clangrefactoringbackend/source/symbolscollectormanagerinterface.h
View File

@@ -34,6 +34,10 @@ class SymbolsCollectorInterface;
class SymbolsCollectorManagerInterface
{
public:
SymbolsCollectorManagerInterface() = default;
SymbolsCollectorManagerInterface(const SymbolsCollectorManagerInterface &) = delete;
SymbolsCollectorManagerInterface &operator=(const SymbolsCollectorManagerInterface &) = delete;
virtual SymbolsCollectorInterface &unusedSymbolsCollector() = 0;
protected:

4
src/tools/clangrefactoringbackend/source/symbolstorage.h
View File

@@ -64,7 +64,7 @@ public:
deleteNewLocationsTable();
}
void insertOrUpdateProjectPart(Utils::SmallStringView projectPartName,
int insertOrUpdateProjectPart(Utils::SmallStringView projectPartName,
const Utils::SmallStringVector &commandLineArguments,
const CompilerMacros &compilerMacros,
const Utils::SmallStringVector &includeSearchPaths) override
@@ -88,6 +88,8 @@ public:
includeSearchPathsAsJason,
projectPartName);
}
return int(m_statementFactory.database.lastInsertedRowId());
}
Utils::optional<ProjectPartArtefact> fetchProjectPartArtefact(FilePathId sourceId) const override

8
src/tools/clangrefactoringbackend/source/symbolstorageinterface.h
View File

@@ -49,10 +49,10 @@ public:
virtual void addSymbolsAndSourceLocations(const SymbolEntries &symbolEntries,
const SourceLocationEntries &sourceLocations) = 0;
virtual void insertOrUpdateProjectPart(Utils::SmallStringView projectPartName,
const Utils::SmallStringVector &commandLineArguments,
const CompilerMacros &compilerMacros,
const Utils::SmallStringVector &includeSearchPaths) = 0;
virtual int insertOrUpdateProjectPart(Utils::SmallStringView projectPartName,
const Utils::SmallStringVector &commandLineArguments,
const CompilerMacros &compilerMacros,
const Utils::SmallStringVector &includeSearchPaths) = 0;
virtual void updateProjectPartSources(Utils::SmallStringView projectPartName,
const FilePathIds &sourceFilePathIds) = 0;
virtual void updateProjectPartSources(int projectPartId,

1
tests/unit/unittest/google-using-declarations.h
View File

@@ -39,6 +39,7 @@ using testing::ElementsAre;
using testing::Field;
using testing::HasSubstr;
using testing::InSequence;
using testing::Invoke;
using testing::IsEmpty;
using testing::Matcher;
using testing::Mock;

4
tests/unit/unittest/mocksymbolindexertaskqueue.h
View File

@@ -32,5 +32,9 @@
class MockSymbolIndexerTaskQueue : public ClangBackEnd::SymbolIndexerTaskQueueInterface
{
public:
MOCK_METHOD1(addOrUpdateTasks,
void (std::vector< ClangBackEnd::SymbolIndexerTask> &&tasks));
MOCK_METHOD1(removeTasks,
void (const Utils::SmallStringVector &projectPartIds));
MOCK_METHOD0(processTasks, void());
};

44
tests/unit/unittest/mocksymbolindexertaskscheduler.h Normal file
View File

@@ -0,0 +1,44 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "googletest.h"
#include <symbolindexertaskschedulerinterface.h>
class MockSymbolIndexerTaskScheduler : public ClangBackEnd::SymbolIndexerTaskSchedulerInterface
{
public:
MOCK_METHOD1(addTasks,
void (const std::vector<ClangBackEnd::SymbolIndexerTaskSchedulerInterface::Task> &));
MOCK_METHOD0(freeSlots,
uint ());
void addTasks(std::vector<ClangBackEnd::SymbolIndexerTaskSchedulerInterface::Task> &&tasks) override
{
addTasks(tasks);
}
};

43
tests/unit/unittest/mocksymbolscollector.h
View File

@@ -29,14 +29,28 @@
#include <symbolscollectorinterface.h>
namespace Sqlite {
class Database;
}
class MockSymbolsCollector : public ClangBackEnd::SymbolsCollectorInterface
{
public:
MockSymbolsCollector()
{
}
MockSymbolsCollector(const Sqlite::Database &)
{
ON_CALL(*this, setIsUsed(_)).WillByDefault(Invoke(this, &MockSymbolsCollector::setIsUsed2));
ON_CALL(*this, isUsed()).WillByDefault(Invoke(this, &MockSymbolsCollector::isUsed2));
}
MOCK_METHOD0(collectSymbols,
void());
MOCK_METHOD2(addFiles,
void(const ClangBackEnd::FilePathIds &filePathIds,
MOCK_METHOD2(addFile,
void(ClangBackEnd::FilePathId filePathId,
const Utils::SmallStringVector &arguments));
MOCK_METHOD1(addUnsavedFiles,
@@ -60,12 +74,25 @@ public:
MOCK_CONST_METHOD0(fileStatuses,
const ClangBackEnd::FileStatuses &());
MOCK_CONST_METHOD0(sourceDependencies,
const ClangBackEnd::SourceDependencies &());
MOCK_CONST_METHOD0(sourceDependencies,
const ClangBackEnd::SourceDependencies &());
MOCK_CONST_METHOD0(isUsed,
bool());
MOCK_CONST_METHOD0(isUsed,
bool());
MOCK_METHOD1(setIsUsed,
void(bool));
MOCK_METHOD1(setIsUsed,
void(bool));
void setIsUsed2(bool isUsed)
{
used = isUsed;
}
bool isUsed2() const
{
return used;
}
public:
bool used = false;
};

8
tests/unit/unittest/mocksymbolstorage.h
View File

@@ -38,10 +38,10 @@ public:
void(const ClangBackEnd::SymbolEntries &symbolEentries,
const ClangBackEnd::SourceLocationEntries &sourceLocations));
MOCK_METHOD4(insertOrUpdateProjectPart,
void(Utils::SmallStringView projectPartName,
const Utils::SmallStringVector &commandLineArgument,
const ClangBackEnd::CompilerMacros &compilerMacros,
const Utils::SmallStringVector &includeSearchPaths));
int(Utils::SmallStringView projectPartName,
const Utils::SmallStringVector &commandLineArgument,
const ClangBackEnd::CompilerMacros &compilerMacros,
const Utils::SmallStringVector &includeSearchPaths));
MOCK_METHOD2(updateProjectPartSources,
void(Utils::SmallStringView projectPartName,
const ClangBackEnd::FilePathIds &sourceFilePathIds));

130
tests/unit/unittest/symbolindexer-test.cpp
View File

@@ -34,9 +34,13 @@
#include <filestatuscache.h>
#include <projectpartcontainerv2.h>
#include <refactoringdatabaseinitializer.h>
#include <symbolscollectormanager.h>
#include <symbolindexer.h>
#include <symbolindexertaskqueue.h>
#include <symbolindexertaskscheduler.h>
#include <updateprojectpartsmessage.h>
#include <QCoreApplication>
#include <QDateTime>
#include <fstream>
@@ -54,6 +58,10 @@ using ClangBackEnd::V2::ProjectPartContainers;
using ClangBackEnd::V2::FileContainers;
using ClangBackEnd::SymbolEntries;
using ClangBackEnd::SymbolEntry;
using ClangBackEnd::SymbolIndexerTask;
using ClangBackEnd::SymbolIndexerTaskQueue;
using ClangBackEnd::SymbolIndexerTaskScheduler;
using ClangBackEnd::SymbolsCollectorManager;
using ClangBackEnd::SourceDependencies;
using ClangBackEnd::SourceLocationEntries;
using ClangBackEnd::SourceLocationEntry;
@@ -89,6 +97,21 @@ protected:
ON_CALL(mockCollector, sourceDependencies()).WillByDefault(ReturnRef(sourceDependencies));
ON_CALL(mockStorage, fetchProjectPartArtefact(A<FilePathId>())).WillByDefault(Return(artefact));
ON_CALL(mockStorage, fetchLowestLastModifiedTime(A<FilePathId>())).WillByDefault(Return(QDateTime::currentSecsSinceEpoch()));
mockCollector.setIsUsed(false);
}
void TearDown()
{
syncTasks();
}
void syncTasks()
{
while (!indexerQueue.tasks().empty() || !indexerScheduler.futures().empty()) {
indexerScheduler.syncTasks();
QCoreApplication::processEvents();
}
}
static void SetUpTestCase()
@@ -153,23 +176,26 @@ protected:
ClangBackEnd::ProjectPartArtefact emptyArtefact{"", "", "", 74};
ClangBackEnd::ProjectPartPch projectPartPch{"/path/to/pch", 4};
NiceMock<MockSqliteTransactionBackend> mockSqliteTransactionBackend;
NiceMock<MockSymbolsCollector> mockCollector;
NiceMock<MockSymbolStorage> mockStorage;
NiceMock<MockClangPathWatcher> mockPathWatcher;
ClangBackEnd::FileStatusCache fileStatusCache{filePathCache};
ClangBackEnd::SymbolIndexer indexer{mockCollector,
SymbolsCollectorManager<NiceMock<MockSymbolsCollector>> collectorManger{data->database};
SymbolIndexerTaskScheduler indexerScheduler{collectorManger, mockStorage, mockSqliteTransactionBackend, indexerQueue, 1};
SymbolIndexerTaskQueue indexerQueue{indexerScheduler};
ClangBackEnd::SymbolIndexer indexer{indexerQueue,
mockStorage,
mockPathWatcher,
filePathCache,
fileStatusCache,
mockSqliteTransactionBackend};
NiceMock<MockSymbolsCollector> &mockCollector{collectorManger.unusedSymbolsCollector()};
};
std::unique_ptr<Data> SymbolIndexer::data;
TEST_F(SymbolIndexer, UpdateProjectPartsCallsAddFilesInCollector)
{
EXPECT_CALL(mockCollector, addFiles(projectPart1.sourcePathIds, projectPart1.arguments));
EXPECT_CALL(mockCollector, addFile(main1PathId, projectPart1.arguments));
indexer.updateProjectParts({projectPart1}, Utils::clone(unsaved));
}
@@ -179,14 +205,14 @@ TEST_F(SymbolIndexer, UpdateProjectPartsCallsAddFilesWithPrecompiledHeaderInColl
ON_CALL(mockStorage, fetchProjectPartArtefact(TypedEq<Utils::SmallStringView>(projectPart1.projectPartId))).WillByDefault(Return(emptyArtefact));
ON_CALL(mockStorage, fetchPrecompiledHeader(Eq(artefact.projectPartId))).WillByDefault(Return(projectPartPch));
EXPECT_CALL(mockCollector, addFiles(projectPart1.sourcePathIds,
ElementsAre(Eq("-I"),
Eq(TESTDATA_DIR),
Eq("-Wno-pragma-once-outside-header"),
Eq("-Xclang"),
Eq("-include-pch"),
Eq("-Xclang"),
Eq("/path/to/pch"))));
EXPECT_CALL(mockCollector, addFile(main1PathId,
ElementsAre(Eq("-I"),
Eq(TESTDATA_DIR),
Eq("-Wno-pragma-once-outside-header"),
Eq("-Xclang"),
Eq("-include-pch"),
Eq("-Xclang"),
Eq("/path/to/pch"))));
indexer.updateProjectParts({projectPart1}, Utils::clone(unsaved));
}
@@ -195,10 +221,10 @@ TEST_F(SymbolIndexer, UpdateProjectPartsCallsAddFilesWithoutPrecompiledHeaderInC
{
ON_CALL(mockStorage, fetchProjectPartArtefact(TypedEq<Utils::SmallStringView>(projectPart1.projectPartId))).WillByDefault(Return(emptyArtefact));
EXPECT_CALL(mockCollector, addFiles(projectPart1.sourcePathIds,
ElementsAre(Eq("-I"),
Eq(TESTDATA_DIR),
Eq("-Wno-pragma-once-outside-header"))));
EXPECT_CALL(mockCollector, addFile(main1PathId,
ElementsAre(Eq("-I"),
Eq(TESTDATA_DIR),
Eq("-Wno-pragma-once-outside-header"))));
indexer.updateProjectParts({projectPart1}, Utils::clone(unsaved));
}
@@ -212,14 +238,14 @@ TEST_F(SymbolIndexer, UpdateProjectPartsCallsClearInCollector)
TEST_F(SymbolIndexer, UpdateProjectPartsCallsAddFilesInCollectorForEveryProjectPart)
{
EXPECT_CALL(mockCollector, addFiles(_, _)).Times(2);
EXPECT_CALL(mockCollector, addFile(_, _)).Times(2);
indexer.updateProjectParts({projectPart1, projectPart2}, Utils::clone(unsaved));
}
TEST_F(SymbolIndexer, UpdateProjectPartsDoesNotCallAddFilesInCollectorForEmptyEveryProjectParts)
{
EXPECT_CALL(mockCollector, addFiles(_, _))
EXPECT_CALL(mockCollector, addFile(_, _))
.Times(0);
indexer.updateProjectParts({}, {});
@@ -255,9 +281,7 @@ TEST_F(SymbolIndexer, UpdateProjectPartsCallsAddUnsavedFilesInCollector)
TEST_F(SymbolIndexer, UpdateProjectPartsCallsAddSymbolsAndSourceLocationsInStorage)
{
EXPECT_CALL(mockSqliteTransactionBackend, immediateBegin()).Times(2);
EXPECT_CALL(mockStorage, addSymbolsAndSourceLocations(symbolEntries, sourceLocations)).Times(2);
EXPECT_CALL(mockSqliteTransactionBackend, commit()).Times(2);
indexer.updateProjectParts({projectPart1, projectPart2}, Utils::clone(unsaved));
}
@@ -320,14 +344,15 @@ TEST_F(SymbolIndexer, UpdateProjectPartsCallsInOrder)
{
InSequence s;
EXPECT_CALL(mockCollector, clear());
EXPECT_CALL(mockSqliteTransactionBackend, immediateBegin());
EXPECT_CALL(mockStorage, fetchProjectPartArtefact(TypedEq<Utils::SmallStringView>(projectPart1.projectPartId)));
EXPECT_CALL(mockCollector, addFiles(_, _));
EXPECT_CALL(mockStorage, insertOrUpdateProjectPart(Eq(projectPart1.projectPartId), Eq(projectPart1.arguments), Eq(projectPart1.compilerMacros), Eq(projectPart1.includeSearchPaths)));
EXPECT_CALL(mockSqliteTransactionBackend, commit());
EXPECT_CALL(mockCollector, addFile(main1PathId, projectPart1.arguments));
EXPECT_CALL(mockCollector, addUnsavedFiles(unsaved));
EXPECT_CALL(mockCollector, collectSymbols());
EXPECT_CALL(mockSqliteTransactionBackend, immediateBegin());
EXPECT_CALL(mockStorage, addSymbolsAndSourceLocations(symbolEntries, sourceLocations));
EXPECT_CALL(mockStorage, insertOrUpdateProjectPart(Eq(projectPart1.projectPartId), Eq(projectPart1.arguments), Eq(projectPart1.compilerMacros), Eq(projectPart1.includeSearchPaths)));
EXPECT_CALL(mockStorage, updateProjectPartSources(TypedEq<Utils::SmallStringView>(projectPart1.projectPartId), Eq(sourceFileIds)));
EXPECT_CALL(mockStorage, insertOrUpdateUsedMacros(Eq(usedMacros)));
EXPECT_CALL(mockStorage, insertFileStatuses(Eq(fileStatus)));
@@ -341,7 +366,7 @@ TEST_F(SymbolIndexer, CallSetNotifier)
{
EXPECT_CALL(mockPathWatcher, setNotifier(_));
ClangBackEnd::SymbolIndexer indexer{mockCollector, mockStorage, mockPathWatcher, filePathCache, fileStatusCache, mockSqliteTransactionBackend};
ClangBackEnd::SymbolIndexer indexer{indexerQueue, mockStorage, mockPathWatcher, filePathCache, fileStatusCache, mockSqliteTransactionBackend};
}
TEST_F(SymbolIndexer, PathChangedCallsFetchProjectPartArtefactInStorage)
@@ -356,9 +381,10 @@ TEST_F(SymbolIndexer, UpdateChangedPathCallsInOrder)
{
InSequence s;
EXPECT_CALL(mockCollector, clear());
EXPECT_CALL(mockStorage, fetchProjectPartArtefact(sourceFileIds[0]));
EXPECT_CALL(mockCollector, addFiles(ElementsAre(sourceFileIds[0]), Eq(artefact.compilerArguments)));
EXPECT_CALL(mockSqliteTransactionBackend, immediateBegin()).Times(0);
EXPECT_CALL(mockStorage, fetchProjectPartArtefact(sourceFileIds[0])).WillOnce(Return(artefact));
EXPECT_CALL(mockSqliteTransactionBackend, commit());
EXPECT_CALL(mockCollector, addFile(Eq(sourceFileIds[0]), Eq(artefact.compilerArguments)));
EXPECT_CALL(mockCollector, collectSymbols());
EXPECT_CALL(mockSqliteTransactionBackend, immediateBegin());
EXPECT_CALL(mockStorage, addSymbolsAndSourceLocations(symbolEntries, sourceLocations));
@@ -368,17 +394,17 @@ TEST_F(SymbolIndexer, UpdateChangedPathCallsInOrder)
EXPECT_CALL(mockStorage, insertOrUpdateSourceDependencies(Eq(sourceDependencies)));
EXPECT_CALL(mockSqliteTransactionBackend, commit());
indexer.updateChangedPath(sourceFileIds[0]);
indexer.pathsChanged({sourceFileIds[0]});
}
TEST_F(SymbolIndexer, HandleEmptyOptionalArtifactInUpdateChangedPath)
{
ON_CALL(mockStorage, fetchProjectPartArtefact(TypedEq<FilePathId>(sourceFileIds[0])))
.WillByDefault(Return(emptyArtefact));
InSequence s;
EXPECT_CALL(mockCollector, clear());
EXPECT_CALL(mockStorage, fetchProjectPartArtefact(sourceFileIds[0]));
EXPECT_CALL(mockCollector, addFiles(_, _)).Times(0);
EXPECT_CALL(mockSqliteTransactionBackend, immediateBegin()).Times(0);
EXPECT_CALL(mockStorage, fetchProjectPartArtefact(sourceFileIds[0])).WillOnce(Return(emptyArtefact));
EXPECT_CALL(mockSqliteTransactionBackend, commit());
EXPECT_CALL(mockCollector, addFile(_, _)).Times(0);
EXPECT_CALL(mockCollector, collectSymbols()).Times(0);
EXPECT_CALL(mockSqliteTransactionBackend, immediateBegin()).Times(0);
EXPECT_CALL(mockStorage, addSymbolsAndSourceLocations(_, _)).Times(0);
@@ -388,7 +414,7 @@ TEST_F(SymbolIndexer, HandleEmptyOptionalArtifactInUpdateChangedPath)
EXPECT_CALL(mockStorage, insertOrUpdateSourceDependencies(_)).Times(0);
EXPECT_CALL(mockSqliteTransactionBackend, commit()).Times(0);
indexer.updateChangedPath(sourceFileIds[0]);
indexer.pathsChanged({sourceFileIds[0]});
}
TEST_F(SymbolIndexer, UpdateChangedPathIsUsingPrecompiledHeader)
@@ -397,26 +423,28 @@ TEST_F(SymbolIndexer, UpdateChangedPathIsUsingPrecompiledHeader)
.WillByDefault(Return(artefact));
ON_CALL(mockStorage, fetchPrecompiledHeader(Eq(artefact.projectPartId)))
.WillByDefault(Return(projectPartPch));
std::vector<SymbolIndexerTask> symbolIndexerTask;
EXPECT_CALL(mockCollector, addFiles(projectPart1.sourcePathIds,
ElementsAre(Eq("-DFOO"),
Eq("-Xclang"),
Eq("-include-pch"),
Eq("-Xclang"),
Eq("/path/to/pch"))));
EXPECT_CALL(mockCollector, addFile(Eq(sourceFileIds[0]),
ElementsAre(Eq("-DFOO"),
Eq("-Xclang"),
Eq("-include-pch"),
Eq("-Xclang"),
Eq("/path/to/pch"))));
indexer.updateChangedPath(sourceFileIds[0]);
indexer.pathsChanged({sourceFileIds[0]});
}
TEST_F(SymbolIndexer, UpdateChangedPathIsNotUsingPrecompiledHeaderIfItNotExists)
{
ON_CALL(mockStorage, fetchProjectPartArtefact(TypedEq<FilePathId>(sourceFileIds[0])))
.WillByDefault(Return(artefact));
std::vector<SymbolIndexerTask> symbolIndexerTask;
EXPECT_CALL(mockCollector, addFiles(projectPart1.sourcePathIds,
ElementsAre(Eq("-DFOO"))));
EXPECT_CALL(mockCollector, addFile(Eq(sourceFileIds[0]),
ElementsAre(Eq("-DFOO"))));
indexer.updateChangedPath(sourceFileIds[0]);
indexer.pathsChanged({sourceFileIds[0]});
}
@@ -458,11 +486,13 @@ TEST_F(SymbolIndexer, IncludeSearchPathsAreDifferent)
TEST_F(SymbolIndexer, DontReparseInUpdateProjectPartsIfDefinesAreTheSame)
{
ON_CALL(mockStorage, fetchProjectPartArtefact(An<Utils::SmallStringView>())).WillByDefault(Return(artefact));
InSequence s;
EXPECT_CALL(mockCollector, clear());
EXPECT_CALL(mockStorage, fetchProjectPartArtefact(TypedEq<Utils::SmallStringView>(projectPart1.projectPartId)));
EXPECT_CALL(mockCollector, addFiles(_, _)).Times(0);
EXPECT_CALL(mockSqliteTransactionBackend, immediateBegin());
EXPECT_CALL(mockStorage, fetchProjectPartArtefact(TypedEq<Utils::SmallStringView>(projectPart1.projectPartId))).WillRepeatedly(Return(artefact));
EXPECT_CALL(mockStorage, insertOrUpdateProjectPart(Eq(projectPart1.projectPartId), Eq(projectPart1.arguments), Eq(projectPart1.compilerMacros), Eq(projectPart1.includeSearchPaths)));
EXPECT_CALL(mockSqliteTransactionBackend, commit());
EXPECT_CALL(mockCollector, addFile(_, _)).Times(0);
EXPECT_CALL(mockCollector, collectSymbols()).Times(0);
EXPECT_CALL(mockSqliteTransactionBackend, immediateBegin()).Times(0);
EXPECT_CALL(mockStorage, addSymbolsAndSourceLocations(_, _)).Times(0);
@@ -540,11 +570,12 @@ TEST_F(SymbolIndexer, UpToDateFilesDontPassFilteredUpdatableFilePathIds)
TEST_F(SymbolIndexer, OutdatedFilesAreParsedInUpdateProjectParts)
{
indexer.pathsChanged({main1PathId});
indexerScheduler.syncTasks();
ON_CALL(mockStorage, fetchProjectPartArtefact(An<Utils::SmallStringView>())).WillByDefault(Return(artefact));
ON_CALL(mockStorage, fetchLowestLastModifiedTime(A<FilePathId>()))
.WillByDefault(Return(0));
EXPECT_CALL(mockCollector, addFiles(ElementsAre(main1PathId), _));
EXPECT_CALL(mockCollector, addFile(Eq(main1PathId), _));
indexer.updateProjectParts({projectPart1}, {});
}
@@ -552,11 +583,12 @@ TEST_F(SymbolIndexer, OutdatedFilesAreParsedInUpdateProjectParts)
TEST_F(SymbolIndexer, UpToDateFilesAreNotParsedInUpdateProjectParts)
{
indexer.pathsChanged({main1PathId});
indexerScheduler.syncTasks();
ON_CALL(mockStorage, fetchProjectPartArtefact(An<Utils::SmallStringView>())).WillByDefault(Return(artefact));
ON_CALL(mockStorage, fetchLowestLastModifiedTime(A<FilePathId>()))
.WillByDefault(Return(QDateTime::currentSecsSinceEpoch()));
EXPECT_CALL(mockCollector, addFiles(_, _)).Times(0);
EXPECT_CALL(mockCollector, addFile(_, _)).Times(0);
indexer.updateProjectParts({projectPart1}, {});
}

57
tests/unit/unittest/symbolindexertaskqueue-test.cpp
View File

@@ -25,6 +25,8 @@
#include "googletest.h"
#include "mocksymbolindexertaskscheduler.h"
#include <symbolindexertaskqueue.h>
namespace {
@@ -48,12 +50,13 @@ MATCHER_P2(IsTask, filePathId, projectPartId,
class SymbolIndexerTaskQueue : public testing::Test
{
protected:
std::size_t projectPartId(const Utils::SmallString &projectPartId)
int projectPartId(const Utils::SmallString &projectPartId)
{
return queue.projectPartNumberId(projectPartId);
return int(queue.projectPartNumberId(projectPartId));
}
protected:
ClangBackEnd::SymbolIndexerTaskQueue queue;
NiceMock<MockSymbolIndexerTaskScheduler> mockSymbolIndexerTaskScheduler;
ClangBackEnd::SymbolIndexerTaskQueue queue{mockSymbolIndexerTaskScheduler};
};
TEST_F(SymbolIndexerTaskQueue, AddTasks)
@@ -162,4 +165,52 @@ TEST_F(SymbolIndexerTaskQueue, GetProjectPartIds)
ASSERT_THAT(ids , ElementsAre(0, 2, 3));
}
TEST_F(SymbolIndexerTaskQueue, ProcessTasksCallsFreeSlotsAndAddTasksInScheduler)
{
InSequence s;
queue.addOrUpdateTasks({{{1, 1}, projectPartId("yi"), Callable{}},
{{1, 3}, projectPartId("yi"), Callable{}},
{{1, 5}, projectPartId("yi"), Callable{}}});
EXPECT_CALL(mockSymbolIndexerTaskScheduler, freeSlots()).WillRepeatedly(Return(2));
EXPECT_CALL(mockSymbolIndexerTaskScheduler, addTasks(SizeIs(2)));
queue.processTasks();
}
TEST_F(SymbolIndexerTaskQueue, ProcessTasksCallsFreeSlotsAndAddTasksWithNoTaskInSchedulerIfTaskAreEmpty)
{
InSequence s;
EXPECT_CALL(mockSymbolIndexerTaskScheduler, freeSlots()).WillRepeatedly(Return(2));
EXPECT_CALL(mockSymbolIndexerTaskScheduler, addTasks(IsEmpty()));
queue.processTasks();
}
TEST_F(SymbolIndexerTaskQueue, ProcessTasksCallsFreeSlotsAndMoveAllTasksInSchedulerIfMoreSlotsAreFree)
{
InSequence s;
queue.addOrUpdateTasks({{{1, 1}, projectPartId("yi"), Callable{}},
{{1, 3}, projectPartId("yi"), Callable{}},
{{1, 5}, projectPartId("yi"), Callable{}}});
EXPECT_CALL(mockSymbolIndexerTaskScheduler, freeSlots()).WillRepeatedly(Return(4));
EXPECT_CALL(mockSymbolIndexerTaskScheduler, addTasks(SizeIs(3)));
queue.processTasks();
}
TEST_F(SymbolIndexerTaskQueue, ProcessTasksRemovesProcessedTasks)
{
queue.addOrUpdateTasks({{{1, 1}, projectPartId("yi"), Callable{}},
{{1, 3}, projectPartId("yi"), Callable{}},
{{1, 5}, projectPartId("yi"), Callable{}}});
ON_CALL(mockSymbolIndexerTaskScheduler, freeSlots()).WillByDefault(Return(2));
queue.processTasks();
ASSERT_THAT(queue.tasks(), SizeIs(1));
}
}

42
tests/unit/unittest/symbolindexertaskscheduler-test.cpp
View File

@@ -29,6 +29,7 @@
#include "mocksymbolscollectormanager.h"
#include "mocksymbolscollector.h"
#include "mocksymbolstorage.h"
#include "mocksqlitetransactionbackend.h"
#include <symbolindexertaskscheduler.h>
@@ -55,18 +56,27 @@ protected:
protected:
MockFunction<void()> mock;
ClangBackEnd::SymbolIndexerTaskScheduler::Task call{
[&] (SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage) {
[&] (SymbolsCollectorInterface &, SymbolStorageInterface &, Sqlite::TransactionInterface &) {
mock.Call(); }};
ClangBackEnd::SymbolIndexerTaskScheduler::Task nocall{
[&] (SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage) {}};
[&] (SymbolsCollectorInterface &, SymbolStorageInterface &, Sqlite::TransactionInterface &) {
}};
NiceMock<MockSymbolsCollectorManager> mockSymbolsCollectorManager;
NiceMock<MockSymbolsCollector> mockSymbolsCollector;
MockSymbolStorage mockSymbolStorage;
NiceMock<MockSymbolIndexerTaskQueue> mockSymbolIndexerTaskQueue;
ClangBackEnd::SymbolIndexerTaskScheduler scheduler{mockSymbolsCollectorManager, mockSymbolStorage, mockSymbolIndexerTaskQueue, 4};
ClangBackEnd::SymbolIndexerTaskScheduler deferedScheduler{mockSymbolsCollectorManager, mockSymbolStorage, mockSymbolIndexerTaskQueue, 4, std::launch::deferred};
MockSqliteTransactionBackend mockSqliteTransactionBackend;
ClangBackEnd::SymbolIndexerTaskScheduler scheduler{mockSymbolsCollectorManager,
mockSymbolStorage,
mockSqliteTransactionBackend,
mockSymbolIndexerTaskQueue,
4};
ClangBackEnd::SymbolIndexerTaskScheduler deferedScheduler{mockSymbolsCollectorManager,
mockSymbolStorage,
mockSqliteTransactionBackend,
mockSymbolIndexerTaskQueue,
4,
std::launch::deferred};
};
TEST_F(SymbolIndexerTaskScheduler, AddTasks)
@@ -123,6 +133,7 @@ TEST_F(SymbolIndexerTaskScheduler, NoFuturesAfterFreeSlots)
TEST_F(SymbolIndexerTaskScheduler, FreeSlotsCallSymbolsCollectorSetIsUnused)
{
InSequence s;
scheduler.addTasks({nocall, nocall});
scheduler.syncTasks();
@@ -131,6 +142,17 @@ TEST_F(SymbolIndexerTaskScheduler, FreeSlotsCallSymbolsCollectorSetIsUnused)
scheduler.freeSlots();
}
TEST_F(SymbolIndexerTaskScheduler, FreeSlotsCallClearsSymbolsCollector)
{
InSequence s;
scheduler.addTasks({nocall, nocall});
scheduler.syncTasks();
EXPECT_CALL(mockSymbolsCollector, clear()).Times(2);
scheduler.freeSlots();
}
TEST_F(SymbolIndexerTaskScheduler, AddTaskCallSymbolsCollectorManagerUnusedSymbolsCollector)
{
EXPECT_CALL(mockSymbolsCollectorManager, unusedSymbolsCollector()).Times(2);
@@ -140,8 +162,12 @@ TEST_F(SymbolIndexerTaskScheduler, AddTaskCallSymbolsCollectorManagerUnusedSymbo
TEST_F(SymbolIndexerTaskScheduler, CallProcessTasksInQueueAfterFinishedTasks)
{
EXPECT_CALL(mockSymbolIndexerTaskQueue, processTasks()).Times(2);
InSequence s;
scheduler.addTasks({nocall, nocall});
EXPECT_CALL(mock, Call());
EXPECT_CALL(mockSymbolIndexerTaskQueue, processTasks());
scheduler.addTasks({call});
scheduler.syncTasks();
}
}

4
tests/unit/unittest/symbolindexing-test.cpp
View File

@@ -96,6 +96,7 @@ protected:
TEST_F(SymbolIndexing, Locations)
{
indexing.indexer().updateProjectParts({projectPart1}, {});
indexing.syncTasks();
auto locations = query.locationsAt(filePathId(TESTDATA_DIR "/symbolindexing_main1.cpp"), 1, 6);
ASSERT_THAT(locations,
@@ -108,6 +109,7 @@ TEST_F(SymbolIndexing, Locations)
TEST_F(SymbolIndexing, DISABLED_TemplateFunction)
{
indexing.indexer().updateProjectParts({projectPart1}, {});
indexing.syncTasks();
auto locations = query.locationsAt(filePathId(TESTDATA_DIR "/symbolindexing_main1.cpp"), 21, 24);
ASSERT_THAT(locations,
@@ -122,6 +124,8 @@ TEST_F(SymbolIndexing, PathsAreUpdated)
indexing.indexer().pathsChanged({filePathId(main1Path)});
indexing.indexer().pathsChanged({filePathId(main1Path)});
indexing.syncTasks();
auto locations = query.locationsAt(filePathId(TESTDATA_DIR "/symbolindexing_main1.cpp"), 1, 6);
ASSERT_THAT(locations,
ElementsAre(

2
tests/unit/unittest/symbolscollector-test.cpp
View File

@@ -173,7 +173,7 @@ protected:
Sqlite::Database database{":memory:", Sqlite::JournalMode::Memory};
ClangBackEnd::RefactoringDatabaseInitializer<Sqlite::Database> initializer{database};
FilePathCaching filePathCache{database};
ClangBackEnd::SymbolsCollector collector{filePathCache};
ClangBackEnd::SymbolsCollector collector{database};
};
TEST_F(SymbolsCollector, CollectSymbolName)

80
tests/unit/unittest/symbolscollectormanager-test.cpp Normal file
View File

@@ -0,0 +1,80 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "googletest.h"
#include "mocksymbolscollector.h"
#include <sqlitedatabase.h>
#include <refactoringdatabaseinitializer.h>
#include <symbolscollectormanager.h>
namespace {
class SymbolsCollectorManager : public testing::Test
{
protected:
Sqlite::Database database{":memory:", Sqlite::JournalMode::Memory};
ClangBackEnd::RefactoringDatabaseInitializer<Sqlite::Database> initializer{database};
ClangBackEnd::SymbolsCollectorManager<NiceMock<MockSymbolsCollector>> manager{database};
};
TEST_F(SymbolsCollectorManager, CreateUnsedSystemCollector)
{
manager.unusedSymbolsCollector();
manager.unusedSymbolsCollector();
ASSERT_THAT(manager.collectors(), SizeIs(2));
}
TEST_F(SymbolsCollectorManager, ReuseUnsedSystemCollector)
{
auto &collector = manager.unusedSymbolsCollector();
collector.setIsUsed(false);
manager.unusedSymbolsCollector();
ASSERT_THAT(manager.collectors(), SizeIs(1));
}
TEST_F(SymbolsCollectorManager, AsGetNewUnusedSymbolsCollectorItIsSetUsed)
{
auto &collector = manager.unusedSymbolsCollector();
ASSERT_TRUE(collector.isUsed());
}
TEST_F(SymbolsCollectorManager, AsGetReusedUnusedSymbolsCollectorItIsSetUsed)
{
auto &collector = manager.unusedSymbolsCollector();
collector.setIsUsed(false);
auto &collector2 = manager.unusedSymbolsCollector();
ASSERT_TRUE(collector2.isUsed());
}
}

3
tests/unit/unittest/symbolstorage-test.cpp
View File

@@ -194,7 +194,7 @@ TEST_F(SymbolStorage, InsertProjectPart)
TypedEq<Utils::SmallStringView>("[\"foo\"]"),
TypedEq<Utils::SmallStringView>("{\"FOO\":\"1\"}"),
TypedEq<Utils::SmallStringView>("[\"/includes\"]")));
EXPECT_CALL(mockDatabase, lastInsertedRowId());
EXPECT_CALL(mockDatabase, lastInsertedRowId()).Times(2);
storage.insertOrUpdateProjectPart("project", {"foo"}, {{"FOO", "1"}}, {"/includes"});
}
@@ -216,6 +216,7 @@ TEST_F(SymbolStorage, UpdateProjectPart)
TypedEq<Utils::SmallStringView>("{\"FOO\":\"1\"}"),
TypedEq<Utils::SmallStringView>("[\"/includes\"]"),
TypedEq<Utils::SmallStringView>("project")));
EXPECT_CALL(mockDatabase, lastInsertedRowId());
storage.insertOrUpdateProjectPart("project", {"foo"}, {{"FOO", "1"}}, {"/includes"});
}

9
tests/unit/unittest/unittest.pro
View File

@@ -97,7 +97,8 @@ SOURCES += \
generatedfiles-test.cpp \
sourcesmanager-test.cpp \
symbolindexertaskqueue-test.cpp \
symbolindexertaskscheduler-test.cpp
symbolindexertaskscheduler-test.cpp \
symbolscollectormanager-test.cpp
!isEmpty(LIBCLANG_LIBS) {
SOURCES += \
@@ -174,7 +175,7 @@ SOURCES += \
symbolindexing-test.cpp \
symbolscollector-test.cpp \
symbolfinder-test.cpp \
testclangtool.cpp \
testclangtool.cpp
}
exists($$GOOGLEBENCHMARK_DIR) {
@@ -235,7 +236,9 @@ HEADERS += \
mockprecompiledheaderstorage.h \
mockeditormanager.h \
mocksymbolscollectormanager.h \
mocksymbolindexertaskqueue.h
mocksymbolindexertaskqueue.h \
mocksymbolindexertaskscheduler.h
!isEmpty(LIBCLANG_LIBS) {
HEADERS += \
chunksreportedmonitor.h \