feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

Clang: Minimize reindexing

Browse Source 
We optimal indexer is only reindexing if the index would be changed. This
patch is a step in that direction. We only reindex now if the file or
project has changed. It fixes some typos too.

Task-number: QTCREATORBUG-21150
Change-Id: I6ea1c13282fbcd70253b9b2939aed37580dbd160
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>
This commit is contained in:
Marco Bubke
2019-03-21 17:55:24 +01:00
parent 7595c9f305
commit 56b01f7463
37 changed files with 736 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
src/tools/clangrefactoringbackend/source/symbolindexer.cpp
View File

@@ -66,7 +66,8 @@ SymbolIndexer::SymbolIndexer(SymbolIndexerTaskQueueInterface &symbolIndexerTaskQ
FilePathCachingInterface &filePathCache,
FileStatusCache &fileStatusCache,
Sqlite::TransactionInterface &transactionInterface,
ProjectPartsStorageInterface &projectPartsStorage)
ProjectPartsStorageInterface &projectPartsStorage,
ModifiedTimeCheckerInterface<SourceTimeStamps> &modifiedTimeChecker)
: m_symbolIndexerTaskQueue(symbolIndexerTaskQueue)
, m_symbolStorage(symbolStorage)
, m_buildDependencyStorage(buildDependenciesStorage)
@@ -76,6 +77,7 @@ SymbolIndexer::SymbolIndexer(SymbolIndexerTaskQueueInterface &symbolIndexerTaskQ
, m_fileStatusCache(fileStatusCache)
, m_transactionInterface(transactionInterface)
, m_projectPartsStorage(projectPartsStorage)
, m_modifiedTimeChecker(modifiedTimeChecker)
{
pathWatcher.setNotifier(this);
}
@@ -109,23 +111,27 @@ void SymbolIndexer::updateProjectPart(ProjectPartContainer &&projectPart)
std::vector<SymbolIndexerTask> symbolIndexerTask;
symbolIndexerTask.reserve(projectPart.sourcePathIds.size());
for (FilePathId sourcePathId : projectPart.sourcePathIds) {
auto indexing = [arguments = commandLineBuilder.commandLine,
sourcePathId,
this](SymbolsCollectorInterface &symbolsCollector) {
symbolsCollector.setFile(sourcePathId, arguments);
SourceTimeStamps dependentTimeStamps = m_symbolStorage.fetchIncludedIndexingTimeStamps(
sourcePathId);
bool success = symbolsCollector.collectSymbols();
if (!m_modifiedTimeChecker.isUpToDate(dependentTimeStamps)) {
auto indexing = [arguments = commandLineBuilder.commandLine, sourcePathId, this](
SymbolsCollectorInterface &symbolsCollector) {
symbolsCollector.setFile(sourcePathId, arguments);
if (success) {
Sqlite::ImmediateTransaction transaction{m_transactionInterface};
bool success = symbolsCollector.collectSymbols();
m_symbolStorage.addSymbolsAndSourceLocations(symbolsCollector.symbols(),
symbolsCollector.sourceLocations());
transaction.commit();
}
};
if (success) {
Sqlite::ImmediateTransaction transaction{m_transactionInterface};
m_symbolStorage.insertOrUpdateIndexingTimeStamps(symbolsCollector.fileStatuses());
m_symbolStorage.addSymbolsAndSourceLocations(symbolsCollector.symbols(),
symbolsCollector.sourceLocations());
transaction.commit();
}
};
symbolIndexerTask.emplace_back(sourcePathId, projectPartId, std::move(indexing));
symbolIndexerTask.emplace_back(sourcePathId, projectPartId, std::move(indexing));
}
}
m_symbolIndexerTaskQueue.addOrUpdateTasks(std::move(symbolIndexerTask));
@@ -136,11 +142,13 @@ void SymbolIndexer::pathsWithIdsChanged(const ProjectPartIds &) {}
void SymbolIndexer::pathsChanged(const FilePathIds &filePathIds)
{
std::vector<SymbolIndexerTask> symbolIndexerTask;
symbolIndexerTask.reserve(filePathIds.size());
FilePathIds dependentSourcePathIds = m_symbolStorage.fetchDependentSourceIds(filePathIds);
for (FilePathId filePathId : filePathIds)
updateChangedPath(filePathId, symbolIndexerTask);
std::vector<SymbolIndexerTask> symbolIndexerTask;
symbolIndexerTask.reserve(dependentSourcePathIds.size());
for (FilePathId dependentSourcePathId : dependentSourcePathIds)
updateChangedPath(dependentSourcePathId, symbolIndexerTask);
m_symbolIndexerTaskQueue.addOrUpdateTasks(std::move(symbolIndexerTask));
m_symbolIndexerTaskQueue.processEntries();
@@ -161,6 +169,8 @@ void SymbolIndexer::updateChangedPath(FilePathId filePathId,
= m_precompiledHeaderStorage.fetchPrecompiledHeader(optionalArtefact->projectPartId);
transaction.commit();
SourceTimeStamps dependentTimeStamps = m_symbolStorage.fetchIncludedIndexingTimeStamps(filePathId);
const ProjectPartArtefact &artefact = *optionalArtefact;
auto pchPath = optionalProjectPartPch ? optionalProjectPartPch->pchPath : FilePath{};
@@ -176,10 +186,9 @@ void SymbolIndexer::updateChangedPath(FilePathId filePathId,
if (success) {
Sqlite::ImmediateTransaction transaction{m_transactionInterface};
m_symbolStorage.insertOrUpdateIndexingTimeStamps(symbolsCollector.fileStatuses());
m_symbolStorage.addSymbolsAndSourceLocations(symbolsCollector.symbols(),
symbolsCollector.sourceLocations());
transaction.commit();
}
};

5
src/tools/clangrefactoringbackend/source/symbolindexer.h
View File

@@ -32,6 +32,7 @@
#include "clangpathwatcher.h"
#include <filecontainerv2.h>
#include <modifiedtimecheckerinterface.h>
#include <precompiledheaderstorageinterface.h>
#include <projectpartcontainer.h>
#include <projectpartsstorageinterface.h>
@@ -51,7 +52,8 @@ public:
FilePathCachingInterface &filePathCache,
FileStatusCache &fileStatusCache,
Sqlite::TransactionInterface &transactionInterface,
ProjectPartsStorageInterface &projectPartsStorage);
ProjectPartsStorageInterface &projectPartsStorage,
ModifiedTimeCheckerInterface<SourceTimeStamps> &modifiedTimeChecker);
void updateProjectParts(ProjectPartContainers &&projectParts);
void updateProjectPart(ProjectPartContainer &&projectPart);
@@ -81,6 +83,7 @@ private:
FileStatusCache &m_fileStatusCache;
Sqlite::TransactionInterface &m_transactionInterface;
ProjectPartsStorageInterface &m_projectPartsStorage;
ModifiedTimeCheckerInterface<SourceTimeStamps> &m_modifiedTimeChecker;
};
} // namespace ClangBackEnd

14
src/tools/clangrefactoringbackend/source/symbolindexing.h
View File

@@ -38,13 +38,16 @@
#include <precompiledheaderstorage.h>
#include <projectpartsstorage.h>
#include <refactoringdatabaseinitializer.h>
#include <filepathcachingfwd.h>
#include <modifiedtimechecker.h>
#include <refactoringdatabaseinitializer.h>
#include <sqlitedatabase.h>
#include <sqlitereadstatement.h>
#include <sqlitewritestatement.h>
#include <QDateTime>
#include <QFileInfo>
#include <QFileSystemWatcher>
#include <thread>
@@ -130,6 +133,12 @@ private:
FileStatusCache m_fileStatusCache{m_filePathCache};
SymbolsCollectorManager m_collectorManger;
ProgressCounter m_progressCounter;
std::function<TimeStamp(FilePathView filePath)> getModifiedTime{
[&](ClangBackEnd::FilePathView path) -> TimeStamp {
return QFileInfo(QString(path)).lastModified().toSecsSinceEpoch();
}};
ModifiedTimeChecker<ClangBackEnd::SourceTimeStamps> m_modifiedTimeChecker{getModifiedTime,
m_filePathCache};
SymbolIndexer m_indexer{m_indexerQueue,
m_symbolStorage,
m_buildDependencyStorage,
@@ -138,7 +147,8 @@ private:
m_filePathCache,
m_fileStatusCache,
m_symbolStorage.database,
m_projectPartsStorage};
m_projectPartsStorage,
m_modifiedTimeChecker};
SymbolIndexerTaskQueue m_indexerQueue{m_indexerScheduler, m_progressCounter};
SymbolIndexerTaskScheduler m_indexerScheduler;
};

108
src/tools/clangrefactoringbackend/source/symbolstorage.h
View File

@@ -41,7 +41,7 @@
namespace ClangBackEnd {
template <typename DatabaseType>
template<typename DatabaseType = Sqlite::Database>
class SymbolStorage final : public SymbolStorageInterface
{
using Database = DatabaseType;
@@ -70,6 +70,93 @@ public:
deleteNewLocationsTable();
}
void insertOrUpdateIndexingTimeStamps(const FilePathIds &filePathIds, TimeStamp indexingTimeStamp)
{
try {
Sqlite::ImmediateTransaction transaction{database};
for (FilePathId filePathId : filePathIds) {
inserOrUpdateIndexingTimesStampStatement.write(filePathId.filePathId,
indexingTimeStamp.value);
}
transaction.commit();
} catch (const Sqlite::StatementIsBusy &) {
insertOrUpdateIndexingTimeStamps(filePathIds, indexingTimeStamp);
}
}
void insertOrUpdateIndexingTimeStamps(const FileStatuses &fileStatuses) override
{
for (FileStatus fileStatus : fileStatuses) {
inserOrUpdateIndexingTimesStampStatement.write(fileStatus.filePathId.filePathId,
fileStatus.lastModified);
}
}
SourceTimeStamps fetchIndexingTimeStamps() const
{
try {
Sqlite::DeferredTransaction transaction{database};
auto timeStamps = fetchIndexingTimeStampsStatement.template values<SourceTimeStamp, 2>(
1024);
transaction.commit();
return timeStamps;
} catch (const Sqlite::StatementIsBusy &) {
return fetchIndexingTimeStamps();
}
}
SourceTimeStamps fetchIncludedIndexingTimeStamps(FilePathId sourcePathId) const
{
try {
Sqlite::DeferredTransaction transaction{database};
auto timeStamps = fetchIncludedIndexingTimeStampsStatement
.template values<SourceTimeStamp, 2>(1024, sourcePathId.filePathId);
transaction.commit();
return timeStamps;
} catch (const Sqlite::StatementIsBusy &) {
return fetchIncludedIndexingTimeStamps(sourcePathId);
}
}
FilePathIds fetchDependentSourceIds(const FilePathIds &sourcePathIds) const override
{
try {
FilePathIds dependentSourceIds;
Sqlite::DeferredTransaction transaction{database};
for (FilePathId sourcePathId : sourcePathIds) {
FilePathIds newDependentSourceIds;
newDependentSourceIds.reserve(dependentSourceIds.size() + 1024);
auto newIds = fetchDependentSourceIdsStatement
.template values<FilePathId>(1024, sourcePathId.filePathId);
std::set_union(dependentSourceIds.begin(),
dependentSourceIds.end(),
newIds.begin(),
newIds.end(),
std::back_inserter(newDependentSourceIds));
dependentSourceIds = std::move(newDependentSourceIds);
}
transaction.commit();
return dependentSourceIds;
} catch (const Sqlite::StatementIsBusy &) {
return fetchDependentSourceIds(sourcePathIds);
}
}
void fillTemporarySymbolsTable(const SymbolEntries &symbolEntries)
{
WriteStatement &statement = insertSymbolsToNewSymbolsStatement;
@@ -191,6 +278,25 @@ public:
database};
WriteStatement deleteNewSymbolsTableStatement{"DELETE FROM newSymbols", database};
WriteStatement deleteNewLocationsTableStatement{"DELETE FROM newLocations", database};
WriteStatement inserOrUpdateIndexingTimesStampStatement{
"INSERT INTO fileStatuses(sourceId, indexingTimeStamp) VALUES (?001, ?002) ON "
"CONFLICT(sourceId) DO UPDATE SET indexingTimeStamp = ?002",
database};
mutable ReadStatement fetchIncludedIndexingTimeStampsStatement{
"WITH RECURSIVE collectedDependencies(sourceId) AS (VALUES(?) UNION SELECT "
"dependencySourceId FROM sourceDependencies, collectedDependencies WHERE "
"sourceDependencies.sourceId == collectedDependencies.sourceId) SELECT DISTINCT sourceId, "
"indexingTimeStamp FROM collectedDependencies NATURAL JOIN fileStatuses ORDER BY sourceId",
database};
mutable ReadStatement fetchIndexingTimeStampsStatement{
"SELECT sourceId, indexingTimeStamp FROM fileStatuses", database};
mutable ReadStatement fetchDependentSourceIdsStatement{
"WITH RECURSIVE collectedDependencies(sourceId) AS (VALUES(?) UNION SELECT "
"sourceDependencies.sourceId FROM sourceDependencies, collectedDependencies WHERE "
"sourceDependencies.dependencySourceId == collectedDependencies.sourceId) SELECT sourceId "
"FROM collectedDependencies WHERE sourceId NOT IN (SELECT dependencySourceId FROM "
"sourceDependencies) ORDER BY sourceId",
database};
};
} // namespace ClangBackEnd

8
src/tools/clangrefactoringbackend/source/symbolstorageinterface.h
View File

@@ -28,6 +28,9 @@
#include "sourcelocationentry.h"
#include "symbolentry.h"
#include <filestatus.h>
#include <sourceentry.h>
#include <compilermacro.h>
#include <sqlitetransaction.h>
@@ -45,6 +48,11 @@ public:
virtual void addSymbolsAndSourceLocations(const SymbolEntries &symbolEntries,
const SourceLocationEntries &sourceLocations)
= 0;
virtual void insertOrUpdateIndexingTimeStamps(const FilePathIds &filePathIds, TimeStamp indexingTimeStamp) = 0;
virtual void insertOrUpdateIndexingTimeStamps(const FileStatuses &fileStatuses) = 0;
virtual SourceTimeStamps fetchIndexingTimeStamps() const = 0;
virtual SourceTimeStamps fetchIncludedIndexingTimeStamps(FilePathId sourcePathId) const = 0;
virtual FilePathIds fetchDependentSourceIds(const FilePathIds &sourcePathIds) const = 0;
protected:
~SymbolStorageInterface() = default;