Clang: Add symbol storage

Extend file path cache to 64 bit integer.

Change-Id: I5627f13d59a3214f389087038482cbcc8d0eb484
Reviewed-by: Tim Jenssen <tim.jenssen@qt.io>
This commit is contained in:
Marco Bubke
2017-08-21 12:00:27 +02:00
parent 81d43b8a11
commit f49a1d721c
48 changed files with 1297 additions and 231 deletions

View File

@@ -54,7 +54,7 @@ struct CollectBoundNodes : MatchFinder::MatchCallback {
}
};
ClangQuery::ClangQuery(StringCache<Utils::PathString, std::mutex> &filePathCache,
ClangQuery::ClangQuery(FilePathCache<std::mutex> &filePathCache,
Utils::SmallString &&query)
: query(std::move(query)),
filePathCache(filePathCache)

View File

@@ -51,7 +51,7 @@ namespace ClangBackEnd {
class ClangQuery : public ClangTool
{
public:
ClangQuery(StringCache<Utils::PathString, std::mutex> &filePathCache, Utils::SmallString &&query={});
ClangQuery(FilePathCache<std::mutex> &filePathCache, Utils::SmallString &&query={});
void setQuery(Utils::SmallString &&query);
@@ -69,7 +69,7 @@ private:
SourceRangesContainer sourceRangesContainer;
Utils::SmallString query;
std::vector<DynamicASTMatcherDiagnosticContainer> diagnosticContainers_;
StringCache<Utils::PathString, std::mutex> &filePathCache;
FilePathCache<std::mutex> &filePathCache;
};
} // namespace ClangBackEnd

View File

@@ -29,7 +29,7 @@
namespace ClangBackEnd {
ClangQueryGatherer::ClangQueryGatherer(StringCache<Utils::PathString, std::mutex> *filePathCache,
ClangQueryGatherer::ClangQueryGatherer(FilePathCache<std::mutex> *filePathCache,
std::vector<V2::FileContainer> &&sources,
std::vector<V2::FileContainer> &&unsaved,
Utils::SmallString &&query)
@@ -43,7 +43,7 @@ ClangQueryGatherer::ClangQueryGatherer(StringCache<Utils::PathString, std::mutex
SourceRangesForQueryMessage
ClangQueryGatherer::createSourceRangesForSource(
StringCache<Utils::PathString, std::mutex> *filePathCache,
FilePathCache<std::mutex> *filePathCache,
V2::FileContainer &&source,
const std::vector<V2::FileContainer> &unsaved,
Utils::SmallString &&query)

View File

@@ -41,13 +41,13 @@ public:
using Future = std::future<SourceRangesForQueryMessage>;
ClangQueryGatherer() = default;
ClangQueryGatherer(StringCache<Utils::PathString, std::mutex> *filePathCache,
ClangQueryGatherer(FilePathCache<std::mutex> *filePathCache,
std::vector<V2::FileContainer> &&sources,
std::vector<V2::FileContainer> &&unsaved,
Utils::SmallString &&query);
static SourceRangesForQueryMessage createSourceRangesForSource(
StringCache<Utils::PathString, std::mutex> *filePathCache,
FilePathCache<std::mutex> *filePathCache,
V2::FileContainer &&source,
const std::vector<V2::FileContainer> &unsaved,
Utils::SmallString &&query);
@@ -69,7 +69,7 @@ protected:
std::vector<Future> finishedFutures();
private:
StringCache<Utils::PathString, std::mutex> *m_filePathCache = nullptr;
FilePathCache<std::mutex> *m_filePathCache = nullptr;
SourceRangeFilter m_sourceRangeFilter;
std::vector<V2::FileContainer> m_sources;
std::vector<V2::FileContainer> m_unsaved;

View File

@@ -12,7 +12,9 @@ HEADERS += \
$$PWD/collectsymbolsastvisitor.h \
$$PWD/sourcelocationentry.h \
$$PWD/symbolscollectorinterface.h \
$$PWD/symbolstorageinterface.h
$$PWD/symbolstorageinterface.h \
$$PWD/symbolstorage.h \
$$PWD/storagesqlitestatementfactory.h
!isEmpty(LIBTOOLING_LIBS) {
SOURCES += \
@@ -52,4 +54,5 @@ SOURCES += \
$$PWD/collectsymbolsaction.cpp \
$$PWD/collectmacrossourcefilecallbacks.cpp \
$$PWD/symbolentry.cpp \
$$PWD/sourcelocationentry.cpp
$$PWD/sourcelocationentry.cpp \
$$PWD/symbolstorage.cpp

View File

@@ -60,7 +60,7 @@ public:
bool VisitNamedDecl(const clang::NamedDecl *declaration)
{
auto globalId = declaration->getCanonicalDecl()->getLocation().getRawEncoding();
SymbolIndex globalId = toSymbolIndex(declaration->getCanonicalDecl());
auto sourceLocation = declaration->getLocation();
auto found = m_symbolEntries.find(globalId);
@@ -81,7 +81,7 @@ public:
bool VisitDeclRefExpr(const clang::DeclRefExpr *expression)
{
auto declaration = expression->getFoundDecl();
auto globalId = declaration->getCanonicalDecl()->getLocation().getRawEncoding();
SymbolIndex globalId = toSymbolIndex(declaration->getCanonicalDecl());
auto sourceLocation = expression->getLocation();
m_sourceLocationEntries.emplace_back(globalId,
@@ -92,7 +92,7 @@ public:
return true;
}
uint filePathId(clang::SourceLocation sourceLocation)
FilePathIndex filePathId(clang::SourceLocation sourceLocation)
{
auto filePath = m_sourceManager.getFilename(sourceLocation);
@@ -114,6 +114,11 @@ public:
return usr;
}
static SymbolIndex toSymbolIndex(const void *pointer)
{
return SymbolIndex(reinterpret_cast<std::uintptr_t>(pointer));
}
private:
SymbolEntries &m_symbolEntries;
SourceLocationEntries &m_sourceLocationEntries;

View File

@@ -75,7 +75,7 @@ private:
Utils::SmallString &&query);
private:
StringCache<Utils::PathString, std::mutex> m_filePathCache;
FilePathCache<std::mutex> m_filePathCache;
ClangQueryGatherer m_gatherer;
QTimer m_pollTimer;
};

View File

@@ -25,6 +25,8 @@
#pragma once
#include <stringcachefwd.h>
#include <limits>
#include <vector>
#include <iosfwd>
@@ -51,11 +53,13 @@ public:
uint column = 0;
};
using SymbolIndex = long long;
class SourceLocationEntry
{
public:
SourceLocationEntry(uint symbolId,
uint fileId,
SourceLocationEntry(SymbolIndex symbolId,
FilePathIndex fileId,
LineColumn lineColumn,
SymbolType symbolType)
: symbolId(symbolId),
@@ -65,8 +69,8 @@ public:
symbolType(symbolType)
{}
uint symbolId = 0;
uint fileId = std::numeric_limits<uint>::max();
SymbolIndex symbolId = 0;
FilePathIndex fileId = std::numeric_limits<uint>::max();
uint line = 0;
uint column = 0;
SymbolType symbolType;

View File

@@ -54,7 +54,7 @@ namespace ClangBackEnd {
SourceRangeExtractor::SourceRangeExtractor(
const clang::SourceManager &sourceManager,
const clang::LangOptions &languageOptions,
ClangBackEnd::StringCache<Utils::PathString, std::mutex> &filePathCache,
ClangBackEnd::FilePathCache<std::mutex> &filePathCache,
SourceRangesContainer &sourceRangesContainer)
: sourceManager(sourceManager),
languageOptions(languageOptions),
@@ -145,7 +145,7 @@ void SourceRangeExtractor::insertSourceRange(uint fileId,
std::move(lineSnippet));
}
uint SourceRangeExtractor::findFileId(clang::FileID fileId, const clang::FileEntry *fileEntry) const
FilePathIndex SourceRangeExtractor::findFileId(clang::FileID fileId, const clang::FileEntry *fileEntry) const
{
auto found = m_fileIdMapping.find(fileId.getHashValue());
if (found != m_fileIdMapping.end()) {

View File

@@ -59,7 +59,7 @@ class SourceRangeExtractor
public:
SourceRangeExtractor(const clang::SourceManager &sourceManager,
const clang::LangOptions &languageOptions,
ClangBackEnd::StringCache<Utils::PathString, std::mutex> &filePathCache,
ClangBackEnd::FilePathCache<std::mutex> &filePathCache,
SourceRangesContainer &sourceRangesContainer);
void addSourceRange(const clang::SourceRange &sourceRange);
@@ -82,13 +82,13 @@ private:
uint endOffset,
Utils::SmallString &&lineSnippet);
uint findFileId(clang::FileID fileId, const clang::FileEntry *fileEntry) const;
FilePathIndex findFileId(clang::FileID fileId, const clang::FileEntry *fileEntry) const;
private:
mutable std::unordered_map<uint, uint> m_fileIdMapping;
const clang::SourceManager &sourceManager;
const clang::LangOptions &languageOptions;
ClangBackEnd::StringCache<Utils::PathString, std::mutex> &filePathCache;
ClangBackEnd::FilePathCache<std::mutex> &filePathCache;
SourceRangesContainer &sourceRangesContainer;
};

View File

@@ -0,0 +1,180 @@
/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include <createtablesqlstatementbuilder.h>
#include <sqlitetransaction.h>
#include <sqlitetable.h>
namespace ClangBackEnd {
template<typename Database,
typename ReadStatement,
typename WriteStatement>
class StorageSqliteStatementFactory
{
public:
using DatabaseType = Database;
using ReadStatementType = ReadStatement;
using WriteStatementType = WriteStatement;
StorageSqliteStatementFactory(Database &database)
: database(database)
{
}
Sqlite::SqliteTable createSymbolsTable()
{
Sqlite::SqliteTable table;
table.setUseIfNotExists(true);
table.setName("symbols");
table.addColumn("symbolId", Sqlite::ColumnType::Integer, Sqlite::Contraint::PrimaryKey);
table.addColumn("usr", Sqlite::ColumnType::Text);
table.addColumn("symbolName", Sqlite::ColumnType::Text);
Sqlite::SqliteImmediateTransaction<DatabaseType> transaction(database);
table.initialize(database);
transaction.commit();
return table;
}
Sqlite::SqliteTable createLocationsTable()
{
Sqlite::SqliteTable table;
table.setUseIfNotExists(true);
table.setName("locations");
table.addColumn("symbolId", Sqlite::ColumnType::Integer);
table.addColumn("line", Sqlite::ColumnType::Integer);
table.addColumn("column", Sqlite::ColumnType::Integer);
table.addColumn("sourceId", Sqlite::ColumnType::Integer);
Sqlite::SqliteImmediateTransaction<DatabaseType> transaction(database);
table.initialize(database);
transaction.commit();
return table;
}
Sqlite::SqliteTable createSourcesTable()
{
Sqlite::SqliteTable table;
table.setUseIfNotExists(true);
table.setName("sources");
table.addColumn("sourceId", Sqlite::ColumnType::Integer, Sqlite::Contraint::PrimaryKey);
table.addColumn("sourcePath", Sqlite::ColumnType::Text);
Sqlite::SqliteImmediateTransaction<DatabaseType> transaction(database);
table.initialize(database);
transaction.commit();
return table;
}
Sqlite::SqliteTable createNewSymbolsTable() const
{
Sqlite::SqliteTable table;
table.setName("newSymbols");
table.setUseTemporaryTable(true);
table.addColumn("temporarySymbolId", Sqlite::ColumnType::Integer, Sqlite::Contraint::PrimaryKey);
table.addColumn("symbolId", Sqlite::ColumnType::Integer);
table.addColumn("usr", Sqlite::ColumnType::Text);
table.addColumn("symbolName", Sqlite::ColumnType::Text);
Sqlite::SqliteImmediateTransaction<DatabaseType> transaction(database);
table.initialize(database);
transaction.commit();
return table;
}
Sqlite::SqliteTable createNewLocationsTable() const
{
Sqlite::SqliteTable table;
table.setName("newLocations");
table.setUseTemporaryTable(true);
table.addColumn("temporarySymbolId", Sqlite::ColumnType::Integer);
table.addColumn("symbolId", Sqlite::ColumnType::Integer);
table.addColumn("line", Sqlite::ColumnType::Integer);
table.addColumn("column", Sqlite::ColumnType::Integer);
table.addColumn("sourceId", Sqlite::ColumnType::Integer);
Sqlite::SqliteImmediateTransaction<DatabaseType> transaction(database);
table.initialize(database);
transaction.commit();
return table;
}
public:
Database &database;
Sqlite::SqliteTable symbolsTable{createSymbolsTable()};
Sqlite::SqliteTable locationsTable{createLocationsTable()};
Sqlite::SqliteTable sourcesTable{createSourcesTable()};
Sqlite::SqliteTable newSymbolsTablet{createNewSymbolsTable()};
Sqlite::SqliteTable newLocationsTable{createNewLocationsTable()};
WriteStatement insertSymbolsToNewSymbolsStatement{
"INSERT INTO newSymbols(temporarySymbolId, usr, symbolName) VALUES(?,?,?)",
database};
WriteStatement insertLocationsToNewLocationsStatement{
"INSERT INTO newLocations(temporarySymbolId, line, column, sourceId) VALUES(?,?,?,?)",
database};
// WriteStatement syncNewLocationsToLocationsStatement{
// "INSERT INTO locations(symbolId, line, column, sourceId) SELECT symbolId, line, column, sourceId FROM newLocations",
// database};
ReadStatement selectNewSourceIdsStatement{
"SELECT DISTINCT sourceId FROM newLocations WHERE NOT EXISTS (SELECT sourceId FROM sources WHERE newLocations.sourceId == sources.sourceId)",
database};
WriteStatement addNewSymbolsToSymbolsStatement{
"INSERT INTO symbols(usr, symbolname) "
"SELECT usr, symbolname FROM newsymbols WHERE NOT EXISTS "
"(SELECT usr FROM symbols WHERE usr == newsymbols.usr)",
database};
WriteStatement insertSourcesStatement{
"INSERT INTO sources(sourceId, sourcePath) VALUES(?,?)",
database};
WriteStatement syncNewSymbolsFromSymbolsStatement{
"UPDATE newSymbols SET symbolId = (SELECT symbolId FROM symbols WHERE newSymbols.usr = symbols.usr)",
database};
WriteStatement syncSymbolsIntoNewLocationsStatement{
"UPDATE newLocations SET symbolId = (SELECT symbolId FROM newSymbols WHERE newSymbols.temporarySymbolId = newLocations.temporarySymbolId)",
database};
WriteStatement deleteAllLocationsFromUpdatedFilesStatement{
"DELETE FROM locations WHERE sourceId IN (SELECT DISTINCT sourceId FROM newLocations)",
database};
WriteStatement insertNewLocationsInLocationsStatement{
"INSERT INTO locations(symbolId, line, column, sourceId) SELECT symbolId, line, column, sourceId FROM newLocations",
database};
WriteStatement deleteNewSymbolsTableStatement{
"DELETE FROM newSymbols",
database};
WriteStatement deleteNewLocationsTableStatement{
"DELETE FROM newLocations",
database};
};
} // namespace ClangBackEnd

View File

@@ -25,6 +25,8 @@
#pragma once
#include <stringcachefwd.h>
#include <utils/smallstring.h>
#include <llvm/ADT/SmallVector.h>
@@ -36,6 +38,8 @@
namespace ClangBackEnd {
using SymbolIndex = long long;
class SymbolEntry
{
public:
@@ -60,7 +64,7 @@ public:
}
};
using SymbolEntries = std::unordered_map<uint, SymbolEntry>;
using SymbolEntries = std::unordered_map<SymbolIndex, SymbolEntry>;
std::ostream &operator<<(std::ostream &out, const SymbolEntry &entry);

View File

@@ -0,0 +1,30 @@
/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "symbolstorage.h"
namespace ClangBackEnd {
} // namespace ClangBackEnd

View File

@@ -0,0 +1,155 @@
/****************************************************************************
**
** Copyright (C) 2017 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "symbolstorageinterface.h"
#include <sqlitetransaction.h>
#include <stringcache.h>
namespace ClangBackEnd {
template <typename StatementFactory>
class SymbolStorage : public SymbolStorageInterface
{
using Transaction = Sqlite::SqliteImmediateTransaction<typename StatementFactory::DatabaseType>;
using ReadStatement = typename StatementFactory::ReadStatementType;
using WriteStatement = typename StatementFactory::WriteStatementType;
using Database = typename StatementFactory::DatabaseType;
public:
SymbolStorage(StatementFactory &statementFactory,
FilePathCache<> &filePathCache)
: m_statementFactory(statementFactory),
m_filePathCache(filePathCache)
{
}
void addSymbolsAndSourceLocations(const SymbolEntries &symbolEntries,
const SourceLocationEntries &sourceLocations) override
{
Transaction transaction{m_statementFactory.database};
fillTemporarySymbolsTable(symbolEntries);
fillTemporaryLocationsTable(sourceLocations);
addNewSymbolsToSymbols();
syncNewSymbolsFromSymbols();
syncSymbolsIntoNewLocations();
insertNewSources();
deleteAllLocationsFromUpdatedFiles();
insertNewLocationsInLocations();
deleteNewSymbolsTable();
deleteNewLocationsTable();
transaction.commit();
}
void fillTemporarySymbolsTable(const SymbolEntries &symbolEntries)
{
WriteStatement &statement = m_statementFactory.insertSymbolsToNewSymbolsStatement;
for (const auto &symbolEntry : symbolEntries) {
statement.write(symbolEntry.first,
symbolEntry.second.usr,
symbolEntry.second.symbolName);
}
}
void fillTemporaryLocationsTable(const SourceLocationEntries &sourceLocations)
{
WriteStatement &statement = m_statementFactory.insertLocationsToNewLocationsStatement;
for (const auto &locationsEntry : sourceLocations) {
statement.write(locationsEntry.symbolId,
locationsEntry.line,
locationsEntry.column,
locationsEntry.fileId);
}
}
void addNewSymbolsToSymbols()
{
m_statementFactory.addNewSymbolsToSymbolsStatement.execute();
}
void syncNewSymbolsFromSymbols()
{
m_statementFactory.syncNewSymbolsFromSymbolsStatement.execute();
}
void syncSymbolsIntoNewLocations()
{
m_statementFactory.syncSymbolsIntoNewLocationsStatement.execute();
}
void deleteAllLocationsFromUpdatedFiles()
{
m_statementFactory.deleteAllLocationsFromUpdatedFilesStatement.execute();
}
void insertNewLocationsInLocations()
{
m_statementFactory.insertNewLocationsInLocationsStatement.execute();
}
FilePathIndices selectNewSourceIds() const
{
ReadStatement &statement = m_statementFactory.selectNewSourceIdsStatement;
return statement.template values<FilePathIndex>(16);
}
void insertNewSources()
{
WriteStatement &statement = m_statementFactory.insertSourcesStatement;
FilePathIndices newSourceIds = selectNewSourceIds();
for (FilePathIndex sourceId : newSourceIds)
statement.write(sourceId, m_filePathCache.string(sourceId));
}
void deleteNewSymbolsTable()
{
m_statementFactory.deleteNewSymbolsTableStatement.execute();
}
void deleteNewLocationsTable()
{
m_statementFactory.deleteNewLocationsTableStatement.execute();
}
SourceLocationEntries sourceLocations() const
{
return SourceLocationEntries();
}
private:
StatementFactory &m_statementFactory;
FilePathCache<> &m_filePathCache;
};
} // namespace ClangBackEnd

View File

@@ -33,7 +33,7 @@ namespace ClangBackEnd {
class SymbolStorageInterface
{
public:
virtual void addSymbolsAndSourceLocations(const SymbolEntries &symbolEentries,
virtual void addSymbolsAndSourceLocations(const SymbolEntries &symbolEntries,
const SourceLocationEntries &sourceLocations) = 0;
};