feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

Clang: Improve performance by reducing parsing

Browse Source 
We generate one big file per project part so the preprocessor is skipping
the recurring includes.

This generated many errors about missing macros but we don't care
much about them during dependency collection step so we just
silence these errors with ignoring diagnostics consumer.

Change-Id: I5581d623b5d5f9995496252735577ea6b54790d9
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>
This commit is contained in:
Marco Bubke
2019-02-06 17:18:15 +01:00
parent 7fe65851d9
commit ffa043fb99
6 changed files with 131 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/tools/clangpchmanagerbackend/clangpchmanagerbackendmain.cpp
View File

@@ -191,7 +191,9 @@ struct Data // because we have a cycle dependency
database};
ClangBackEnd::PchTasksMerger pchTaskMerger{pchTaskQueue};
ClangBackEnd::BuildDependenciesStorage<> buildDependencyStorage{database};
ClangBackEnd::BuildDependencyCollector buildDependencyCollector{filePathCache, generatedFiles};
ClangBackEnd::BuildDependencyCollector buildDependencyCollector{filePathCache,
generatedFiles,
environment};
std::function<TimeStamp(FilePathView filePath)> getModifiedTime{
[&](ClangBackEnd::FilePathView path) -> TimeStamp {
return QFileInfo(QString(path)).lastModified().toSecsSinceEpoch();

48
src/tools/clangpchmanagerbackend/source/builddependencycollector.cpp
View File

@@ -28,10 +28,12 @@
#include "collectbuilddependencytoolaction.h"
#include "commandlinebuilder.h"
#include <environment.h>
#include <utils/smallstring.h>
#include <algorithm>
#include <iostream>
namespace ClangBackEnd {
namespace {
@@ -50,7 +52,7 @@ BuildDependency BuildDependencyCollector::create(const ProjectPartContainer &pro
CommandLineBuilder<ProjectPartContainer, Utils::SmallStringVector>
builder{projectPart, projectPart.toolChainArguments, InputFileType::Source};
addFiles(projectPart.sourcePathIds, builder.commandLine);
addFiles(projectPart.sourcePathIds, std::move(builder.commandLine));
setExcludedFilePaths(
m_filePathCache.filePaths(projectPart.headerPathIds + projectPart.sourcePathIds));
@@ -66,6 +68,33 @@ BuildDependency BuildDependencyCollector::create(const ProjectPartContainer &pro
return buildDependency;
}
namespace {
std::size_t contentSize(const FilePaths &includes)
{
auto countIncludeSize = [](std::size_t size, const auto &include) {
return size + include.size();
};
return std::accumulate(includes.begin(), includes.end(), std::size_t(0), countIncludeSize);
}
} // namespace
Utils::SmallString BuildDependencyCollector::generateFakeFileContent(
const FilePathIds &includeIds) const
{
Utils::SmallString fileContent;
const std::size_t lineTemplateSize = 12;
auto includes = m_filePathCache.filePaths(includeIds);
fileContent.reserve(includes.size() * lineTemplateSize + contentSize(includes));
for (Utils::SmallStringView include : includes)
fileContent += {"#include \"", include, "\"\n"};
return fileContent;
}
void BuildDependencyCollector::collect()
{
clang::tooling::ClangTool tool = m_clangTool.createTool();
@@ -96,25 +125,26 @@ void BuildDependencyCollector::setExcludedFilePaths(ClangBackEnd::FilePaths &&ex
}
void BuildDependencyCollector::addFiles(const FilePathIds &filePathIds,
const Utils::SmallStringVector &arguments)
Utils::SmallStringVector &&arguments)
{
m_clangTool.addFiles(m_filePathCache.filePaths(filePathIds), arguments);
m_clangTool.addFile(FilePath{m_environment.pchBuildDirectory().toStdString(), "dummy.cpp"},
generateFakeFileContent(filePathIds),
std::move(arguments));
m_buildDependency.sourceFiles.insert(m_buildDependency.sourceFiles.end(),
filePathIds.begin(),
filePathIds.end());
}
void BuildDependencyCollector::addFile(FilePathId filePathId,
const Utils::SmallStringVector &arguments)
void BuildDependencyCollector::addFile(FilePathId filePathId, Utils::SmallStringVector &&arguments)
{
addFiles({filePathId}, arguments);
addFiles({filePathId}, std::move(arguments));
}
void BuildDependencyCollector::addFile(FilePath filePath,
const FilePathIds &sourceFileIds,
const Utils::SmallStringVector &arguments)
Utils::SmallStringVector &&arguments)
{
m_clangTool.addFiles({filePath}, arguments);
m_clangTool.addFiles({filePath}, std::move(arguments));
m_buildDependency.sourceFiles.insert(m_buildDependency.sourceFiles.end(),
sourceFileIds.begin(),
sourceFileIds.end());

19
src/tools/clangpchmanagerbackend/source/builddependencycollector.h
View File

@@ -34,33 +34,35 @@
#include <filepathcachingfwd.h>
namespace ClangBackEnd {
class Environment;
class BuildDependencyCollector : public BuildDependencyGeneratorInterface
{
public:
BuildDependencyCollector(const FilePathCachingInterface &filePathCache,
const GeneratedFilesInterface &generatedFiles)
const GeneratedFilesInterface &generatedFiles,
const Environment &environment)
: m_filePathCache(filePathCache)
, m_generatedFiles(generatedFiles)
{
}
, m_environment(environment)
{}
BuildDependency create(const ProjectPartContainer &projectPart) override;
void collect();
void setExcludedFilePaths(ClangBackEnd::FilePaths &&excludedIncludes);
void addFiles(const FilePathIds &filePathIds,
const Utils::SmallStringVector &arguments);
void addFile(FilePathId filePathId,
const Utils::SmallStringVector &arguments);
void addFiles(const FilePathIds &filePathIds, Utils::SmallStringVector &&arguments);
void addFile(FilePathId filePathId, Utils::SmallStringVector &&arguments);
void addFile(FilePath filePath,
const FilePathIds &sourceFileIds,
const Utils::SmallStringVector &arguments);
Utils::SmallStringVector &&arguments);
void addUnsavedFiles(const V2::FileContainers &unsavedFiles);
void clear();
Utils::SmallString generateFakeFileContent(const FilePathIds &includeIds) const;
const FileStatuses &fileStatuses() const
{
return m_buildDependency.fileStatuses;
@@ -96,6 +98,7 @@ private:
SourcesManager m_sourcesManager;
const FilePathCachingInterface &m_filePathCache;
const GeneratedFilesInterface &m_generatedFiles;
const Environment &m_environment;
};
} // namespace ClangBackEnd

3
src/tools/clangpchmanagerbackend/source/collectbuilddependencyaction.h
View File

@@ -69,7 +69,8 @@ public:
m_sourcesManager,
compilerInstance.getPreprocessorPtr());
preprocessor.addPPCallbacks(std::unique_ptr<clang::PPCallbacks>(macroPreprocessorCallbacks));
preprocessor.addPPCallbacks(
std::unique_ptr<clang::PPCallbacks>(macroPreprocessorCallbacks));
return true;
}

24
src/tools/clangpchmanagerbackend/source/collectbuilddependencypreprocessorcallbacks.h
View File

@@ -72,18 +72,30 @@ public:
void FileChanged(clang::SourceLocation sourceLocation,
clang::PPCallbacks::FileChangeReason reason,
clang::SrcMgr::CharacteristicKind,
clang::FileID) override
{
if (reason == clang::PPCallbacks::EnterFile)
clang::FileID previousFileId) override
{
if (reason == clang::PPCallbacks::EnterFile) {
clang::FileID currentFileId = m_sourceManager->getFileID(sourceLocation);
if (m_mainFileId.isInvalid()) {
m_mainFileId = currentFileId;
} else {
const clang::FileEntry *fileEntry = m_sourceManager->getFileEntryForID(
m_sourceManager->getFileID(sourceLocation));
currentFileId);
if (fileEntry) {
if (previousFileId == m_mainFileId) {
uint sourceFileUID = fileEntry->getUID();
auto notAlreadyIncluded = isNotAlreadyIncluded(sourceFileUID);
if (notAlreadyIncluded.first)
m_alreadyIncludedFileUIDs.insert(notAlreadyIncluded.second,
sourceFileUID);
} else {
addFileStatus(fileEntry);
addSourceFile(fileEntry);
}
}
}
}
}
void InclusionDirective(clang::SourceLocation hashLocation,
const clang::Token & /*includeToken*/,
@@ -96,7 +108,8 @@ public:
const clang::Module * /*imported*/,
clang::SrcMgr::CharacteristicKind fileType) override
{
if (file) {
clang::FileID currentFileId = m_sourceManager->getFileID(hashLocation);
if (file && currentFileId != m_mainFileId) {
addSourceDependency(file, hashLocation);
auto fileUID = file->getUID();
auto sourceFileUID = m_sourceManager
@@ -343,6 +356,7 @@ private:
BuildDependency &m_buildDependency;
const std::vector<uint> &m_excludedIncludeUID;
std::vector<uint> &m_alreadyIncludedFileUIDs;
clang::FileID m_mainFileId;
};
} // namespace ClangBackEnd

41
tests/unit/unittest/builddependencycollector-test.cpp
View File

@@ -25,6 +25,8 @@
#include "googletest.h"
#include "testenvironment.h"
#include <refactoringdatabaseinitializer.h>
#include <filepathcaching.h>
#include <generatedfiles.h>
@@ -67,15 +69,8 @@ protected:
{
setFilePathCache(&filePathCache);
collector.addFile(id(TESTDATA_DIR "/builddependencycollector/project/main.cpp"),
{"cc",
"-I",
TESTDATA_DIR "/builddependencycollector/external",
"-I",
TESTDATA_DIR "/builddependencycollector/project",
"-isystem",
TESTDATA_DIR "/builddependencycollector/system"});
collector.addFile(id(TESTDATA_DIR "/builddependencycollector/project/main2.cpp"),
collector.addFiles({id(TESTDATA_DIR "/builddependencycollector/project/main.cpp"),
id(TESTDATA_DIR "/builddependencycollector/project/main2.cpp")},
{"cc",
"-I",
TESTDATA_DIR "/builddependencycollector/external",
@@ -160,12 +155,14 @@ protected:
protected:
Sqlite::Database database{":memory:", Sqlite::JournalMode::Memory};
TestEnvironment environment;
ClangBackEnd::RefactoringDatabaseInitializer<Sqlite::Database> databaseInitializer{database};
ClangBackEnd::FilePathCaching filePathCache{database};
ClangBackEnd::GeneratedFiles generatedFiles;
ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles};
ClangBackEnd::BuildDependencyCollector emptyCollector{filePathCache, generatedFiles};
ClangBackEnd::FilePaths excludePaths = {TESTDATA_DIR "/builddependencycollector/project/main.cpp",
ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles, environment};
ClangBackEnd::BuildDependencyCollector emptyCollector{filePathCache, generatedFiles, environment};
ClangBackEnd::FilePaths excludePaths = {
TESTDATA_DIR "/builddependencycollector/project/main.cpp",
TESTDATA_DIR "/builddependencycollector/project/main2.cpp",
TESTDATA_DIR "/builddependencycollector/project/header1.h",
TESTDATA_DIR "/builddependencycollector/project/header2.h",
@@ -548,10 +545,26 @@ TEST_F(BuildDependencyCollector, GeneratedFile)
SourceType::UserInclude)));
}
TEST_F(BuildDependencyCollector, CreateFakeFileContent)
{
auto content = collector.generateFakeFileContent(
{id(TESTDATA_DIR "/builddependencycollector/project/header2.h"),
id(TESTDATA_DIR "/builddependencycollector/external/external1.h"),
id(TESTDATA_DIR "/builddependencycollector/external/external2.h")});
ASSERT_THAT(std::string(content),
AllOf(HasSubstr("#include \"" TESTDATA_DIR
"/builddependencycollector/project/header2.h\"\n"),
HasSubstr("#include \"" TESTDATA_DIR
"/builddependencycollector/external/external1.h\"\n"),
HasSubstr("#include \"" TESTDATA_DIR
"/builddependencycollector/external/external2.h\"\n")));
}
TEST_F(BuildDependencyCollector, Create)
{
using ClangBackEnd::IncludeSearchPathType;
ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles};
ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles, environment};
generatedFiles.update(
{{TESTDATA_DIR "/builddependencycollector/project/generated_file.h", "#pragma once"}});
ClangBackEnd::ProjectPartContainer projectPart{
@@ -702,7 +715,7 @@ TEST_F(BuildDependencyCollector, Create)
TEST_F(BuildDependencyCollector, Clear)
{
using ClangBackEnd::IncludeSearchPathType;
ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles};
ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles, environment};
ClangBackEnd::ProjectPartContainer projectPart{
"project1",
{},