Clang: Add top system includes to include collector

If we want to split the project part we have to filter out the top system
includes.

Task-number: QTCREATORBUG-21345
Change-Id: I258fa33ac39bd94b4699f4f39923d6ad274c1dc7
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>
This commit is contained in:
Marco Bubke
2018-10-18 11:27:49 +02:00
parent be3e06cc7e
commit e571220663
39 changed files with 137 additions and 81 deletions

View File

@@ -41,11 +41,13 @@ class CollectIncludesAction final : public clang::PreprocessOnlyAction
public:
CollectIncludesAction(FilePathIds &includeIds,
FilePathIds &topIncludeIds,
FilePathIds &topsSystemIncludeIds,
const FilePathCachingInterface &filePathCache,
std::vector<uint> &excludedIncludeUID,
std::vector<uint> &alreadyIncludedFileUIDs)
: m_includeIds(includeIds),
m_topIncludeIds(topIncludeIds),
m_topsSystemIncludeIds(topsSystemIncludeIds),
m_filePathCache(filePathCache),
m_excludedIncludeUID(excludedIncludeUID),
m_alreadyIncludedFileUIDs(alreadyIncludedFileUIDs)
@@ -62,6 +64,7 @@ public:
auto macroPreprocessorCallbacks = new CollectIncludesPreprocessorCallbacks(
m_includeIds,
m_topIncludeIds,
m_topsSystemIncludeIds,
m_filePathCache,
m_excludedIncludeUID,
m_alreadyIncludedFileUIDs,
@@ -83,6 +86,7 @@ public:
private:
FilePathIds &m_includeIds;
FilePathIds &m_topIncludeIds;
FilePathIds &m_topsSystemIncludeIds;
const FilePathCachingInterface &m_filePathCache;
std::vector<uint> &m_excludedIncludeUID;
std::vector<uint> &m_alreadyIncludedFileUIDs;

View File

@@ -50,12 +50,14 @@ class CollectIncludesPreprocessorCallbacks final : public clang::PPCallbacks
public:
CollectIncludesPreprocessorCallbacks(FilePathIds &includeIds,
FilePathIds &topIncludeIds,
FilePathIds &topsSystemIncludeIds,
const FilePathCachingInterface &filePathCache,
const std::vector<uint> &excludedIncludeUID,
std::vector<uint> &alreadyIncludedFileUIDs,
clang::SourceManager &sourceManager)
: m_includeIds(includeIds),
m_topIncludeIds(topIncludeIds),
m_topsSystemIncludeIds(topsSystemIncludeIds),
m_filePathCache(filePathCache),
m_excludedIncludeUID(excludedIncludeUID),
m_alreadyIncludedFileUIDs(alreadyIncludedFileUIDs),
@@ -70,11 +72,8 @@ public:
const clang::FileEntry *file,
llvm::StringRef /*searchPath*/,
llvm::StringRef /*relativePath*/,
const clang::Module * /*imported*/
#if LLVM_VERSION_MAJOR >= 7
, clang::SrcMgr::CharacteristicKind /*fileType*/
#endif
) override
const clang::Module * /*imported*/,
clang::SrcMgr::CharacteristicKind fileType) override
{
if (!m_skipInclude && file) {
auto fileUID = file->getUID();
@@ -87,6 +86,8 @@ public:
if (!filePath.empty()) {
FilePathId includeId = m_filePathCache.filePathId(filePath);
m_includeIds.emplace_back(includeId);
if (isSystem(fileType) && !isInSystemHeader(hashLocation))
m_topsSystemIncludeIds.emplace_back(includeId);
if (isInExcludedIncludeUID(sourceFileUID))
m_topIncludeIds.emplace_back(includeId);
}
@@ -116,6 +117,16 @@ public:
return true;
}
bool isSystem(clang::SrcMgr::CharacteristicKind kind)
{
return kind != clang::SrcMgr::C_User && kind != clang::SrcMgr::C_User_ModuleMap;
}
bool isInSystemHeader(clang::SourceLocation location)
{
return m_sourceManager.isInSystemHeader(location);
}
void ensureDirectory(const QString &directory, const QString &fileName)
{
QStringList directoryEntries = fileName.split('/');
@@ -163,6 +174,7 @@ public:
private:
FilePathIds &m_includeIds;
FilePathIds &m_topIncludeIds;
FilePathIds &m_topsSystemIncludeIds;
const FilePathCachingInterface &m_filePathCache;
const std::vector<uint> &m_excludedIncludeUID;
std::vector<uint> &m_alreadyIncludedFileUIDs;

View File

@@ -39,10 +39,12 @@ class CollectIncludesToolAction final : public clang::tooling::FrontendActionFac
public:
CollectIncludesToolAction(FilePathIds &includeIds,
FilePathIds &topIncludeIds,
FilePathIds &topsSystemIncludeIds,
const FilePathCachingInterface &filePathCache,
const Utils::PathStringVector &excludedIncludes)
: m_includeIds(includeIds),
m_topIncludeIds(topIncludeIds),
m_topsSystemIncludeIds(topsSystemIncludeIds),
m_filePathCache(filePathCache),
m_excludedIncludes(excludedIncludes)
{}
@@ -66,6 +68,7 @@ public:
{
return new CollectIncludesAction(m_includeIds,
m_topIncludeIds,
m_topsSystemIncludeIds,
m_filePathCache,
m_excludedIncludeUIDs,
m_alreadyIncludedFileUIDs);
@@ -93,6 +96,7 @@ private:
std::vector<uint> m_excludedIncludeUIDs;
FilePathIds &m_includeIds;
FilePathIds &m_topIncludeIds;
FilePathIds &m_topsSystemIncludeIds;
const FilePathCachingInterface &m_filePathCache;
const Utils::PathStringVector &m_excludedIncludes;
};

View File

@@ -45,6 +45,7 @@ void IncludeCollector::collectIncludes()
auto action = std::unique_ptr<CollectIncludesToolAction>(
new CollectIncludesToolAction(m_includeIds,
m_topIncludeIds,
m_topsSystemIncludeIds,
m_filePathCache,
m_excludedIncludes));
@@ -84,6 +85,11 @@ FilePathIds IncludeCollector::takeTopIncludeIds()
return std::move(m_topIncludeIds);
}
FilePathIds IncludeCollector::takeTopsSystemIncludeIds()
{
return std::move(m_topsSystemIncludeIds);
}
} // namespace ClangBackEnd

View File

@@ -42,11 +42,13 @@ public:
FilePathIds takeIncludeIds();
FilePathIds takeTopIncludeIds();
FilePathIds takeTopsSystemIncludeIds();
private:
Utils::PathStringVector m_excludedIncludes;
FilePathIds m_includeIds;
FilePathIds m_topIncludeIds;
FilePathIds m_topsSystemIncludeIds;
Utils::SmallStringVector m_directories;
const FilePathCachingInterface &m_filePathCache;
};

View File

@@ -236,7 +236,7 @@ Utils::PathStringVector PchCreator::generateProjectPartSourcePaths(
return includeAndSources;
}
std::pair<FilePathIds,FilePathIds> PchCreator::generateProjectPartPchIncludes(
PchCreatorIncludes PchCreator::generateProjectPartPchIncludes(
const V2::ProjectPartContainer &projectPart) const
{
Utils::SmallString jointedFileContent = generateProjectPartSourcesContent(projectPart);
@@ -261,7 +261,7 @@ std::pair<FilePathIds,FilePathIds> PchCreator::generateProjectPartPchIncludes(
jointFile->remove();
return {collector.takeIncludeIds(), collector.takeTopIncludeIds()};
return {collector.takeIncludeIds(), collector.takeTopIncludeIds(), collector.takeTopsSystemIncludeIds()};
}
Utils::SmallString PchCreator::generateProjectPathPchHeaderFilePath(
@@ -312,10 +312,8 @@ Utils::SmallStringVector PchCreator::generateProjectPartClangCompilerArguments(
IdPaths PchCreator::generateProjectPartPch(const V2::ProjectPartContainer &projectPart)
{
long long lastModified = QDateTime::currentSecsSinceEpoch();
FilePathIds allExternalIncludes;
FilePathIds topExternalIncludes;
std::tie(allExternalIncludes, topExternalIncludes) = generateProjectPartPchIncludes(projectPart);
auto content = generatePchIncludeFileContent(topExternalIncludes);
auto includes = generateProjectPartPchIncludes(projectPart);
auto content = generatePchIncludeFileContent(includes.topIncludeIds);
auto pchIncludeFilePath = generateProjectPathPchHeaderFilePath(projectPart);
auto pchFilePath = generateProjectPartPchFilePath(projectPart);
generateFileWithContent(pchIncludeFilePath, content);
@@ -329,7 +327,7 @@ IdPaths PchCreator::generateProjectPartPch(const V2::ProjectPartContainer &proje
m_projectPartPch.lastModified = lastModified;
}
return {projectPart.projectPartId.clone(), std::move(allExternalIncludes)};
return {projectPart.projectPartId.clone(), std::move(includes.includeIds)};
}
void PchCreator::generatePch(const V2::ProjectPartContainer &projectPart)

View File

@@ -46,6 +46,14 @@ class GeneratedFiles;
class PchManagerClientInterface;
class ClangPathWatcherInterface;
class PchCreatorIncludes
{
public:
FilePathIds includeIds;
FilePathIds topIncludeIds;
FilePathIds topSystemIncludeIds;
};
class PchCreator final : public PchCreatorInterface
{
public:
@@ -87,7 +95,7 @@ public:
const V2::ProjectPartContainer &projectPart) const;
Utils::PathStringVector generateProjectPartSourcePaths(
const V2::ProjectPartContainer &projectPart) const;
std::pair<FilePathIds,FilePathIds> generateProjectPartPchIncludes(
PchCreatorIncludes generateProjectPartPchIncludes(
const V2::ProjectPartContainer &projectPart) const;
Utils::SmallString generateProjectPathPchHeaderFilePath(
const V2::ProjectPartContainer &projectPart) const;

View File

@@ -0,0 +1,3 @@
#pragma once
#include "indirect_external.h"

View File

@@ -0,0 +1,3 @@
#pragma once
#include "indirect_external2.h"

View File

@@ -0,0 +1 @@
#include "faulty.h"

View File

@@ -0,0 +1,3 @@
#pragma once
#include "external3.h"

View File

@@ -0,0 +1,7 @@
#if 0
#include "false.h"
#endif
#if 1
#include "true.h"
#endif

View File

@@ -0,0 +1,4 @@
#include "header1.h"
#include "header2.h"
#include "external1.h"
#include "../external/external2.h"

View File

@@ -0,0 +1,5 @@
#include "header1.h"
#include "header2.h"
#include "external1.h"
#include "../external/external2.h"
#include "generated_file.h"

View File

@@ -0,0 +1,6 @@
#include <header1.h>
#include <header2.h>
#include <external1.h>
#include <external2.h>
#include <system1.h>

View File

@@ -3,4 +3,4 @@
#include <missing_file2.moc>
#include <foo2/missing_file2.moc>
#include "includecollector_external1.h"
#include "external1.h"

View File

@@ -0,0 +1,3 @@
#pragma once
#include "indirect_system2.h"

View File

@@ -0,0 +1,2 @@
#pragma once

View File

@@ -0,0 +1,3 @@
#pragma once
#include "indirect_system.h"

View File

@@ -0,0 +1 @@
#pragma once

View File

@@ -0,0 +1 @@
#pragma once

View File

@@ -1,3 +0,0 @@
#pragma once
#include "includecollector_indirect_external.h"

View File

@@ -1 +0,0 @@
#include "includecollector_faulty.h"

View File

@@ -1,3 +0,0 @@
#pragma once
#include "includecollector_external3.h"

View File

@@ -1,7 +0,0 @@
#if 0
#include "includecollector_false.h"
#endif
#if 1
#include "includecollector_true.h"
#endif

View File

@@ -1,3 +0,0 @@
#pragma once
#include "includecollector_indirect_external2.h"

View File

@@ -1,4 +0,0 @@
#include "includecollector_header1.h"
#include "includecollector_header2.h"
#include "includecollector_external1.h"
#include "../data/includecollector_external2.h"

View File

@@ -1,5 +0,0 @@
#include "includecollector_header1.h"
#include "includecollector_header2.h"
#include "includecollector_external1.h"
#include "../data/includecollector_external2.h"
#include "includecollector_generated_file.h"

View File

@@ -1,4 +0,0 @@
#include <includecollector_header1.h>
#include <includecollector_header2.h>
#include <includecollector_external1.h>
#include <includecollector_external2.h>

View File

@@ -42,6 +42,7 @@
#include <filepath.h>
#include <fulltokeninfo.h>
#include <nativefilepath.h>
#include <pchcreator.h>
#include <precompiledheadersupdatedmessage.h>
#include <projectpartartefact.h>
#include <sourcedependency.h>
@@ -1000,6 +1001,11 @@ std::ostream &operator<<(std::ostream &out, const SymbolIndexerTask &task)
return out << "(" << task.filePathId << ", " << task.projectPartId << ")";
}
std::ostream &operator<<(std::ostream &out, const PchCreatorIncludes &includes)
{
return out << "(" << includes.includeIds << ", " << includes.topIncludeIds << ", " << includes.topSystemIncludeIds << ")";
}
void PrintTo(const FilePath &filePath, ::std::ostream *os)
{
*os << filePath;

View File

@@ -169,6 +169,7 @@ class RemoveGeneratedFilesMessage;
class SuspendResumeJobsEntry;
class ReferencesResult;
class SymbolIndexerTask;
class PchCreatorIncludes;
std::ostream &operator<<(std::ostream &out, const SourceLocationEntry &entry);
std::ostream &operator<<(std::ostream &out, const IdPaths &idPaths);
@@ -248,6 +249,7 @@ std::ostream &operator<<(std::ostream &out, const RemoveGeneratedFilesMessage &m
std::ostream &operator<<(std::ostream &os, const SuspendResumeJobsEntry &entry);
std::ostream &operator<<(std::ostream &os, const ReferencesResult &value);
std::ostream &operator<<(std::ostream &out, const SymbolIndexerTask &task);
std::ostream &operator<<(std::ostream &out, const PchCreatorIncludes &includes);
void PrintTo(const FilePath &filePath, ::std::ostream *os);
void PrintTo(const FilePathView &filePathView, ::std::ostream *os);

View File

@@ -74,27 +74,27 @@ protected:
protected:
Sqlite::Database database{":memory:", Sqlite::JournalMode::Memory};
ClangBackEnd::RefactoringDatabaseInitializer<Sqlite::Database> databaseInitializer{database};
FilePath main1Path = TESTDATA_DIR "/includecollector_main3.cpp";
FilePath main2Path = TESTDATA_DIR "/includecollector_main2.cpp";
FilePath header1Path = TESTDATA_DIR "/includecollector_header1.h";
FilePath header2Path = TESTDATA_DIR "/includecollector_header2.h";
Utils::SmallStringView generatedFileName = "includecollector_generated_file.h";
FilePath generatedFilePath = TESTDATA_DIR "/includecollector_generated_file.h";
FilePath main1Path = TESTDATA_DIR "/includecollector/project/main3.cpp";
FilePath main2Path = TESTDATA_DIR "/includecollector/project/main2.cpp";
FilePath header1Path = TESTDATA_DIR "/includecollector/project/header1.h";
FilePath header2Path = TESTDATA_DIR "/includecollector/project/header2.h";
Utils::SmallStringView generatedFileName = "includecollector/project/generated_file.h";
FilePath generatedFilePath = TESTDATA_DIR "/includecollector/project/generated_file.h";
TestEnvironment environment;
FileContainer generatedFile{{TESTDATA_DIR, generatedFileName}, "#pragma once", {}};
NiceMock<MockPchManagerClient> mockPchManagerClient;
NiceMock<MockClangPathWatcher> mockClangPathWatcher;
ClangBackEnd::PchCreator creator{environment, database, mockPchManagerClient, mockClangPathWatcher};
ProjectPartContainer projectPart1{"project1",
{"-I", TESTDATA_DIR, "-Wno-pragma-once-outside-header"},
{"-I", TESTDATA_DIR, "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system", "-Wno-pragma-once-outside-header"},
{{"DEFINE", "1"}},
{"/includes"},
{TESTDATA_DIR "/includecollector/external", TESTDATA_DIR "/includecollector/project"},
{id(header1Path)},
{id(main1Path)}};
ProjectPartContainer projectPart2{"project2",
{"-I", TESTDATA_DIR, "-x", "c++-header", "-Wno-pragma-once-outside-header"},
{"-I", TESTDATA_DIR, "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-x", "c++-header", "-Wno-pragma-once-outside-header"},
{{"DEFINE", "1"}},
{"/includes"},
{TESTDATA_DIR "/includecollector/external", TESTDATA_DIR "/includecollector/project"},
{id(header2Path)},
{id(main2Path)}};
};
@@ -112,7 +112,7 @@ TEST_F(PchCreator, CreateProjectPartCommandLine)
{
auto commandLine = creator.generateProjectPartCommandLine(projectPart1);
ASSERT_THAT(commandLine, ElementsAre(environment.clangCompilerPath(), "-I", TESTDATA_DIR, "-Wno-pragma-once-outside-header"));
ASSERT_THAT(commandLine, ElementsAre(environment.clangCompilerPath(), "-I", TESTDATA_DIR, "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system", "-Wno-pragma-once-outside-header"));
}
TEST_F(PchCreator, CreateProjectPartHeaders)
@@ -131,39 +131,41 @@ TEST_F(PchCreator, CreateProjectPartSources)
TEST_F(PchCreatorSlowTest, CreateProjectPartPchIncludes)
{
using IncludePair = decltype(creator.generateProjectPartPchIncludes(projectPart1));
using ClangBackEnd::PchCreatorIncludes;
auto includeIds = creator.generateProjectPartPchIncludes(projectPart1);
ASSERT_THAT(includeIds,
AllOf(
Field(&IncludePair::first,
AllOf(Contains(id(TESTDATA_DIR "/includecollector_external1.h")),
Contains(id(TESTDATA_DIR "/includecollector_external2.h")),
Contains(id(TESTDATA_DIR "/includecollector_header2.h")))),
Field(&IncludePair::second,
AllOf(Contains(id(TESTDATA_DIR "/includecollector_external1.h")),
Contains(id(TESTDATA_DIR "/includecollector_external2.h"))))));
Field(&PchCreatorIncludes::includeIds,
AllOf(Contains(id(TESTDATA_DIR "/includecollector/external/external1.h")),
Contains(id(TESTDATA_DIR "/includecollector/external/external2.h")),
Contains(id(TESTDATA_DIR "/includecollector/project/header2.h")),
Contains(id(TESTDATA_DIR "/includecollector/system/system1.h")))),
Field(&PchCreatorIncludes::topSystemIncludeIds,
AllOf(Contains(id(TESTDATA_DIR "/includecollector/system/system1.h")),
Not(Contains(id(TESTDATA_DIR "/includecollector/system/indirect_system.h"))))),
Field(&PchCreatorIncludes::topIncludeIds,
AllOf(Contains(id(TESTDATA_DIR "/includecollector/external/external1.h")),
Contains(id(TESTDATA_DIR "/includecollector/external/external2.h"))))));
}
TEST_F(PchCreatorSlowTest, CreateProjectPartPchFileContent)
{
FilePathIds topExternalIncludes;
std::tie(std::ignore, topExternalIncludes) = creator.generateProjectPartPchIncludes(projectPart1);
auto includes = creator.generateProjectPartPchIncludes(projectPart1);
auto content = creator.generatePchIncludeFileContent(topExternalIncludes);
auto content = creator.generatePchIncludeFileContent(includes.topIncludeIds);
ASSERT_THAT(std::string(content),
AllOf(HasSubstr("#include \"" TESTDATA_DIR "/includecollector_header2.h\"\n"),
HasSubstr("#include \"" TESTDATA_DIR "/includecollector_external1.h\"\n"),
HasSubstr("#include \"" TESTDATA_DIR "/includecollector_external2.h\"\n")));
AllOf(HasSubstr("#include \"" TESTDATA_DIR "/includecollector/project/header2.h\"\n"),
HasSubstr("#include \"" TESTDATA_DIR "/includecollector/external/external1.h\"\n"),
HasSubstr("#include \"" TESTDATA_DIR "/includecollector/external/external2.h\"\n")));
}
TEST_F(PchCreatorSlowTest, CreateProjectPartPchIncludeFile)
{
FilePathIds topExternalIncludes;
std::tie(std::ignore, topExternalIncludes) = creator.generateProjectPartPchIncludes(projectPart1);
auto content = creator.generatePchIncludeFileContent(topExternalIncludes);
auto includes = creator.generateProjectPartPchIncludes(projectPart1);
auto content = creator.generatePchIncludeFileContent(includes.topIncludeIds);
auto pchIncludeFilePath = creator.generateProjectPathPchHeaderFilePath(projectPart1);
auto file = creator.generateFileWithContent(pchIncludeFilePath, content);
file->open(QIODevice::ReadOnly);
@@ -171,9 +173,9 @@ TEST_F(PchCreatorSlowTest, CreateProjectPartPchIncludeFile)
auto fileContent = file->readAll();
ASSERT_THAT(fileContent.toStdString(),
AllOf(HasSubstr("#include \"" TESTDATA_DIR "/includecollector_header2.h\"\n"),
HasSubstr("#include \"" TESTDATA_DIR "/includecollector_external1.h\"\n"),
HasSubstr("#include \"" TESTDATA_DIR "/includecollector_external2.h\"\n")));
AllOf(HasSubstr("#include \"" TESTDATA_DIR "/includecollector/project/header2.h\"\n"),
HasSubstr("#include \"" TESTDATA_DIR "/includecollector/external/external1.h\"\n"),
HasSubstr("#include \"" TESTDATA_DIR "/includecollector/external/external2.h\"\n")));
}
TEST_F(PchCreator, CreateProjectPartPchCompilerArguments)
@@ -233,9 +235,9 @@ TEST_F(PchCreatorVerySlowTest, IdPathsForCreatePchsForProjectParts)
ASSERT_THAT(creator.takeProjectIncludes(),
AllOf(Field(&IdPaths::id, "project1"),
Field(&IdPaths::filePathIds, AllOf(Contains(id(TESTDATA_DIR "/includecollector_header2.h")),
Contains(id(TESTDATA_DIR "/includecollector_external1.h")),
Contains(id(TESTDATA_DIR "/includecollector_external2.h"))))));
Field(&IdPaths::filePathIds, AllOf(Contains(id(TESTDATA_DIR "/includecollector/project/header2.h")),
Contains(id(TESTDATA_DIR "/includecollector/external/external1.h")),
Contains(id(TESTDATA_DIR "/includecollector/external/external2.h"))))));
}
TEST_F(PchCreatorVerySlowTest, ProjectPartPchForCreatesPchForProjectPart)
@@ -274,7 +276,7 @@ TEST_F(PchCreatorVerySlowTest, FaultyProjectPartPchForCreatesNoPchForProjectPart
{{"DEFINE", "1"}},
{"/includes"},
{},
{id(TESTDATA_DIR "/includecollector_faulty.cpp")}};
{id(TESTDATA_DIR "/includecollector/project/faulty.cpp")}};
creator.generatePch(faultyProjectPart);
@@ -288,14 +290,14 @@ TEST_F(PchCreator, CreateProjectPartSourcesContent)
{
auto content = creator.generateProjectPartSourcesContent(projectPart1);
ASSERT_THAT(content, Eq("#include \"" TESTDATA_DIR "/includecollector_main3.cpp\"\n"));
ASSERT_THAT(content, Eq("#include \"" TESTDATA_DIR "/includecollector/project/main3.cpp\"\n"));
}
TEST_F(PchCreator, Call)
{
auto content = creator.generateProjectPartSourcesContent(projectPart1);
ASSERT_THAT(content, Eq("#include \"" TESTDATA_DIR "/includecollector_main3.cpp\"\n"));
ASSERT_THAT(content, Eq("#include \"" TESTDATA_DIR "/includecollector/project/main3.cpp\"\n"));
}