Clang: Improve performance by reducing parsing

We generate one big file per project part so the preprocessor is skipping
the recurring includes.

This generated many errors about missing macros but we don't care
much about them during dependency collection step so we just
silence these errors with ignoring diagnostics consumer.

Change-Id: I5581d623b5d5f9995496252735577ea6b54790d9
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>

src/tools/clangpchmanagerbackend/clangpchmanagerbackendmain.cpp

@@ -191,7 +191,9 @@ struct Data // because we have a cycle dependency
                                             database};
     ClangBackEnd::PchTasksMerger pchTaskMerger{pchTaskQueue};
     ClangBackEnd::BuildDependenciesStorage<> buildDependencyStorage{database};
-    ClangBackEnd::BuildDependencyCollector buildDependencyCollector{filePathCache, generatedFiles};
+    ClangBackEnd::BuildDependencyCollector buildDependencyCollector{filePathCache,
+                                                                    generatedFiles,
+                                                                    environment};
     std::function<TimeStamp(FilePathView filePath)> getModifiedTime{
         [&](ClangBackEnd::FilePathView path) -> TimeStamp {
             return QFileInfo(QString(path)).lastModified().toSecsSinceEpoch();

src/tools/clangpchmanagerbackend/source/builddependencycollector.cpp

@@ -28,10 +28,12 @@
 #include "collectbuilddependencytoolaction.h"
 #include "commandlinebuilder.h"
 
+#include <environment.h>
+
 #include <utils/smallstring.h>
 
 #include <algorithm>
-
+#include <iostream>
 namespace ClangBackEnd {
 
 namespace {
@@ -50,7 +52,7 @@ BuildDependency BuildDependencyCollector::create(const ProjectPartContainer &pro
     CommandLineBuilder<ProjectPartContainer, Utils::SmallStringVector>
         builder{projectPart, projectPart.toolChainArguments, InputFileType::Source};
 
-    addFiles(projectPart.sourcePathIds, builder.commandLine);
+    addFiles(projectPart.sourcePathIds, std::move(builder.commandLine));
 
     setExcludedFilePaths(
         m_filePathCache.filePaths(projectPart.headerPathIds + projectPart.sourcePathIds));
@@ -66,6 +68,33 @@ BuildDependency BuildDependencyCollector::create(const ProjectPartContainer &pro
     return buildDependency;
 }
 
+namespace {
+
+std::size_t contentSize(const FilePaths &includes)
+{
+    auto countIncludeSize = [](std::size_t size, const auto &include) {
+        return size + include.size();
+    };
+
+    return std::accumulate(includes.begin(), includes.end(), std::size_t(0), countIncludeSize);
+}
+} // namespace
+
+Utils::SmallString BuildDependencyCollector::generateFakeFileContent(
+    const FilePathIds &includeIds) const
+{
+    Utils::SmallString fileContent;
+    const std::size_t lineTemplateSize = 12;
+    auto includes = m_filePathCache.filePaths(includeIds);
+
+    fileContent.reserve(includes.size() * lineTemplateSize + contentSize(includes));
+
+    for (Utils::SmallStringView include : includes)
+        fileContent += {"#include \"", include, "\"\n"};
+
+    return fileContent;
+}
+
 void BuildDependencyCollector::collect()
 {
     clang::tooling::ClangTool tool = m_clangTool.createTool();
@@ -96,25 +125,26 @@ void BuildDependencyCollector::setExcludedFilePaths(ClangBackEnd::FilePaths &&ex
 }
 
 void BuildDependencyCollector::addFiles(const FilePathIds &filePathIds,
-                                        const Utils::SmallStringVector &arguments)
+                                        Utils::SmallStringVector &&arguments)
 {
-    m_clangTool.addFiles(m_filePathCache.filePaths(filePathIds), arguments);
+    m_clangTool.addFile(FilePath{m_environment.pchBuildDirectory().toStdString(), "dummy.cpp"},
+                        generateFakeFileContent(filePathIds),
+                        std::move(arguments));
     m_buildDependency.sourceFiles.insert(m_buildDependency.sourceFiles.end(),
                                          filePathIds.begin(),
                                          filePathIds.end());
 }
 
-void BuildDependencyCollector::addFile(FilePathId filePathId,
+void BuildDependencyCollector::addFile(FilePathId filePathId, Utils::SmallStringVector &&arguments)
-                                       const Utils::SmallStringVector &arguments)
 {
-    addFiles({filePathId}, arguments);
+    addFiles({filePathId}, std::move(arguments));
 }
 
 void BuildDependencyCollector::addFile(FilePath filePath,
                                        const FilePathIds &sourceFileIds,
-                                       const Utils::SmallStringVector &arguments)
+                                       Utils::SmallStringVector &&arguments)
 {
-    m_clangTool.addFiles({filePath}, arguments);
+    m_clangTool.addFiles({filePath}, std::move(arguments));
     m_buildDependency.sourceFiles.insert(m_buildDependency.sourceFiles.end(),
                                          sourceFileIds.begin(),
                                          sourceFileIds.end());

src/tools/clangpchmanagerbackend/source/builddependencycollector.h

@@ -34,33 +34,35 @@
 #include <filepathcachingfwd.h>
 
 namespace ClangBackEnd {
+class Environment;
 
 class BuildDependencyCollector : public BuildDependencyGeneratorInterface
 {
 public:
     BuildDependencyCollector(const FilePathCachingInterface &filePathCache,
-                             const GeneratedFilesInterface &generatedFiles)
+                             const GeneratedFilesInterface &generatedFiles,
+                             const Environment &environment)
         : m_filePathCache(filePathCache)
         , m_generatedFiles(generatedFiles)
-    {
+        , m_environment(environment)
-    }
+    {}
 
     BuildDependency create(const ProjectPartContainer &projectPart) override;
 
     void collect();
 
     void setExcludedFilePaths(ClangBackEnd::FilePaths &&excludedIncludes);
-    void addFiles(const FilePathIds &filePathIds,
+    void addFiles(const FilePathIds &filePathIds, Utils::SmallStringVector &&arguments);
-                  const Utils::SmallStringVector &arguments);
+    void addFile(FilePathId filePathId, Utils::SmallStringVector &&arguments);
-    void addFile(FilePathId filePathId,
-                 const Utils::SmallStringVector &arguments);
     void addFile(FilePath filePath,
                  const FilePathIds &sourceFileIds,
-                 const Utils::SmallStringVector &arguments);
+                 Utils::SmallStringVector &&arguments);
     void addUnsavedFiles(const V2::FileContainers &unsavedFiles);
 
     void clear();
 
+    Utils::SmallString generateFakeFileContent(const FilePathIds &includeIds) const;
+
     const FileStatuses &fileStatuses() const
     {
         return m_buildDependency.fileStatuses;
@@ -96,6 +98,7 @@ private:
     SourcesManager m_sourcesManager;
     const FilePathCachingInterface &m_filePathCache;
     const GeneratedFilesInterface &m_generatedFiles;
+    const Environment &m_environment;
 };
 
 } // namespace ClangBackEnd

src/tools/clangpchmanagerbackend/source/collectbuilddependencyaction.h

@@ -54,27 +54,28 @@ public:
 
     bool BeginSourceFileAction(clang::CompilerInstance &compilerInstance) override
     {
-      if (clang::PreprocessOnlyAction::BeginSourceFileAction(compilerInstance)) {
+        if (clang::PreprocessOnlyAction::BeginSourceFileAction(compilerInstance)) {
-          auto &preprocessor = compilerInstance.getPreprocessor();
+            auto &preprocessor = compilerInstance.getPreprocessor();
 
-          preprocessor.SetSuppressIncludeNotFoundError(true);
+            preprocessor.SetSuppressIncludeNotFoundError(true);
-          preprocessor.SetMacroExpansionOnlyInDirectives();
+            preprocessor.SetMacroExpansionOnlyInDirectives();
 
-          auto macroPreprocessorCallbacks = new CollectBuildDependencyPreprocessorCallbacks(
+            auto macroPreprocessorCallbacks = new CollectBuildDependencyPreprocessorCallbacks(
-                      m_buildDependency,
+                m_buildDependency,
-                      m_filePathCache,
+                m_filePathCache,
-                      m_excludedIncludeUID,
+                m_excludedIncludeUID,
-                      m_alreadyIncludedFileUIDs,
+                m_alreadyIncludedFileUIDs,
-                      compilerInstance.getSourceManager(),
+                compilerInstance.getSourceManager(),
-                      m_sourcesManager,
+                m_sourcesManager,
-                      compilerInstance.getPreprocessorPtr());
+                compilerInstance.getPreprocessorPtr());
 
-          preprocessor.addPPCallbacks(std::unique_ptr<clang::PPCallbacks>(macroPreprocessorCallbacks));
+            preprocessor.addPPCallbacks(
+                std::unique_ptr<clang::PPCallbacks>(macroPreprocessorCallbacks));
 
-          return true;
+            return true;
-      }
+        }
 
-      return false;
+        return false;
     }
 
     void EndSourceFileAction() override

src/tools/clangpchmanagerbackend/source/collectbuilddependencypreprocessorcallbacks.h

@@ -72,15 +72,27 @@ public:
     void FileChanged(clang::SourceLocation sourceLocation,
                      clang::PPCallbacks::FileChangeReason reason,
                      clang::SrcMgr::CharacteristicKind,
-                     clang::FileID) override
+                     clang::FileID previousFileId) override
     {
-        if (reason == clang::PPCallbacks::EnterFile)
+        if (reason == clang::PPCallbacks::EnterFile) {
-        {
+            clang::FileID currentFileId = m_sourceManager->getFileID(sourceLocation);
-            const clang::FileEntry *fileEntry = m_sourceManager->getFileEntryForID(
+            if (m_mainFileId.isInvalid()) {
-                        m_sourceManager->getFileID(sourceLocation));
+                m_mainFileId = currentFileId;
-            if (fileEntry) {
+            } else {
-                addFileStatus(fileEntry);
+                const clang::FileEntry *fileEntry = m_sourceManager->getFileEntryForID(
-                addSourceFile(fileEntry);
+                    currentFileId);
+                if (fileEntry) {
+                    if (previousFileId == m_mainFileId) {
+                        uint sourceFileUID = fileEntry->getUID();
+                        auto notAlreadyIncluded = isNotAlreadyIncluded(sourceFileUID);
+                        if (notAlreadyIncluded.first)
+                            m_alreadyIncludedFileUIDs.insert(notAlreadyIncluded.second,
+                                                             sourceFileUID);
+                    } else {
+                        addFileStatus(fileEntry);
+                        addSourceFile(fileEntry);
+                    }
+                }
             }
         }
     }
@@ -96,7 +108,8 @@ public:
                             const clang::Module * /*imported*/,
                             clang::SrcMgr::CharacteristicKind fileType) override
     {
-        if (file) {
+        clang::FileID currentFileId = m_sourceManager->getFileID(hashLocation);
+        if (file && currentFileId != m_mainFileId) {
             addSourceDependency(file, hashLocation);
             auto fileUID = file->getUID();
             auto sourceFileUID = m_sourceManager
@@ -343,6 +356,7 @@ private:
     BuildDependency &m_buildDependency;
     const std::vector<uint> &m_excludedIncludeUID;
     std::vector<uint> &m_alreadyIncludedFileUIDs;
+    clang::FileID m_mainFileId;
 };
 
 } // namespace ClangBackEnd

tests/unit/unittest/builddependencycollector-test.cpp

@@ -25,6 +25,8 @@
 
 #include "googletest.h"
 
+#include "testenvironment.h"
+
 #include <refactoringdatabaseinitializer.h>
 #include <filepathcaching.h>
 #include <generatedfiles.h>
@@ -67,22 +69,15 @@ protected:
     {
         setFilePathCache(&filePathCache);
 
-        collector.addFile(id(TESTDATA_DIR "/builddependencycollector/project/main.cpp"),
+        collector.addFiles({id(TESTDATA_DIR "/builddependencycollector/project/main.cpp"),
-                          {"cc",
+                            id(TESTDATA_DIR "/builddependencycollector/project/main2.cpp")},
-                           "-I",
+                           {"cc",
-                           TESTDATA_DIR "/builddependencycollector/external",
+                            "-I",
-                           "-I",
+                            TESTDATA_DIR "/builddependencycollector/external",
-                           TESTDATA_DIR "/builddependencycollector/project",
+                            "-I",
-                           "-isystem",
+                            TESTDATA_DIR "/builddependencycollector/project",
-                           TESTDATA_DIR "/builddependencycollector/system"});
+                            "-isystem",
-        collector.addFile(id(TESTDATA_DIR "/builddependencycollector/project/main2.cpp"),
+                            TESTDATA_DIR "/builddependencycollector/system"});
-                          {"cc",
-                           "-I",
-                           TESTDATA_DIR "/builddependencycollector/external",
-                           "-I",
-                           TESTDATA_DIR "/builddependencycollector/project",
-                           "-isystem",
-                           TESTDATA_DIR "/builddependencycollector/system"});
 
         collector.addUnsavedFiles(
             {{{TESTDATA_DIR, "BuildDependencyCollector/project/generated_file.h"},
@@ -160,16 +155,18 @@ protected:
 
 protected:
     Sqlite::Database database{":memory:", Sqlite::JournalMode::Memory};
+    TestEnvironment environment;
     ClangBackEnd::RefactoringDatabaseInitializer<Sqlite::Database> databaseInitializer{database};
     ClangBackEnd::FilePathCaching filePathCache{database};
     ClangBackEnd::GeneratedFiles generatedFiles;
-    ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles};
+    ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles, environment};
-    ClangBackEnd::BuildDependencyCollector emptyCollector{filePathCache, generatedFiles};
+    ClangBackEnd::BuildDependencyCollector emptyCollector{filePathCache, generatedFiles, environment};
-    ClangBackEnd::FilePaths excludePaths = {TESTDATA_DIR "/builddependencycollector/project/main.cpp",
+    ClangBackEnd::FilePaths excludePaths = {
-                                            TESTDATA_DIR "/builddependencycollector/project/main2.cpp",
+        TESTDATA_DIR "/builddependencycollector/project/main.cpp",
-                                            TESTDATA_DIR "/builddependencycollector/project/header1.h",
+        TESTDATA_DIR "/builddependencycollector/project/main2.cpp",
-                                            TESTDATA_DIR "/builddependencycollector/project/header2.h",
+        TESTDATA_DIR "/builddependencycollector/project/header1.h",
-                                            TESTDATA_DIR "/builddependencycollector/project/generated_file.h"};
+        TESTDATA_DIR "/builddependencycollector/project/header2.h",
+        TESTDATA_DIR "/builddependencycollector/project/generated_file.h"};
 };
 
 TEST_F(BuildDependencyCollector, IncludesExternalHeader)
@@ -548,10 +545,26 @@ TEST_F(BuildDependencyCollector, GeneratedFile)
                                SourceType::UserInclude)));
 }
 
+TEST_F(BuildDependencyCollector, CreateFakeFileContent)
+{
+    auto content = collector.generateFakeFileContent(
+        {id(TESTDATA_DIR "/builddependencycollector/project/header2.h"),
+         id(TESTDATA_DIR "/builddependencycollector/external/external1.h"),
+         id(TESTDATA_DIR "/builddependencycollector/external/external2.h")});
+
+    ASSERT_THAT(std::string(content),
+                AllOf(HasSubstr("#include \"" TESTDATA_DIR
+                                "/builddependencycollector/project/header2.h\"\n"),
+                      HasSubstr("#include \"" TESTDATA_DIR
+                                "/builddependencycollector/external/external1.h\"\n"),
+                      HasSubstr("#include \"" TESTDATA_DIR
+                                "/builddependencycollector/external/external2.h\"\n")));
+}
+
 TEST_F(BuildDependencyCollector, Create)
 {
     using ClangBackEnd::IncludeSearchPathType;
-    ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles};
+    ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles, environment};
     generatedFiles.update(
         {{TESTDATA_DIR "/builddependencycollector/project/generated_file.h", "#pragma once"}});
     ClangBackEnd::ProjectPartContainer projectPart{
@@ -702,7 +715,7 @@ TEST_F(BuildDependencyCollector, Create)
 TEST_F(BuildDependencyCollector, Clear)
 {
     using ClangBackEnd::IncludeSearchPathType;
-    ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles};
+    ClangBackEnd::BuildDependencyCollector collector{filePathCache, generatedFiles, environment};
     ClangBackEnd::ProjectPartContainer projectPart{
         "project1",
         {},