ClangPchManager: Add build dependencies to include collector

It's a first step but we have to refactor the interface later.

Task-number: QTCREATORBUG-21379
Change-Id: Idda666bcaec950203f001c993c54a926779527e0
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>

src/tools/clangpchmanagerbackend/source/collectincludesaction.h

@@ -44,13 +44,23 @@ public:
                           FilePathIds &topsSystemIncludeIds,
                           const FilePathCachingInterface &filePathCache,
                           std::vector<uint> &excludedIncludeUID,
-                          std::vector<uint> &alreadyIncludedFileUIDs)
+                          std::vector<uint> &alreadyIncludedFileUIDs,
+                          UsedMacros &usedMacros,
+                          SourcesManager &sourcesManager,
+                          SourceDependencies &sourceDependencies,
+                          FilePathIds &sourceFiles,
+                          FileStatuses &fileStatuses)
         : m_includeIds(includeIds),
           m_topIncludeIds(topIncludeIds),
           m_topsSystemIncludeIds(topsSystemIncludeIds),
           m_filePathCache(filePathCache),
           m_excludedIncludeUID(excludedIncludeUID),
-          m_alreadyIncludedFileUIDs(alreadyIncludedFileUIDs)
+          m_alreadyIncludedFileUIDs(alreadyIncludedFileUIDs),
+          m_usedMacros(usedMacros),
+          m_sourcesManager(sourcesManager),
+          m_sourceDependencies(sourceDependencies),
+          m_sourceFiles(sourceFiles),
+          m_fileStatuses(fileStatuses)
     {
     }
 
@@ -68,7 +78,13 @@ public:
                       m_filePathCache,
                       m_excludedIncludeUID,
                       m_alreadyIncludedFileUIDs,
-                      compilerInstance.getSourceManager());
+                      compilerInstance.getSourceManager(),
+                      m_usedMacros,
+                      m_sourcesManager,
+                      compilerInstance.getPreprocessorPtr(),
+                      m_sourceDependencies,
+                      m_sourceFiles,
+                      m_fileStatuses);
 
           preprocessor.addPPCallbacks(std::unique_ptr<clang::PPCallbacks>(macroPreprocessorCallbacks));
 
@@ -90,6 +106,11 @@ private:
     const FilePathCachingInterface &m_filePathCache;
     std::vector<uint> &m_excludedIncludeUID;
     std::vector<uint> &m_alreadyIncludedFileUIDs;
+    UsedMacros &m_usedMacros;
+    SourcesManager &m_sourcesManager;
+    SourceDependencies &m_sourceDependencies;
+    FilePathIds &m_sourceFiles;
+    FileStatuses &m_fileStatuses;
 };
 
 } // namespace ClangBackEnd

src/tools/clangpchmanagerbackend/source/collectincludespreprocessorcallbacks.h

@@ -25,18 +25,14 @@
 
 #pragma once
 
+#include "collectmacrospreprocessorcallbacks.h"
 #include "sourcelocationsutils.h"
+
 #include <filepathcachinginterface.h>
 #include <filepathid.h>
 
 #include <utils/smallstringvector.h>
 
-#include <clang/Basic/SourceManager.h>
-#include <clang/Lex/MacroInfo.h>
-#include <clang/Lex/HeaderSearch.h>
-#include <clang/Lex/PPCallbacks.h>
-#include <clang/Lex/Preprocessor.h>
-
 #include <QFile>
 #include <QDir>
 #include <QTemporaryDir>
@@ -45,7 +41,8 @@
 
 namespace ClangBackEnd {
 
-class CollectIncludesPreprocessorCallbacks final : public clang::PPCallbacks
+class CollectIncludesPreprocessorCallbacks final : public clang::PPCallbacks,
+                                                   public CollectUsedMacrosAndSourcesPreprocessorCallbacksBase
 {
 public:
     CollectIncludesPreprocessorCallbacks(FilePathIds &includeIds,
@@ -54,16 +51,44 @@ public:
                                          const FilePathCachingInterface &filePathCache,
                                          const std::vector<uint> &excludedIncludeUID,
                                          std::vector<uint> &alreadyIncludedFileUIDs,
-                                         clang::SourceManager &sourceManager)
+                                         const clang::SourceManager &sourceManager,
-        : m_includeIds(includeIds),
+                                         UsedMacros &usedMacros,
+                                         SourcesManager &sourcesManager,
+                                         std::shared_ptr<clang::Preprocessor> preprocessor,
+                                         SourceDependencies &sourceDependencies,
+                                         FilePathIds &sourceFiles,
+                                         FileStatuses &fileStatuses)
+        : CollectUsedMacrosAndSourcesPreprocessorCallbacksBase(usedMacros,
+                                                               filePathCache,
+                                                               sourceManager,
+                                                               sourcesManager,
+                                                               preprocessor,
+                                                               sourceDependencies,
+                                                               sourceFiles,
+                                                               fileStatuses),
+          m_includeIds(includeIds),
           m_topIncludeIds(topIncludeIds),
           m_topsSystemIncludeIds(topsSystemIncludeIds),
-          m_filePathCache(filePathCache),
           m_excludedIncludeUID(excludedIncludeUID),
-          m_alreadyIncludedFileUIDs(alreadyIncludedFileUIDs),
+          m_alreadyIncludedFileUIDs(alreadyIncludedFileUIDs)
-          m_sourceManager(sourceManager)
     {}
 
+    void FileChanged(clang::SourceLocation sourceLocation,
+                     clang::PPCallbacks::FileChangeReason reason,
+                     clang::SrcMgr::CharacteristicKind,
+                     clang::FileID) override
+    {
+        if (reason == clang::PPCallbacks::EnterFile)
+        {
+            const clang::FileEntry *fileEntry = m_sourceManager->getFileEntryForID(
+                        m_sourceManager->getFileID(sourceLocation));
+            if (fileEntry) {
+                addFileStatus(fileEntry);
+                addSourceFile(fileEntry);
+            }
+        }
+    }
+
     void InclusionDirective(clang::SourceLocation hashLocation,
                             const clang::Token &/*includeToken*/,
                             llvm::StringRef /*fileName*/,
@@ -76,8 +101,9 @@ public:
                             clang::SrcMgr::CharacteristicKind fileType) override
     {
         if (!m_skipInclude && file) {
+            addSourceDependency(file, hashLocation);
             auto fileUID = file->getUID();
-            auto sourceFileUID = m_sourceManager.getFileEntryForID(m_sourceManager.getFileID(hashLocation))->getUID();
+            auto sourceFileUID = m_sourceManager->getFileEntryForID(m_sourceManager->getFileID(hashLocation))->getUID();
             if (isNotInExcludedIncludeUID(fileUID)) {
                 auto notAlreadyIncluded = isNotAlreadyIncluded(fileUID);
                 if (notAlreadyIncluded.first) {
@@ -117,14 +143,41 @@ public:
         return true;
     }
 
-    bool isSystem(clang::SrcMgr::CharacteristicKind kind)
+
+    void Ifndef(clang::SourceLocation,
+                const clang::Token &macroNameToken,
+                const clang::MacroDefinition &macroDefinition) override
     {
-        return kind != clang::SrcMgr::C_User && kind != clang::SrcMgr::C_User_ModuleMap;
+        addUsedMacro(macroNameToken, macroDefinition);
     }
 
-    bool isInSystemHeader(clang::SourceLocation location)
+    void Ifdef(clang::SourceLocation,
+               const clang::Token &macroNameToken,
+               const clang::MacroDefinition &macroDefinition) override
     {
-        return m_sourceManager.isInSystemHeader(location);
+        addUsedMacro( macroNameToken, macroDefinition);
+    }
+
+    void Defined(const clang::Token &macroNameToken,
+                 const clang::MacroDefinition &macroDefinition,
+                 clang::SourceRange) override
+    {
+        addUsedMacro(macroNameToken, macroDefinition);
+    }
+
+    void MacroExpands(const clang::Token &macroNameToken,
+                      const clang::MacroDefinition &macroDefinition,
+                      clang::SourceRange,
+                      const clang::MacroArgs *) override
+    {
+        addUsedMacro(macroNameToken, macroDefinition);
+    }
+
+    void EndOfMainFile() override
+    {
+        filterOutHeaderGuards();
+        mergeUsedMacros();
+        m_sourcesManager.updateModifiedTimeStamps();
     }
 
     void ensureDirectory(const QString &directory, const QString &fileName)
@@ -175,10 +228,8 @@ private:
     FilePathIds &m_includeIds;
     FilePathIds &m_topIncludeIds;
     FilePathIds &m_topsSystemIncludeIds;
-    const FilePathCachingInterface &m_filePathCache;
     const std::vector<uint> &m_excludedIncludeUID;
     std::vector<uint> &m_alreadyIncludedFileUIDs;
-    clang::SourceManager &m_sourceManager;
     bool m_skipInclude = false;
 };
 

src/tools/clangpchmanagerbackend/source/collectincludestoolaction.h

@@ -41,12 +41,22 @@ public:
                               FilePathIds &topIncludeIds,
                               FilePathIds &topsSystemIncludeIds,
                               const FilePathCachingInterface &filePathCache,
-                              const Utils::PathStringVector &excludedIncludes)
+                              const Utils::PathStringVector &excludedIncludes,
+                              UsedMacros &usedMacros,
+                              SourcesManager &sourcesManager,
+                              SourceDependencies &sourceDependencies,
+                              FilePathIds &sourceFiles,
+                              FileStatuses &fileStatuses)
         : m_includeIds(includeIds),
           m_topIncludeIds(topIncludeIds),
           m_topsSystemIncludeIds(topsSystemIncludeIds),
           m_filePathCache(filePathCache),
-          m_excludedIncludes(excludedIncludes)
+          m_excludedIncludes(excludedIncludes),
+          m_usedMacros(usedMacros),
+          m_sourcesManager(sourcesManager),
+          m_sourceDependencies(sourceDependencies),
+          m_sourceFiles(sourceFiles),
+          m_fileStatuses(fileStatuses)
     {}
 
 
@@ -71,7 +81,12 @@ public:
                                          m_topsSystemIncludeIds,
                                          m_filePathCache,
                                          m_excludedIncludeUIDs,
-                                         m_alreadyIncludedFileUIDs);
+                                         m_alreadyIncludedFileUIDs,
+                                         m_usedMacros,
+                                         m_sourcesManager,
+                                         m_sourceDependencies,
+                                         m_sourceFiles,
+                                         m_fileStatuses);
     }
 
     std::vector<uint> generateExcludedIncludeFileUIDs(clang::FileManager &fileManager) const
@@ -99,6 +114,11 @@ private:
     FilePathIds &m_topsSystemIncludeIds;
     const FilePathCachingInterface &m_filePathCache;
     const Utils::PathStringVector &m_excludedIncludes;
+    UsedMacros &m_usedMacros;
+    SourcesManager &m_sourcesManager;
+    SourceDependencies &m_sourceDependencies;
+    FilePathIds &m_sourceFiles;
+    FileStatuses &m_fileStatuses;
 };
 
 } // namespace ClangBackEnd

src/tools/clangpchmanagerbackend/source/collectusedmacrosandsourcespreprocessorcallbacks.h

@@ -52,7 +52,7 @@ class CollectUsedMacrosAndSourcesPreprocessorCallbacksBase : public SymbolsVisit
 {
 public:
     CollectUsedMacrosAndSourcesPreprocessorCallbacksBase(UsedMacros &usedMacros,
-                                                         FilePathCachingInterface &filePathCache,
+                                                         const FilePathCachingInterface &filePathCache,
                                                          const clang::SourceManager &sourceManager,
                                                          SourcesManager &sourcesManager,
                                                          std::shared_ptr<clang::Preprocessor> preprocessor,

src/tools/clangpchmanagerbackend/source/includecollector.cpp

@@ -33,21 +33,21 @@
 
 namespace ClangBackEnd {
 
-IncludeCollector::IncludeCollector(const FilePathCachingInterface &filePathCache)
+void IncludeCollector::collect()
-    :  m_filePathCache(filePathCache)
 {
-}
+    clang::tooling::ClangTool tool = m_clangTool.createTool();
-
-void IncludeCollector::collectIncludes()
-{
-    clang::tooling::ClangTool tool = createTool();
 
     auto action = std::unique_ptr<CollectIncludesToolAction>(
                 new CollectIncludesToolAction(m_includeIds,
                                               m_topIncludeIds,
                                               m_topsSystemIncludeIds,
                                               m_filePathCache,
-                                              m_excludedIncludes));
+                                              m_excludedIncludes,
+                                              m_usedMacros,
+                                              m_sourcesManager,
+                                              m_sourceDependencies,
+                                              m_sourceFiles,
+                                              m_fileStatuses));
 
     tool.run(action.get());
 }
@@ -71,25 +71,33 @@ void IncludeCollector::setExcludedIncludes(Utils::PathStringVector &&excludedInc
 #endif
 }
 
-FilePathIds IncludeCollector::takeIncludeIds()
+void IncludeCollector::addFiles(const FilePathIds &filePathIds,
+                                const Utils::SmallStringVector &arguments)
 {
-    std::sort(m_includeIds.begin(), m_includeIds.end());
+    m_clangTool.addFiles(m_filePathCache.filePaths(filePathIds), arguments);
-
+    m_sourceFiles.insert(m_sourceFiles.end(), filePathIds.begin(), filePathIds.end());
-    return std::move(m_includeIds);
 }
 
-FilePathIds IncludeCollector::takeTopIncludeIds()
+void IncludeCollector::addFile(FilePathId filePathId, const Utils::SmallStringVector &arguments)
 {
-    std::sort(m_topIncludeIds.begin(), m_topIncludeIds.end());
+    addFiles({filePathId}, arguments);
-
-    return std::move(m_topIncludeIds);
 }
 
-FilePathIds IncludeCollector::takeTopsSystemIncludeIds()
+void IncludeCollector::addFile(FilePath filePath,
+                               const FilePathIds &sourceFileIds,
+                               const Utils::SmallStringVector &arguments)
 {
-    return std::move(m_topsSystemIncludeIds);
+    m_clangTool.addFiles({filePath}, arguments);
+    m_sourceFiles.insert(m_sourceFiles.end(), sourceFileIds.begin(), sourceFileIds.end());
 }
 
-
+void IncludeCollector::clear()
+{
+    m_clangTool = ClangTool();
+    m_usedMacros.clear();
+    m_sourceFiles.clear();
+    m_fileStatuses.clear();
+    m_sourceDependencies.clear();
+}
 
 } // namespace ClangBackEnd

src/tools/clangpchmanagerbackend/source/includecollector.h

@@ -26,6 +26,10 @@
 #pragma once
 
 #include <clangtool.h>
+#include <filestatus.h>
+#include <sourcedependency.h>
+#include <sourcesmanager.h>
+#include <usedmacro.h>
 
 #include <filepathcachingfwd.h>
 
@@ -34,22 +38,77 @@ namespace ClangBackEnd {
 class IncludeCollector : public ClangTool
 {
 public:
-    IncludeCollector(const FilePathCachingInterface &filePathCache);
+    IncludeCollector(const FilePathCachingInterface &filePathCache)
+        :  m_filePathCache(filePathCache)
+    {
+    }
 
-    void collectIncludes();
+    void collect();
 
     void setExcludedIncludes(Utils::PathStringVector &&excludedIncludes);
+    void addFiles(const FilePathIds &filePathIds,
+                  const Utils::SmallStringVector &arguments);
+    void addFile(FilePathId filePathId,
+                 const Utils::SmallStringVector &arguments);
+    void addFile(FilePath filePath,
+                 const FilePathIds &sourceFileIds,
+                 const Utils::SmallStringVector &arguments);
 
-    FilePathIds takeIncludeIds();
+    void clear();
-    FilePathIds takeTopIncludeIds();
+
-    FilePathIds takeTopsSystemIncludeIds();
+    const FileStatuses &fileStatuses() const
+    {
+        return m_fileStatuses;
+    }
+
+    const FilePathIds &sourceFiles() const
+    {
+        return m_sourceFiles;
+    }
+
+    const UsedMacros &usedMacros() const
+    {
+        return m_usedMacros;
+    }
+
+    const SourceDependencies &sourceDependencies() const
+    {
+        return m_sourceDependencies;
+    }
+
+    FilePathIds takeIncludeIds()
+    {
+        std::sort(m_includeIds.begin(), m_includeIds.end());
+
+        return std::move(m_includeIds);
+    }
+
+    FilePathIds takeTopIncludeIds()
+    {
+        std::sort(m_topIncludeIds.begin(), m_topIncludeIds.end());
+
+        return std::move(m_topIncludeIds);
+    }
+
+    FilePathIds takeTopsSystemIncludeIds()
+    {
+        std::sort(m_topsSystemIncludeIds.begin(), m_topsSystemIncludeIds.end());
+
+        return std::move(m_topsSystemIncludeIds);
+    }
 
 private:
+    ClangTool m_clangTool;
     Utils::PathStringVector m_excludedIncludes;
     FilePathIds m_includeIds;
     FilePathIds m_topIncludeIds;
     FilePathIds m_topsSystemIncludeIds;
     Utils::SmallStringVector m_directories;
+    SourcesManager m_sourcesManager;
+    UsedMacros m_usedMacros;
+    FilePathIds m_sourceFiles;
+    SourceDependencies m_sourceDependencies;
+    FileStatuses m_fileStatuses;
     const FilePathCachingInterface &m_filePathCache;
 };
 

src/tools/clangpchmanagerbackend/source/pchcreator.cpp

@@ -243,21 +243,17 @@ PchCreatorIncludes PchCreator::generateProjectPartPchIncludes(
     Utils::SmallString jointedFilePath = generateProjectPartSourceFilePath(projectPart);
     auto jointFile = generateFileWithContent(jointedFilePath, jointedFileContent);
     Utils::SmallStringVector arguments = generateProjectPartCommandLine(projectPart);
-    arguments.push_back(jointedFilePath);
     FilePath filePath{Utils::PathString(jointedFilePath)};
 
     IncludeCollector collector(m_filePathCache);
 
     collector.setExcludedIncludes(generateProjectPartSourcePaths(projectPart));
 
-    collector.addFile(std::string(filePath.directory()),
+    collector.addFile(filePath, projectPart.sourcePathIds, arguments);
-                      std::string(filePath.name()),
-                      {},
-                      arguments);
 
     collector.addUnsavedFiles(m_unsavedFiles);
 
-    collector.collectIncludes();
+    collector.collect();
 
     jointFile->remove();
 

src/tools/clangrefactoringbackend/source/clangtool.cpp

@@ -25,6 +25,8 @@
 
 #include "clangtool.h"
 
+#include <iostream>
+
 namespace ClangBackEnd {
 
 namespace {

src/tools/clangrefactoringbackend/source/symbolsvisitorbase.h

@@ -44,7 +44,7 @@ namespace ClangBackEnd {
 class SymbolsVisitorBase
 {
 public:
-    SymbolsVisitorBase(FilePathCachingInterface &filePathCache,
+    SymbolsVisitorBase(const FilePathCachingInterface &filePathCache,
                        const clang::SourceManager *sourceManager,
                        SourcesManager &sourcesManager)
         : m_filePathCache(filePathCache),
@@ -152,8 +152,13 @@ public:
 
     bool isInSystemHeader(clang::FileID fileId) const
     {
-        return clang::SrcMgr::isSystem(
+        return isSystem(m_sourceManager->getSLocEntry(fileId).getFile().getFileCharacteristic());
-                    m_sourceManager->getSLocEntry(fileId).getFile().getFileCharacteristic());
+    }
+
+    static
+    bool isSystem(clang::SrcMgr::CharacteristicKind kind)
+    {
+        return clang::SrcMgr::isSystem(kind);
     }
 
     void clear()
@@ -163,7 +168,7 @@ public:
 
 protected:
     std::vector<FilePathId> m_filePathIndices;
-    FilePathCachingInterface &m_filePathCache;
+    const FilePathCachingInterface &m_filePathCache;
     const clang::SourceManager *m_sourceManager = nullptr;
     SourcesManager &m_sourcesManager;
 };

tests/unit/unittest/includecollector-test.cpp

@@ -31,6 +31,7 @@
 
 #include <sqlitedatabase.h>
 
+#include <QDateTime>
 #include <QDir>
 
 using testing::AllOf;
@@ -40,7 +41,10 @@ using testing::ElementsAre;
 using testing::UnorderedElementsAre;
 
 using ClangBackEnd::FilePathId;
+using ClangBackEnd::FilePathIds;
 using ClangBackEnd::FilePathView;
+using ClangBackEnd::SourceDependency;
+using ClangBackEnd::UsedMacro;
 
 namespace {
 
@@ -49,66 +53,80 @@ class IncludeCollector : public ::testing::Test
 protected:
     void SetUp()
     {
-        collector.addFile(TESTDATA_DIR, "includecollector_main.cpp", "", {"cc", "includecollector_main.cpp"});
+        collector.addFile(id(TESTDATA_DIR "/includecollector/project/main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
-        collector.addFile(TESTDATA_DIR, "includecollector_main2.cpp", "", {"cc", "includecollector_main2.cpp"});
+        collector.addFile(id(TESTDATA_DIR "/includecollector/project/main2.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
 
-        collector.addUnsavedFiles({{{TESTDATA_DIR, "includecollector_generated_file.h"}, "#pragma once", {}}});
+        collector.addUnsavedFiles({{{TESTDATA_DIR, "includecollector/project/generated_file.h"}, "#pragma once", {}}});
 
         collector.setExcludedIncludes(excludePaths.clone());
         emptyCollector.setExcludedIncludes(excludePaths.clone());
     }
 
-    FilePathId id(const Utils::SmallStringView &path)
+    FilePathId id(const Utils::SmallStringView &path) const
     {
         return filePathCache.filePathId(FilePathView{path});
     }
 
+    static off_t fileSize(Utils::SmallStringView filePath)
+    {
+        return QFileInfo(QString(filePath)).size();
+    }
+
+   static std::time_t lastModified(Utils::SmallStringView filePath)
+    {
+        return QFileInfo(QString(filePath)).lastModified().toTime_t();
+    }
+
+    ClangBackEnd::FileStatus fileStatus(Utils::SmallStringView filePath) const
+    {
+        return {id(filePath), fileSize(filePath), lastModified(filePath), false};
+    }
 protected:
     Sqlite::Database database{":memory:", Sqlite::JournalMode::Memory};
     ClangBackEnd::RefactoringDatabaseInitializer<Sqlite::Database> databaseInitializer{database};
     ClangBackEnd::FilePathCaching filePathCache{database};
     ClangBackEnd::IncludeCollector collector{filePathCache};
     ClangBackEnd::IncludeCollector emptyCollector{filePathCache};
-    Utils::PathStringVector excludePaths = {TESTDATA_DIR "/includecollector_main.cpp",
+    Utils::PathStringVector excludePaths = {TESTDATA_DIR "/includecollector/project/main.cpp",
-                                            TESTDATA_DIR "/includecollector_main2.cpp",
+                                            TESTDATA_DIR "/includecollector/project/main2.cpp",
-                                            TESTDATA_DIR "/includecollector_header1.h",
+                                            TESTDATA_DIR "/includecollector/project/header1.h",
-                                            TESTDATA_DIR "/includecollector_header2.h",
+                                            TESTDATA_DIR "/includecollector/project/header2.h",
-                                            TESTDATA_DIR "/includecollector_generated_file.h"};
+                                            TESTDATA_DIR "/includecollector/project/generated_file.h"};
 };
 
 TEST_F(IncludeCollector, IncludesExternalHeader)
 {
-    collector.collectIncludes();
+    collector.collect();
 
     ASSERT_THAT(collector.takeIncludeIds(),
-                AllOf(Contains(id(TESTDATA_DIR "/includecollector_external1.h")),
+                AllOf(Contains(id(TESTDATA_DIR "/includecollector/external/external1.h")),
-                      Contains(id(TESTDATA_DIR "/includecollector_external2.h")),
+                      Contains(id(TESTDATA_DIR "/includecollector/external/external2.h")),
-                      Contains(id(TESTDATA_DIR "/includecollector_indirect_external.h")),
+                      Contains(id(TESTDATA_DIR "/includecollector/external/indirect_external.h")),
-                      Contains(id(TESTDATA_DIR "/includecollector_indirect_external2.h"))));
+                      Contains(id(TESTDATA_DIR "/includecollector/external/indirect_external2.h"))));
 }
 
 TEST_F(IncludeCollector, DoesNotIncludesInternalHeader)
 {
-    collector.collectIncludes();
+    collector.collect();
 
-    ASSERT_THAT(collector.takeIncludeIds(), Not(Contains(id(TESTDATA_DIR "/includecollector_header1.h"))));
+    ASSERT_THAT(collector.takeIncludeIds(), Not(Contains(id(TESTDATA_DIR "/includecollector/project/header1.h"))));
 }
 
 TEST_F(IncludeCollector, NoDuplicate)
 {
-    collector.collectIncludes();
+    collector.collect();
 
     ASSERT_THAT(collector.takeIncludeIds(),
-                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector_external1.h"),
+                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector/external/external1.h"),
-                                     id(TESTDATA_DIR "/includecollector_external2.h"),
+                                     id(TESTDATA_DIR "/includecollector/external/external2.h"),
-                                     id(TESTDATA_DIR "/includecollector_external3.h"),
+                                     id(TESTDATA_DIR "/includecollector/external/external3.h"),
-                                     id(TESTDATA_DIR "/includecollector_indirect_external.h"),
+                                     id(TESTDATA_DIR "/includecollector/external/indirect_external.h"),
-                                     id(TESTDATA_DIR "/includecollector_indirect_external2.h")));
+                                     id(TESTDATA_DIR "/includecollector/external/indirect_external2.h")));
 }
 
 TEST_F(IncludeCollector, IncludesAreSorted)
 {
-    collector.collectIncludes();
+    collector.collect();
 
     ASSERT_THAT(collector.takeIncludeIds(),
                 SizeIs(5));
@@ -116,82 +134,289 @@ TEST_F(IncludeCollector, IncludesAreSorted)
 
 TEST_F(IncludeCollector, If)
 {
-    emptyCollector.addFile(TESTDATA_DIR, "includecollector_if.cpp", "", {"cc", "includecollector_if.cpp"});
+    emptyCollector.addFile(id(TESTDATA_DIR "/includecollector/project/if.cpp"),  {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
 
-    emptyCollector.collectIncludes();
+    emptyCollector.collect();
 
     ASSERT_THAT(emptyCollector.takeIncludeIds(),
-                ElementsAre(id(TESTDATA_DIR "/includecollector_true.h")));
+                ElementsAre(id(TESTDATA_DIR "/includecollector/project/true.h")));
 }
 
 TEST_F(IncludeCollector, LocalPath)
 {
-    emptyCollector.addFile(TESTDATA_DIR, "includecollector_main.cpp", "", {"cc", "includecollector_main.cpp"});
+    emptyCollector.addFile(id(TESTDATA_DIR "/includecollector/project/main.cpp"),  {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
 
-    emptyCollector.collectIncludes();
+    emptyCollector.collect();
 
     ASSERT_THAT(emptyCollector.takeIncludeIds(),
-                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector_external1.h"),
+                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector/external/external1.h"),
-                                     id(TESTDATA_DIR "/includecollector_external2.h"),
+                                     id(TESTDATA_DIR "/includecollector/external/external2.h"),
-                                     id(TESTDATA_DIR "/includecollector_external3.h"),
+                                     id(TESTDATA_DIR "/includecollector/external/external3.h"),
-                                     id(TESTDATA_DIR "/includecollector_indirect_external.h"),
+                                     id(TESTDATA_DIR "/includecollector/external/indirect_external.h"),
-                                     id(TESTDATA_DIR "/includecollector_indirect_external2.h")));
+                                     id(TESTDATA_DIR "/includecollector/external/indirect_external2.h")));
 }
 
 TEST_F(IncludeCollector, IgnoreMissingFile)
 {
-    emptyCollector.addFile(TESTDATA_DIR, "includecollector_missingfile.cpp", "", {"cc", "includecollector_missingfile.cpp"});
+    emptyCollector.addFile(id(TESTDATA_DIR "/includecollector/project/missingfile.cpp"),  {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
 
-    emptyCollector.collectIncludes();
+    emptyCollector.collect();
 
     ASSERT_THAT(emptyCollector.takeIncludeIds(),
-                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector_external1.h"),
+                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector/external/external1.h"),
-                                     id(TESTDATA_DIR "/includecollector_indirect_external.h"),
+                                     id(TESTDATA_DIR "/includecollector/external/indirect_external.h"),
-                                     id(TESTDATA_DIR "/includecollector_indirect_external2.h")));
+                                     id(TESTDATA_DIR "/includecollector/external/indirect_external2.h")));
 }
 
 TEST_F(IncludeCollector, IncludesOnlyTopExternalHeader)
 {
-    collector.collectIncludes();
+    collector.collect();
 
     ASSERT_THAT(collector.takeTopIncludeIds(),
-                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector_external1.h"),
+                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector/external/external1.h"),
-                                     id(TESTDATA_DIR "/includecollector_external2.h"),
+                                     id(TESTDATA_DIR "/includecollector/external/external2.h"),
-                                     id(TESTDATA_DIR "/includecollector_external3.h")));
+                                     id(TESTDATA_DIR "/includecollector/external/external3.h")));
 }
 
 TEST_F(IncludeCollector, TopIncludeInIfMacro)
 {
-    emptyCollector.addFile(TESTDATA_DIR, "includecollector_if.cpp", "", {"cc", "includecollector_if.cpp"});
+    emptyCollector.addFile(id(TESTDATA_DIR "/includecollector/project/if.cpp"),  {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
-    emptyCollector.setExcludedIncludes({"includecollector_if.cpp"});
+    emptyCollector.setExcludedIncludes({TESTDATA_DIR "/includecollector/project/if.cpp"});
 
-    emptyCollector.collectIncludes();
+    emptyCollector.collect();
 
     ASSERT_THAT(emptyCollector.takeTopIncludeIds(),
-                ElementsAre(id(TESTDATA_DIR "/includecollector_true.h")));
+                ElementsAre(id(TESTDATA_DIR "/includecollector/project/true.h")));
 }
 
 TEST_F(IncludeCollector, TopIncludeWithLocalPath)
 {
-    emptyCollector.addFile(TESTDATA_DIR, "includecollector_main.cpp", "", {"cc", "includecollector_main.cpp"});
+    emptyCollector.addFile(id(TESTDATA_DIR "/includecollector/project/main.cpp"),  {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
 
-    emptyCollector.collectIncludes();
+    emptyCollector.collect();
 
     ASSERT_THAT(emptyCollector.takeTopIncludeIds(),
-                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector_external1.h"),
+                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector/external/external1.h"),
-                                     id(TESTDATA_DIR "/includecollector_external2.h"),
+                                     id(TESTDATA_DIR "/includecollector/external/external2.h"),
-                                     id(TESTDATA_DIR "/includecollector_external3.h")));
+                                     id(TESTDATA_DIR "/includecollector/external/external3.h")));
 }
 
 TEST_F(IncludeCollector, TopIncludesIgnoreMissingFile)
 {
-    emptyCollector.addFile(TESTDATA_DIR, "includecollector_missingfile.cpp", "", {"cc", "includecollector_missingfile.cpp"});
+    emptyCollector.addFile(id(TESTDATA_DIR "/includecollector/project/missingfile.cpp"),  {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
-    emptyCollector.setExcludedIncludes({"includecollector_missingfile.cpp"});
+    emptyCollector.setExcludedIncludes({TESTDATA_DIR "/includecollector/project/missingfile.cpp"});
 
-    emptyCollector.collectIncludes();
+    emptyCollector.collect();
 
     ASSERT_THAT(emptyCollector.takeTopIncludeIds(),
-                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector_external1.h")));
+                UnorderedElementsAre(id(TESTDATA_DIR "/includecollector/external/external1.h")));
+}
+
+TEST_F(IncludeCollector, SourceFiles)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.sourceFiles(),
+                UnorderedElementsAre(id(TESTDATA_DIR "/symbolscollector_main.cpp"),
+                                     id(TESTDATA_DIR "/symbolscollector_header1.h"),
+                                     id(TESTDATA_DIR "/symbolscollector_header2.h")));
+}
+
+TEST_F(IncludeCollector, MainFileInSourceFiles)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    ASSERT_THAT(emptyCollector.sourceFiles(),
+                ElementsAre(id(TESTDATA_DIR "/symbolscollector_main.cpp")));
+}
+
+TEST_F(IncludeCollector, ResetMainFileInSourceFiles)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    ASSERT_THAT(emptyCollector.sourceFiles(),
+                ElementsAre(id(TESTDATA_DIR "/symbolscollector_main.cpp")));
+}
+
+TEST_F(IncludeCollector, DontDuplicateSourceFiles)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+    emptyCollector.collect();
+
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.sourceFiles(),
+                UnorderedElementsAre(id(TESTDATA_DIR "/symbolscollector_main.cpp"),
+                                     id(TESTDATA_DIR "/symbolscollector_header1.h"),
+                                     id(TESTDATA_DIR "/symbolscollector_header2.h")));
+}
+
+TEST_F(IncludeCollector, ClearSourceFiles)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    emptyCollector.clear();
+
+    ASSERT_THAT(emptyCollector.sourceFiles(), IsEmpty());
+}
+
+TEST_F(IncludeCollector, ClearFileStatus)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+    emptyCollector.collect();
+
+    emptyCollector.clear();
+
+    ASSERT_THAT(emptyCollector.fileStatuses(), IsEmpty());
+}
+
+TEST_F(IncludeCollector, ClearUsedMacros)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_defines.h"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+    emptyCollector.collect();
+
+    emptyCollector.clear();
+
+    ASSERT_THAT(emptyCollector.usedMacros(), IsEmpty());
+}
+
+TEST_F(IncludeCollector, ClearSourceDependencies)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main2.cpp"), {"cc", "-I" TESTDATA_DIR});
+    emptyCollector.collect();
+
+    emptyCollector.clear();
+
+    ASSERT_THAT(emptyCollector.sourceDependencies(), IsEmpty());
+}
+
+TEST_F(IncludeCollector, DontCollectSourceFilesAfterFilesAreCleared)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    emptyCollector.clear();
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.sourceFiles(), IsEmpty());
+}
+
+TEST_F(IncludeCollector, DontCollectFileStatusAfterFilesAreCleared)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    emptyCollector.clear();
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.fileStatuses(), IsEmpty());
+}
+
+TEST_F(IncludeCollector, DontCollectUsedMacrosAfterFilesAreCleared)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    emptyCollector.clear();
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.usedMacros(), IsEmpty());
+}
+
+
+TEST_F(IncludeCollector, DontCollectSourceDependenciesAfterFilesAreCleared)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    emptyCollector.clear();
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.sourceDependencies(), IsEmpty());
+}
+
+TEST_F(IncludeCollector, CollectUsedMacrosWithExternalDefine)
+{
+    auto fileId = id(TESTDATA_DIR "/symbolscollector_defines.h");
+    emptyCollector.addFile(fileId, {"cc", "-DCOMPILER_ARGUMENT"});
+
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.usedMacros(),
+                ElementsAre(Eq(UsedMacro{"DEFINED", fileId}),
+                            Eq(UsedMacro{"IF_DEFINE", fileId}),
+                            Eq(UsedMacro{"__clang__", fileId}),
+                            Eq(UsedMacro{"CLASS_EXPORT", fileId}),
+                            Eq(UsedMacro{"IF_NOT_DEFINE", fileId}),
+                            Eq(UsedMacro{"MACRO_EXPANSION", fileId}),
+                            Eq(UsedMacro{"COMPILER_ARGUMENT", fileId})));
+}
+
+TEST_F(IncludeCollector, CollectUsedMacrosWithoutExternalDefine)
+{
+    auto fileId = id(TESTDATA_DIR "/symbolscollector_defines.h");
+    emptyCollector.addFile(fileId, {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.usedMacros(),
+                ElementsAre(Eq(UsedMacro{"DEFINED", fileId}),
+                            Eq(UsedMacro{"IF_DEFINE", fileId}),
+                            Eq(UsedMacro{"__clang__", fileId}),
+                            Eq(UsedMacro{"CLASS_EXPORT", fileId}),
+                            Eq(UsedMacro{"IF_NOT_DEFINE", fileId}),
+                            Eq(UsedMacro{"MACRO_EXPANSION", fileId}),
+                            Eq(UsedMacro{"COMPILER_ARGUMENT", fileId})));
+}
+
+TEST_F(IncludeCollector, DontCollectHeaderGuards)
+{
+    auto fileId = id(TESTDATA_DIR "/symbolscollector_defines.h");
+    emptyCollector.addFile(fileId, {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.usedMacros(),
+                Not(Contains(Eq(UsedMacro{"SYMBOLSCOLLECTOR_DEFINES_H", fileId}))));
+}
+
+TEST_F(IncludeCollector, DISABLED_DontCollectDynamicLibraryExports)
+{
+    auto fileId = id(TESTDATA_DIR "/symbolscollector_defines.h");
+    emptyCollector.addFile(fileId, {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.usedMacros(),
+                Not(Contains(Eq(UsedMacro{"CLASS_EXPORT", fileId}))));
+}
+
+TEST_F(IncludeCollector, CollectFileStatuses)
+{
+    emptyCollector.addFile(id(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc", "-I", TESTDATA_DIR "/includecollector/external", "-I", TESTDATA_DIR "/includecollector/project", "-isystem", TESTDATA_DIR "/includecollector/system"});
+
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.fileStatuses(),
+                ElementsAre(
+                    fileStatus(TESTDATA_DIR "/symbolscollector_main.cpp"),
+                    fileStatus(TESTDATA_DIR "/symbolscollector_header1.h"),
+                    fileStatus(TESTDATA_DIR "/symbolscollector_header2.h")));
+}
+
+TEST_F(IncludeCollector, CollectSourceDependencies)
+{
+    auto mainFileId = id(TESTDATA_DIR "/symbolscollector_main2.cpp");
+    auto header1FileId = id(TESTDATA_DIR "/symbolscollector_header1.h");
+    auto header2FileId = id(TESTDATA_DIR "/symbolscollector_header2.h");
+    auto header3FileId = id(TESTDATA_DIR "/symbolscollector_header3.h");
+    emptyCollector.addFile(mainFileId, {"cc", "-I" TESTDATA_DIR});
+
+    emptyCollector.collect();
+
+    ASSERT_THAT(emptyCollector.sourceDependencies(),
+                UnorderedElementsAre(SourceDependency(mainFileId, header1FileId),
+                                     SourceDependency(mainFileId, header3FileId),
+                                     SourceDependency(header3FileId, header2FileId),
+                                     SourceDependency(header1FileId, header2FileId)));
 }
 
 }