ClangPchManager: Introduce UsedMacroAndSourcesCollector

It's a subset to thesymbols collecter so we try to share some code.

Change-Id: I058eee39289453e205f039b3850c1816f1d8ec1a
Task-number: QTCREATORBUG-21257
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>

src/plugins/clangrefactoring/refactoringengine.cpp

@@ -104,7 +104,7 @@ CppTools::Usages RefactoringEngine::locationsAt(const CppTools::CursorInEditor &
         const QByteArray filePath = data.filePath().toString().toUtf8();
         const ClangBackEnd::FilePathId filePathId = m_filePathCache.filePathId(ClangBackEnd::FilePathView(filePath));
 
-        usages = m_symbolQuery.sourceUsagesAt(filePathId, lineColumn->line, lineColumn->column + 1);
+        usages = m_symbolQuery.sourceUsagesAt(filePathId, lineColumn->line, lineColumn->column);
     }
 
     return usages;

src/tools/clangpchmanagerbackend/source/clangpchmanagerbackend-source.pri

@@ -3,7 +3,8 @@ INCLUDEPATH += $$PWD
 SOURCES += \
     $$PWD/pchmanagerserver.cpp \
     $$PWD/projectparts.cpp \
-    $$PWD/projectpartqueue.cpp
+    $$PWD/projectpartqueue.cpp \
+    $$PWD/usedmacrosandsourcescollector.cpp
 
 HEADERS += \
     $$PWD/pchmanagerserver.h \
@@ -21,7 +22,11 @@ HEADERS += \
     $$PWD/taskscheduler.h \
     $$PWD/taskschedulerinterface.h \
     $$PWD/precompiledheaderstorage.h \
-    $$PWD/precompiledheaderstorageinterface.h
+    $$PWD/precompiledheaderstorageinterface.h \
+    $$PWD/collectusedmacroactionfactory.h \
+    $$PWD/collectusedmacrosaction.h \
+    $$PWD/collectusedmacrosandsourcespreprocessorcallbacks.h \
+    $$PWD/usedmacrosandsourcescollector.h
 
 !isEmpty(LIBTOOLING_LIBS) {
 SOURCES += \

src/tools/clangpchmanagerbackend/source/collectusedmacroactionfactory.h

@@ -0,0 +1,85 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "collectusedmacrosaction.h"
+
+#include <filepathcachingfwd.h>
+#include <usedmacro.h>
+
+#include <clang/Tooling/Tooling.h>
+
+namespace ClangBackEnd {
+
+class CollectUsedMacrosToolActionFactory final : public clang::tooling::FrontendActionFactory
+{
+public:
+    CollectUsedMacrosToolActionFactory(UsedMacros &usedMacros,
+                                       FilePathCachingInterface &filePathCache,
+                                       SourcesManager &sourcesManager,
+                                       SourceDependencies &sourceDependencies,
+                                       FilePathIds &sourceFiles,
+                                       FileStatuses &fileStatuses)
+        : m_usedMacros(usedMacros),
+          m_filePathCache(filePathCache),
+          m_sourcesManager(sourcesManager),
+          m_sourceDependencies(sourceDependencies),
+          m_sourceFiles(sourceFiles),
+          m_fileStatuses(fileStatuses)
+    {}
+
+
+    bool runInvocation(std::shared_ptr<clang::CompilerInvocation> invocation,
+                       clang::FileManager *fileManager,
+                       std::shared_ptr<clang::PCHContainerOperations> pchContainerOperations,
+                       clang::DiagnosticConsumer *diagnosticConsumer) override
+    {
+        return clang::tooling::FrontendActionFactory::runInvocation(invocation,
+                                                                    fileManager,
+                                                                    pchContainerOperations,
+                                                                    diagnosticConsumer);
+    }
+
+    clang::FrontendAction *create() override
+    {
+        return new CollectUsedMacrosAction(m_usedMacros,
+                                           m_filePathCache,
+                                           m_sourcesManager,
+                                           m_sourceDependencies,
+                                           m_sourceFiles,
+                                           m_fileStatuses);
+    }
+
+private:
+    UsedMacros &m_usedMacros;
+    FilePathCachingInterface &m_filePathCache;
+    SourcesManager &m_sourcesManager;
+    SourceDependencies &m_sourceDependencies;
+    FilePathIds &m_sourceFiles;
+    FileStatuses &m_fileStatuses;
+};
+
+} // namespace ClangBackEnd

src/tools/clangpchmanagerbackend/source/collectusedmacrosaction.h

@@ -0,0 +1,96 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <collectusedmacrosandsourcespreprocessorcallbacks.h>
+
+#include <filepathcachingfwd.h>
+#include <usedmacro.h>
+
+#include <clang/Frontend/FrontendActions.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Lex/Preprocessor.h>
+
+namespace ClangBackEnd {
+
+class CollectUsedMacrosAction final : public clang::PreprocessOnlyAction
+{
+public:
+    CollectUsedMacrosAction(UsedMacros &usedMacros,
+                            FilePathCachingInterface &filePathCache,
+                            SourcesManager &sourcesManager,
+                            SourceDependencies &sourceDependencies,
+                            FilePathIds &sourceFiles,
+                            FileStatuses &fileStatuses)
+        : m_usedMacros(usedMacros),
+          m_filePathCache(filePathCache),
+          m_sourcesManager(sourcesManager),
+          m_sourceDependencies(sourceDependencies),
+          m_sourceFiles(sourceFiles),
+          m_fileStatuses(fileStatuses)
+    {
+    }
+
+    bool BeginSourceFileAction(clang::CompilerInstance &compilerInstance) override
+    {
+      if (clang::PreprocessOnlyAction::BeginSourceFileAction(compilerInstance)) {
+          auto &preprocessor = compilerInstance.getPreprocessor();
+
+          preprocessor.SetSuppressIncludeNotFoundError(true);
+
+          auto macroPreprocessorCallbacks = new CollectUsedMacrosAndSourcesPreprocessorCallbacks(
+                      m_usedMacros,
+                      m_filePathCache,
+                      compilerInstance.getSourceManager(),
+                      m_sourcesManager,
+                      compilerInstance.getPreprocessorPtr(),
+                      m_sourceDependencies,
+                      m_sourceFiles,
+                      m_fileStatuses);
+
+          preprocessor.addPPCallbacks(std::unique_ptr<clang::PPCallbacks>(macroPreprocessorCallbacks));
+
+          return true;
+      }
+
+      return false;
+    }
+
+    void EndSourceFileAction() override
+    {
+      clang::PreprocessOnlyAction::EndSourceFileAction();
+    }
+
+private:
+    UsedMacros &m_usedMacros;
+    FilePathCachingInterface &m_filePathCache;
+    SourcesManager &m_sourcesManager;
+    SourceDependencies &m_sourceDependencies;
+    FilePathIds &m_sourceFiles;
+    FileStatuses &m_fileStatuses;
+};
+
+} // namespace ClangBackEnd

src/tools/clangpchmanagerbackend/source/collectusedmacrosandsourcespreprocessorcallbacks.h

@@ -0,0 +1,246 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "sourcelocationsutils.h"
+#include <filepathcachinginterface.h>
+#include <filestatus.h>
+#include <sourcedependency.h>
+#include <symbolsvisitorbase.h>
+#include <usedmacro.h>
+
+#include <utils/smallstringvector.h>
+
+#include <clang/Basic/SourceManager.h>
+#include <clang/Lex/MacroInfo.h>
+#include <clang/Lex/HeaderSearch.h>
+#include <clang/Lex/PPCallbacks.h>
+#include <clang/Lex/Preprocessor.h>
+
+#include <QFile>
+#include <QDir>
+#include <QTemporaryDir>
+
+#include <algorithm>
+
+namespace ClangBackEnd {
+
+class CollectUsedMacrosAndSourcesPreprocessorCallbacksBase : public SymbolsVisitorBase
+{
+public:
+    CollectUsedMacrosAndSourcesPreprocessorCallbacksBase(UsedMacros &usedMacros,
+                                                         FilePathCachingInterface &filePathCache,
+                                                         const clang::SourceManager &sourceManager,
+                                                         SourcesManager &sourcesManager,
+                                                         std::shared_ptr<clang::Preprocessor> preprocessor,
+                                                         SourceDependencies &sourceDependencies,
+                                                         FilePathIds &sourceFiles,
+                                                         FileStatuses &fileStatuses)
+        : SymbolsVisitorBase(filePathCache, &sourceManager, sourcesManager),
+          m_usedMacros(usedMacros),
+          m_preprocessor(preprocessor),
+          m_sourceDependencies(sourceDependencies),
+          m_sourceFiles(sourceFiles),
+          m_fileStatuses(fileStatuses)
+    {}
+
+    void addSourceFile(const clang::FileEntry *fileEntry)
+    {
+        auto id = filePathId(fileEntry);
+
+        auto found = std::lower_bound(m_sourceFiles.begin(), m_sourceFiles.end(), id);
+
+        if (found == m_sourceFiles.end() || *found != id)
+            m_sourceFiles.insert(found, id);
+    }
+
+    void addFileStatus(const clang::FileEntry *fileEntry)
+    {
+        auto id = filePathId(fileEntry);
+
+        auto found = std::lower_bound(m_fileStatuses.begin(),
+                                      m_fileStatuses.end(),
+                                      id,
+                                      [] (const auto &first, const auto &second) {
+            return first.filePathId < second;
+        });
+
+        if (found == m_fileStatuses.end() || found->filePathId != id) {
+            m_fileStatuses.emplace(found,
+                                   id,
+                                   fileEntry->getSize(),
+                                   fileEntry->getModificationTime(),
+                                   fileEntry->isInPCH());
+        }
+    }
+
+    void addSourceDependency(const clang::FileEntry *file, clang::SourceLocation includeLocation)
+    {
+        auto includeFilePathId = filePathId(includeLocation);
+        auto includedFilePathId = filePathId(file);
+
+        m_sourceDependencies.emplace_back(includeFilePathId, includedFilePathId);
+    }
+
+    void mergeUsedMacros()
+    {
+        m_usedMacros.reserve(m_usedMacros.size() + m_maybeUsedMacros.size());
+        auto insertionPoint = m_usedMacros.insert(m_usedMacros.end(),
+                                                  m_maybeUsedMacros.begin(),
+                                                  m_maybeUsedMacros.end());
+        std::inplace_merge(m_usedMacros.begin(), insertionPoint, m_usedMacros.end());
+    }
+
+    static void addUsedMacro(UsedMacro &&usedMacro, UsedMacros &usedMacros)
+    {
+        if (!usedMacro.filePathId.isValid())
+            return;
+
+        auto found = std::lower_bound(usedMacros.begin(),
+                                      usedMacros.end(), usedMacro);
+
+        if (found == usedMacros.end() || *found != usedMacro)
+            usedMacros.insert(found, std::move(usedMacro));
+    }
+
+    void addUsedMacro(const clang::Token &macroNameToken,
+                      const clang::MacroDefinition &macroDefinition)
+    {
+        clang::MacroInfo *macroInfo = macroDefinition.getMacroInfo();
+        UsedMacro usedMacro{macroNameToken.getIdentifierInfo()->getName(),
+                              filePathId(macroNameToken.getLocation())};
+        if (macroInfo)
+            addUsedMacro(std::move(usedMacro), m_usedMacros);
+        else
+            addUsedMacro(std::move(usedMacro), m_maybeUsedMacros);
+    }
+
+    bool isInSystemHeader(clang::SourceLocation sourceLocation) const
+    {
+        return m_sourceManager->isInSystemHeader(sourceLocation);
+    }
+
+    void filterOutHeaderGuards()
+    {
+        auto partitionPoint = std::stable_partition(m_maybeUsedMacros.begin(),
+                                                    m_maybeUsedMacros.end(),
+                                                    [&] (const UsedMacro &usedMacro) {
+            llvm::StringRef id{usedMacro.macroName.data(), usedMacro.macroName.size()};
+            clang::IdentifierInfo &identifierInfo = m_preprocessor->getIdentifierTable().get(id);
+            clang::MacroInfo *macroInfo = m_preprocessor->getMacroInfo(&identifierInfo);
+            return !macroInfo || !macroInfo->isUsedForHeaderGuard();
+        });
+
+        m_maybeUsedMacros.erase(partitionPoint, m_maybeUsedMacros.end());
+    }
+
+private:
+    UsedMacros m_maybeUsedMacros;
+    UsedMacros &m_usedMacros;
+    std::shared_ptr<clang::Preprocessor> m_preprocessor;
+    SourceDependencies &m_sourceDependencies;
+    FilePathIds &m_sourceFiles;
+    FileStatuses &m_fileStatuses;
+};
+
+class CollectUsedMacrosAndSourcesPreprocessorCallbacks final : public clang::PPCallbacks,
+                                                               public CollectUsedMacrosAndSourcesPreprocessorCallbacksBase
+{
+public:
+    using CollectUsedMacrosAndSourcesPreprocessorCallbacksBase::CollectUsedMacrosAndSourcesPreprocessorCallbacksBase;
+
+    void FileChanged(clang::SourceLocation sourceLocation,
+                     clang::PPCallbacks::FileChangeReason reason,
+                     clang::SrcMgr::CharacteristicKind,
+                     clang::FileID) override
+    {
+        if (reason == clang::PPCallbacks::EnterFile)
+        {
+            const clang::FileEntry *fileEntry = m_sourceManager->getFileEntryForID(
+                        m_sourceManager->getFileID(sourceLocation));
+            if (fileEntry) {
+                addFileStatus(fileEntry);
+                addSourceFile(fileEntry);
+            }
+        }
+    }
+
+    void InclusionDirective(clang::SourceLocation hashLocation,
+                            const clang::Token &/*includeToken*/,
+                            llvm::StringRef /*fileName*/,
+                            bool /*isAngled*/,
+                            clang::CharSourceRange /*fileNameRange*/,
+                            const clang::FileEntry *file,
+                            llvm::StringRef /*searchPath*/,
+                            llvm::StringRef /*relativePath*/,
+                            const clang::Module * /*imported*/) override
+    {
+        if (!m_skipInclude && file)
+            addSourceDependency(file, hashLocation);
+
+        m_skipInclude = false;
+    }
+
+    void Ifndef(clang::SourceLocation,
+                const clang::Token &macroNameToken,
+                const clang::MacroDefinition &macroDefinition) override
+    {
+        addUsedMacro(macroNameToken, macroDefinition);
+    }
+
+    void Ifdef(clang::SourceLocation,
+               const clang::Token &macroNameToken,
+               const clang::MacroDefinition &macroDefinition) override
+    {
+        addUsedMacro( macroNameToken, macroDefinition);
+    }
+
+    void Defined(const clang::Token &macroNameToken,
+                 const clang::MacroDefinition &macroDefinition,
+                 clang::SourceRange) override
+    {
+        addUsedMacro(macroNameToken, macroDefinition);
+    }
+
+    void MacroExpands(const clang::Token &macroNameToken,
+                      const clang::MacroDefinition &macroDefinition,
+                      clang::SourceRange,
+                      const clang::MacroArgs *) override
+    {
+        addUsedMacro(macroNameToken, macroDefinition);
+    }
+
+    void EndOfMainFile() override
+    {
+        filterOutHeaderGuards();
+        mergeUsedMacros();
+        m_sourcesManager.updateModifiedTimeStamps();
+    }
+
+private:
+    bool m_skipInclude = false;
+};
+} // namespace ClangBackEnd

src/tools/clangpchmanagerbackend/source/usedmacrosandsourcescollector.cpp

@@ -0,0 +1,68 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "usedmacrosandsourcescollector.h"
+
+#include "collectusedmacroactionfactory.h"
+
+namespace ClangBackEnd {
+
+void UsedMacroAndSourcesCollector::addFiles(const FilePathIds &filePathIds,
+                                            const Utils::SmallStringVector &arguments)
+{
+    m_clangTool.addFiles(m_filePathCache.filePaths(filePathIds), arguments);
+    m_sourceFiles.insert(m_sourceFiles.end(), filePathIds.begin(), filePathIds.end());
+}
+
+void UsedMacroAndSourcesCollector::addFile(FilePathId filePathId, const Utils::SmallStringVector &arguments)
+{
+    addFiles({filePathId}, arguments);
+}
+
+void UsedMacroAndSourcesCollector::collect()
+{
+    clang::tooling::ClangTool tool = m_clangTool.createTool();
+
+    auto action = std::make_unique<CollectUsedMacrosToolActionFactory>(
+                m_usedMacros,
+                m_filePathCache,
+                m_sourcesManager,
+                m_sourceDependencies,
+                m_sourceFiles,
+                m_fileStatuses);
+
+    tool.run(action.get());
+}
+
+void UsedMacroAndSourcesCollector::clear()
+{
+    m_clangTool = ClangTool();
+    m_usedMacros.clear();
+    m_sourceFiles.clear();
+    m_fileStatuses.clear();
+    m_sourceDependencies.clear();
+}
+
+} // namespace ClangBackEnd

src/tools/clangpchmanagerbackend/source/usedmacrosandsourcescollector.h

@@ -0,0 +1,85 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <filepathcachinginterface.h>
+#include <filestatus.h>
+#include <sourcedependency.h>
+#include <sourcesmanager.h>
+#include <usedmacro.h>
+
+#include <clangtool.h>
+
+namespace ClangBackEnd {
+
+class UsedMacroAndSourcesCollector
+{
+public:
+    UsedMacroAndSourcesCollector(FilePathCachingInterface &filePathCache)
+        : m_filePathCache(filePathCache)
+    {
+    }
+
+    void addFiles(const FilePathIds &filePathIds,
+                  const Utils::SmallStringVector &arguments);
+    void addFile(FilePathId filePathId,
+                 const Utils::SmallStringVector &arguments);
+
+    void collect();
+
+    void clear();
+
+    const FileStatuses &fileStatuses() const
+    {
+        return m_fileStatuses;
+    }
+
+    const FilePathIds &sourceFiles() const
+    {
+        return m_sourceFiles;
+    }
+
+    const UsedMacros &usedMacros() const
+    {
+        return m_usedMacros;
+    }
+
+    const SourceDependencies &sourceDependencies() const
+    {
+        return m_sourceDependencies;
+    }
+
+private:
+    ClangTool m_clangTool;
+    SourcesManager m_sourcesManager;
+    UsedMacros m_usedMacros;
+    FilePathCachingInterface &m_filePathCache;
+    FilePathIds m_sourceFiles;
+    SourceDependencies m_sourceDependencies;
+    FileStatuses m_fileStatuses;
+};
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/collectmacrospreprocessorcallbacks.h

@@ -26,7 +26,6 @@
 #pragma once
 
 #include "filestatus.h"
-#include "symbolsvisitorbase.h"
 #include "sourcedependency.h"
 #include "sourcelocationsutils.h"
 #include "sourcelocationentry.h"
@@ -34,6 +33,8 @@
 #include "symbolentry.h"
 #include "usedmacro.h"
 
+#include <collectusedmacrosandsourcespreprocessorcallbacks.h>
+
 #include <filepath.h>
 #include <filepathid.h>
 
@@ -44,7 +45,7 @@
 namespace ClangBackEnd {
 
 class CollectMacrosPreprocessorCallbacks final : public clang::PPCallbacks,
-                                                 public SymbolsVisitorBase
+                                                 public CollectUsedMacrosAndSourcesPreprocessorCallbacksBase
 {
 public:
     CollectMacrosPreprocessorCallbacks(SymbolEntries &symbolEntries,
@@ -57,21 +58,23 @@ public:
                                        const clang::SourceManager &sourceManager,
                                        std::shared_ptr<clang::Preprocessor> &&preprocessor,
                                        SourcesManager &sourcesManager)
-        : SymbolsVisitorBase(filePathCache, &sourceManager, sourcesManager),
-          m_preprocessor(std::move(preprocessor)),
-          m_sourceDependencies(sourceDependencies),
+        : CollectUsedMacrosAndSourcesPreprocessorCallbacksBase(usedMacros,
+                                                               filePathCache,
+                                                               sourceManager,
+                                                               sourcesManager,
+                                                               std::move(preprocessor),
+                                                               sourceDependencies,
+                                                               sourceFiles,
+                                                               fileStatuses),
           m_symbolEntries(symbolEntries),
-          m_sourceLocationEntries(sourceLocationEntries),
-          m_sourceFiles(sourceFiles),
-          m_usedMacros(usedMacros),
-          m_fileStatuses(fileStatuses)
+          m_sourceLocationEntries(sourceLocationEntries)
     {
     }
 
     void FileChanged(clang::SourceLocation sourceLocation,
                      clang::PPCallbacks::FileChangeReason reason,
                      clang::SrcMgr::CharacteristicKind,
-                     clang::FileID)
+                     clang::FileID) override
     {
         if (reason == clang::PPCallbacks::EnterFile)
         {
@@ -170,56 +173,6 @@ public:
         m_sourcesManager.updateModifiedTimeStamps();
     }
 
-    void filterOutHeaderGuards()
-    {
-        auto partitionPoint = std::stable_partition(m_maybeUsedMacros.begin(),
-                                                    m_maybeUsedMacros.end(),
-                                                    [&] (const UsedMacro &usedMacro) {
-            llvm::StringRef id{usedMacro.macroName.data(), usedMacro.macroName.size()};
-            clang::IdentifierInfo &identifierInfo = m_preprocessor->getIdentifierTable().get(id);
-            clang::MacroInfo *macroInfo = m_preprocessor->getMacroInfo(&identifierInfo);
-            return !macroInfo || !macroInfo->isUsedForHeaderGuard();
-        });
-
-        m_maybeUsedMacros.erase(partitionPoint, m_maybeUsedMacros.end());
-    }
-
-    void mergeUsedMacros()
-    {
-        m_usedMacros.reserve(m_usedMacros.size() + m_maybeUsedMacros.size());
-        auto insertionPoint = m_usedMacros.insert(m_usedMacros.end(),
-                                                  m_maybeUsedMacros.begin(),
-                                                  m_maybeUsedMacros.end());
-        std::inplace_merge(m_usedMacros.begin(), insertionPoint, m_usedMacros.end());
-    }
-
-    static void addUsedMacro(UsedMacro &&usedMacro, UsedMacros &usedMacros)
-    {
-        if (!usedMacro.filePathId.isValid())
-            return;
-
-        auto found = std::lower_bound(usedMacros.begin(),
-                                      usedMacros.end(), usedMacro);
-
-        if (found == usedMacros.end() || *found != usedMacro)
-            usedMacros.insert(found, std::move(usedMacro));
-    }
-
-    void addUsedMacro(const clang::Token &macroNameToken,
-                       const clang::MacroDefinition &macroDefinition)
-    {
-        if (isInSystemHeader(macroNameToken.getLocation()))
-            return;
-
-        clang::MacroInfo *macroInfo = macroDefinition.getMacroInfo();
-        UsedMacro usedMacro{macroNameToken.getIdentifierInfo()->getName(),
-                              filePathId(macroNameToken.getLocation())};
-        if (macroInfo)
-            addUsedMacro(std::move(usedMacro), m_usedMacros);
-        else
-            addUsedMacro(std::move(usedMacro), m_maybeUsedMacros);
-    }
-
     static const clang::MacroInfo *firstMacroInfo(const clang::MacroDirective *macroDirective)
     {
         if (macroDirective) {
@@ -235,11 +188,6 @@ public:
         return nullptr;
     }
 
-    bool isInSystemHeader(clang::SourceLocation sourceLocation) const
-    {
-        return m_sourceManager->isInSystemHeader(sourceLocation);
-    }
-
     void addMacroAsSymbol(const clang::Token &macroNameToken,
                           const clang::MacroInfo *macroInfo,
                           SourceLocationKind symbolType)
@@ -272,53 +220,11 @@ public:
         }
     }
 
-    void addSourceFile(const clang::FileEntry *fileEntry)
-    {
-        auto id = filePathId(fileEntry);
-
-        auto found = std::lower_bound(m_sourceFiles.begin(), m_sourceFiles.end(), id);
-
-        if (found == m_sourceFiles.end() || *found != id)
-            m_sourceFiles.insert(found, id);
-    }
-
-    void addFileStatus(const clang::FileEntry *fileEntry)
-    {
-        auto id = filePathId(fileEntry);
-
-        auto found = std::lower_bound(m_fileStatuses.begin(),
-                                      m_fileStatuses.end(),
-                                      id,
-                                      [] (const auto &first, const auto &second) {
-            return first.filePathId < second;
-        });
-
-        if (found == m_fileStatuses.end() || found->filePathId != id) {
-            m_fileStatuses.emplace(found,
-                                   id,
-                                   fileEntry->getSize(),
-                                   fileEntry->getModificationTime(),
-                                   fileEntry->isInPCH());
-        }
-    }
-
-    void addSourceDependency(const clang::FileEntry *file, clang::SourceLocation includeLocation)
-    {
-        auto includeFilePathId = filePathId(includeLocation);
-        auto includedFilePathId = filePathId(file);
-
-        m_sourceDependencies.emplace_back(includeFilePathId, includedFilePathId);
-    }
-
 private:
     UsedMacros m_maybeUsedMacros;
     std::shared_ptr<clang::Preprocessor> m_preprocessor;
-    SourceDependencies &m_sourceDependencies;
     SymbolEntries &m_symbolEntries;
     SourceLocationEntries &m_sourceLocationEntries;
-    FilePathIds &m_sourceFiles;
-    UsedMacros &m_usedMacros;
-    FileStatuses &m_fileStatuses;
     bool m_skipInclude = false;
 };
 

src/tools/clangrefactoringbackend/source/collectmacrossourcefilecallbacks.h

@@ -70,6 +70,7 @@ public:
         m_sourceFiles.clear();
         m_usedMacros.clear();
         m_fileStatuses.clear();
+        m_sourceDependencies.clear();
     }
 
     const UsedMacros &usedMacros() const

tests/unit/unittest/symbolscollector-test.cpp

@@ -363,6 +363,16 @@ TEST_F(SymbolsCollector, ClearUsedMacros)
     ASSERT_THAT(collector.usedMacros(), IsEmpty());
 }
 
+TEST_F(SymbolsCollector, ClearSourceDependencies)
+{
+    collector.setFile(filePathId(TESTDATA_DIR "/symbolscollector_main2.cpp"), {"cc", "-I" TESTDATA_DIR});
+    collector.collectSymbols();
+
+    collector.clear();
+
+    ASSERT_THAT(collector.sourceDependencies(), IsEmpty());
+}
+
 TEST_F(SymbolsCollector, DontCollectSymbolsAfterFilesAreCleared)
 {
     collector.setFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
@@ -403,6 +413,16 @@ TEST_F(SymbolsCollector, DontCollectUsedMacrosAfterFilesAreCleared)
     ASSERT_THAT(collector.usedMacros(), IsEmpty());
 }
 
+TEST_F(SymbolsCollector, DontCollectSourceDependenciesAfterFilesAreCleared)
+{
+    collector.setFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+
+    collector.clear();
+    collector.collectSymbols();
+
+    ASSERT_THAT(collector.sourceDependencies(), IsEmpty());
+}
+
 TEST_F(SymbolsCollector, CollectUsedMacrosWithExternalDefine)
 {
     auto fileId = filePathId(TESTDATA_DIR "/symbolscollector_defines.h");

tests/unit/unittest/unittest.pro

@@ -103,7 +103,8 @@ SOURCES += \
     projectpartqueue-test.cpp \
     processormanager-test.cpp \
     taskscheduler-test.cpp \
-    compileroptionsbuilder-test.cpp
+    compileroptionsbuilder-test.cpp \
+    usedmacrocollector-test.cpp
 
 !isEmpty(LIBCLANG_LIBS) {
 SOURCES += \

tests/unit/unittest/usedmacrocollector-test.cpp

@@ -0,0 +1,286 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "googletest.h"
+
+#include <refactoringdatabaseinitializer.h>
+#include <filepathcaching.h>
+#include <usedmacrosandsourcescollector.h>
+
+#include <sqlitedatabase.h>
+
+#include <QDateTime>
+#include <QDir>
+
+using testing::AllOf;
+using testing::Contains;
+using testing::Not;
+using testing::ElementsAre;
+using testing::UnorderedElementsAre;
+
+using ClangBackEnd::FilePathId;
+using ClangBackEnd::FilePathIds;
+using ClangBackEnd::FilePathView;
+using ClangBackEnd::SourceDependency;
+using ClangBackEnd::UsedMacro;
+
+namespace {
+
+class UsedMacroAndSourcesCollector : public ::testing::Test
+{
+protected:
+    FilePathId filePathId(Utils::SmallStringView filePath) const
+    {
+        return filePathCache.filePathId(ClangBackEnd::FilePathView{filePath});
+    }
+
+    static off_t fileSize(Utils::SmallStringView filePath)
+    {
+        return QFileInfo(QString(filePath)).size();
+    }
+
+   static std::time_t lastModified(Utils::SmallStringView filePath)
+    {
+        return QFileInfo(QString(filePath)).lastModified().toTime_t();
+    }
+
+    ClangBackEnd::FileStatus fileStatus(Utils::SmallStringView filePath) const
+    {
+        return {filePathId(filePath), fileSize(filePath), lastModified(filePath), false};
+    }
+protected:
+    Sqlite::Database database{":memory:", Sqlite::JournalMode::Memory};
+    ClangBackEnd::RefactoringDatabaseInitializer<Sqlite::Database> databaseInitializer{database};
+    ClangBackEnd::FilePathCaching filePathCache{database};
+    ClangBackEnd::UsedMacroAndSourcesCollector collector{filePathCache};
+};
+
+TEST_F(UsedMacroAndSourcesCollector, SourceFiles)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+
+    collector.collect();
+
+    ASSERT_THAT(collector.sourceFiles(),
+                UnorderedElementsAre(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"),
+                                     filePathId(TESTDATA_DIR "/symbolscollector_header1.h"),
+                                     filePathId(TESTDATA_DIR "/symbolscollector_header2.h")));
+}
+
+TEST_F(UsedMacroAndSourcesCollector, MainFileInSourceFiles)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+
+    ASSERT_THAT(collector.sourceFiles(),
+                ElementsAre(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp")));
+}
+
+TEST_F(UsedMacroAndSourcesCollector, ResetMainFileInSourceFiles)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+
+    ASSERT_THAT(collector.sourceFiles(),
+                ElementsAre(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp")));
+}
+
+TEST_F(UsedMacroAndSourcesCollector, DontDuplicateSourceFiles)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+    collector.collect();
+
+    collector.collect();
+
+    ASSERT_THAT(collector.sourceFiles(),
+                UnorderedElementsAre(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"),
+                                     filePathId(TESTDATA_DIR "/symbolscollector_header1.h"),
+                                     filePathId(TESTDATA_DIR "/symbolscollector_header2.h")));
+}
+
+TEST_F(UsedMacroAndSourcesCollector, ClearSourceFiles)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+
+    collector.clear();
+
+    ASSERT_THAT(collector.sourceFiles(), IsEmpty());
+}
+
+TEST_F(UsedMacroAndSourcesCollector, ClearFileStatus)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+    collector.collect();
+
+    collector.clear();
+
+    ASSERT_THAT(collector.fileStatuses(), IsEmpty());
+}
+
+TEST_F(UsedMacroAndSourcesCollector, ClearUsedMacros)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_defines.h"), {"cc"});
+    collector.collect();
+
+    collector.clear();
+
+    ASSERT_THAT(collector.usedMacros(), IsEmpty());
+}
+
+TEST_F(UsedMacroAndSourcesCollector, ClearSourceDependencies)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main2.cpp"), {"cc", "-I" TESTDATA_DIR});
+    collector.collect();
+
+    collector.clear();
+
+    ASSERT_THAT(collector.sourceDependencies(), IsEmpty());
+}
+
+TEST_F(UsedMacroAndSourcesCollector, DontCollectSourceFilesAfterFilesAreCleared)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+
+    collector.clear();
+    collector.collect();
+
+    ASSERT_THAT(collector.sourceFiles(), IsEmpty());
+}
+
+TEST_F(UsedMacroAndSourcesCollector, DontCollectFileStatusAfterFilesAreCleared)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+
+    collector.clear();
+    collector.collect();
+
+    ASSERT_THAT(collector.fileStatuses(), IsEmpty());
+}
+
+TEST_F(UsedMacroAndSourcesCollector, DontCollectUsedMacrosAfterFilesAreCleared)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+
+    collector.clear();
+    collector.collect();
+
+    ASSERT_THAT(collector.usedMacros(), IsEmpty());
+}
+
+
+TEST_F(UsedMacroAndSourcesCollector, DontCollectSourceDependenciesAfterFilesAreCleared)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+
+    collector.clear();
+    collector.collect();
+
+    ASSERT_THAT(collector.sourceDependencies(), IsEmpty());
+}
+
+TEST_F(UsedMacroAndSourcesCollector, CollectUsedMacrosWithExternalDefine)
+{
+    auto fileId = filePathId(TESTDATA_DIR "/symbolscollector_defines.h");
+    collector.addFile(fileId, {"cc", "-DCOMPILER_ARGUMENT"});
+
+    collector.collect();
+
+    ASSERT_THAT(collector.usedMacros(),
+                ElementsAre(Eq(UsedMacro{"DEFINED", fileId}),
+                            Eq(UsedMacro{"IF_DEFINE", fileId}),
+                            Eq(UsedMacro{"__clang__", fileId}),
+                            Eq(UsedMacro{"CLASS_EXPORT", fileId}),
+                            Eq(UsedMacro{"IF_NOT_DEFINE", fileId}),
+                            Eq(UsedMacro{"MACRO_EXPANSION", fileId}),
+                            Eq(UsedMacro{"COMPILER_ARGUMENT", fileId})));
+}
+
+TEST_F(UsedMacroAndSourcesCollector, CollectUsedMacrosWithoutExternalDefine)
+{
+    auto fileId = filePathId(TESTDATA_DIR "/symbolscollector_defines.h");
+    collector.addFile(fileId, {"cc"});
+
+    collector.collect();
+
+    ASSERT_THAT(collector.usedMacros(),
+                ElementsAre(Eq(UsedMacro{"DEFINED", fileId}),
+                            Eq(UsedMacro{"IF_DEFINE", fileId}),
+                            Eq(UsedMacro{"__clang__", fileId}),
+                            Eq(UsedMacro{"CLASS_EXPORT", fileId}),
+                            Eq(UsedMacro{"IF_NOT_DEFINE", fileId}),
+                            Eq(UsedMacro{"MACRO_EXPANSION", fileId}),
+                            Eq(UsedMacro{"COMPILER_ARGUMENT", fileId})));
+}
+
+TEST_F(UsedMacroAndSourcesCollector, DontCollectHeaderGuards)
+{
+    auto fileId = filePathId(TESTDATA_DIR "/symbolscollector_defines.h");
+    collector.addFile(fileId, {"cc"});
+
+    collector.collect();
+
+    ASSERT_THAT(collector.usedMacros(),
+                Not(Contains(Eq(UsedMacro{"SYMBOLSCOLLECTOR_DEFINES_H", fileId}))));
+}
+
+TEST_F(UsedMacroAndSourcesCollector, DISABLED_DontCollectDynamicLibraryExports)
+{
+    auto fileId = filePathId(TESTDATA_DIR "/symbolscollector_defines.h");
+    collector.addFile(fileId, {"cc"});
+
+    collector.collect();
+
+    ASSERT_THAT(collector.usedMacros(),
+                Not(Contains(Eq(UsedMacro{"CLASS_EXPORT", fileId}))));
+}
+
+TEST_F(UsedMacroAndSourcesCollector, CollectFileStatuses)
+{
+    collector.addFile(filePathId(TESTDATA_DIR "/symbolscollector_main.cpp"), {"cc"});
+
+    collector.collect();
+
+    ASSERT_THAT(collector.fileStatuses(),
+                ElementsAre(
+                    fileStatus(TESTDATA_DIR "/symbolscollector_main.cpp"),
+                    fileStatus(TESTDATA_DIR "/symbolscollector_header1.h"),
+                    fileStatus(TESTDATA_DIR "/symbolscollector_header2.h")));
+}
+
+TEST_F(UsedMacroAndSourcesCollector, CollectSourceDependencies)
+{
+    auto mainFileId = filePathId(TESTDATA_DIR "/symbolscollector_main2.cpp");
+    auto header1FileId = filePathId(TESTDATA_DIR "/symbolscollector_header1.h");
+    auto header2FileId = filePathId(TESTDATA_DIR "/symbolscollector_header2.h");
+    auto header3FileId = filePathId(TESTDATA_DIR "/symbolscollector_header3.h");
+    collector.addFile(mainFileId, {"cc", "-I" TESTDATA_DIR});
+
+    collector.collect();
+
+    ASSERT_THAT(collector.sourceDependencies(),
+                UnorderedElementsAre(SourceDependency(mainFileId, header1FileId),
+                                     SourceDependency(mainFileId, header3FileId),
+                                     SourceDependency(header3FileId, header2FileId),
+                                     SourceDependency(header1FileId, header2FileId)));
+}
+}