Clang: Add time stamp based filtering for project parts

The source ids are now filtered by the modified time. If the modified time
in the database is older than the modified time of the file it will be
parsed. If it is not newer it will be not parsed.

Change-Id: I4ade3443dd66573ac88053a2cafa600e54cfe973
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>

src/tools/clangrefactoringbackend/source/clangrefactoringbackend-source.pri

@@ -19,7 +19,8 @@ HEADERS += \
     $$PWD/sourcedependency.h \
     $$PWD/filestatus.h \
     $$PWD/projectpartartefactexception.h \
-    $$PWD/projectpartartefact.h
+    $$PWD/projectpartartefact.h \
+    $$PWD/filestatuscache.h
 
 !isEmpty(LIBTOOLING_LIBS) {
 SOURCES += \
@@ -67,5 +68,5 @@ SOURCES += \
     $$PWD/symbolindexer.cpp \
     $$PWD/symbolentry.cpp \
     $$PWD/symbolstorageinterface.cpp \
-    $$PWD/projectpartartefact.cpp
-
+    $$PWD/projectpartartefact.cpp \
+    $$PWD/filestatuscache.cpp

src/tools/clangrefactoringbackend/source/filestatuscache.cpp

@@ -0,0 +1,93 @@
+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "filestatuscache.h"
+
+#include <QDateTime>
+#include <QFileInfo>
+
+namespace ClangBackEnd {
+
+FileStatusCache::FileStatusCache(FilePathCachingInterface &filePathCache)
+    : m_filePathCache(filePathCache)
+{
+
+}
+
+long long FileStatusCache::lastModifiedTime(FilePathId filePathId) const
+{
+    return findEntry(filePathId).lastModified;
+}
+
+void FileStatusCache::update(FilePathId filePathId)
+{
+    auto found = std::lower_bound(m_cacheEntries.begin(),
+                                  m_cacheEntries.end(),
+                                  Internal::FileStatusCacheEntry{filePathId},
+                                  [] (const auto &first, const auto &second) {
+        return first.filePathId < second.filePathId;
+    });
+
+    if (found != m_cacheEntries.end() && found->filePathId == filePathId) {
+        QFileInfo fileInfo = qFileInfo(filePathId);
+        found->lastModified = fileInfo.lastModified().toSecsSinceEpoch();
+    }
+}
+
+FileStatusCache::size_type FileStatusCache::size() const
+{
+    return m_cacheEntries.size();
+}
+
+Internal::FileStatusCacheEntry FileStatusCache::findEntry(FilePathId filePathId) const
+{
+    auto found = std::lower_bound(m_cacheEntries.begin(),
+                                  m_cacheEntries.end(),
+                                  Internal::FileStatusCacheEntry{filePathId},
+                                  [] (const auto &first, const auto &second) {
+        return first.filePathId < second.filePathId;
+    });
+
+    if (found != m_cacheEntries.end() && found->filePathId == filePathId)
+        return *found;
+
+    QFileInfo fileInfo = qFileInfo(filePathId);
+    auto inserted = m_cacheEntries.emplace(found,
+                                           filePathId,
+                                           fileInfo.lastModified().toSecsSinceEpoch());
+
+    return *inserted;
+}
+
+QFileInfo FileStatusCache::qFileInfo(FilePathId filePathId) const
+{
+    QFileInfo fileInfo(QString(m_filePathCache.filePath(filePathId)));
+
+    fileInfo.refresh();
+
+    return fileInfo;
+}
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/filestatuscache.h

@@ -0,0 +1,74 @@
+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <filepathcachinginterface.h>
+
+QT_FORWARD_DECLARE_CLASS(QFileInfo)
+
+namespace ClangBackEnd {
+
+namespace Internal {
+class FileStatusCacheEntry
+{
+public:
+    FileStatusCacheEntry(ClangBackEnd::FilePathId filePathId,
+                         long long lastModified = 0)
+        : filePathId(filePathId),
+          lastModified(lastModified)
+    {}
+
+public:
+    ClangBackEnd::FilePathId filePathId;
+    long long lastModified;
+};
+
+using FileStatusCacheEntries = std::vector<FileStatusCacheEntry>;
+
+}
+
+class FileStatusCache
+{
+public:
+    using size_type = Internal::FileStatusCacheEntries::size_type;
+
+    FileStatusCache(FilePathCachingInterface &filePathCache);
+
+    long long lastModifiedTime(ClangBackEnd::FilePathId filePathId) const;
+    void update(ClangBackEnd::FilePathId filePathId);
+
+    size_type size() const;
+
+private:
+    Internal::FileStatusCacheEntry findEntry(ClangBackEnd::FilePathId filePathId) const;
+    QFileInfo qFileInfo(ClangBackEnd::FilePathId filePathId) const;
+
+private:
+    mutable Internal::FileStatusCacheEntries m_cacheEntries;
+    FilePathCachingInterface &m_filePathCache;
+};
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/storagesqlitestatementfactory.h

@@ -225,5 +225,9 @@ public:
        "SELECT compilerArguments, compilerMacros, includeSearchPaths, projectPartId FROM projectParts WHERE projectPartName = ?",
        database
    };
+   ReadStatement getLowestLastModifiedTimeOfDependencies{
+       "WITH RECURSIVE sourceIds(sourceId) AS (VALUES(?) UNION SELECT dependencySourceId FROM sourceDependencies, sourceIds WHERE sourceDependencies.sourceId = sourceIds.sourceId) SELECT min(lastModified) FROM fileStatuses, sourceIds WHERE fileStatuses.sourceId = sourceIds.sourceId",
+       database
+   };
 };
 } // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/symbolindexer.cpp

@@ -31,11 +31,13 @@ SymbolIndexer::SymbolIndexer(SymbolsCollectorInterface &symbolsCollector,
                              SymbolStorageInterface &symbolStorage,
                              ClangPathWatcherInterface &pathWatcher,
                              FilePathCachingInterface &filePathCache,
+                             FileStatusCache &fileStatusCache,
                              Sqlite::TransactionInterface &transactionInterface)
     : m_symbolsCollector(symbolsCollector),
       m_symbolStorage(symbolStorage),
       m_pathWatcher(pathWatcher),
       m_filePathCache(filePathCache),
+      m_fileStatusCache(fileStatusCache),
       m_transactionInterface(transactionInterface)
 {
     pathWatcher.setNotifier(this);
@@ -52,7 +54,9 @@ void SymbolIndexer::updateProjectPart(V2::ProjectPartContainer &&projectPart,
 {
     m_symbolsCollector.clear();
 
-    if (compilerMacrosOrIncludeSearchPathsAreDifferent(projectPart)) {
+    FilePathIds sourcePathIds = updatableFilePathIds(projectPart);
+
+    if (!sourcePathIds.empty()) {
         m_symbolsCollector.addFiles(projectPart.sourcePathIds(), projectPart.arguments());
 
         m_symbolsCollector.addUnsavedFiles(generatedFiles);
@@ -94,6 +98,7 @@ void SymbolIndexer::pathsChanged(const FilePathIds &filePathIds)
 void SymbolIndexer::updateChangedPath(FilePathId filePathId)
 {
     m_symbolsCollector.clear();
+    m_fileStatusCache.update(filePathId);
 
     const Utils::optional<ProjectPartArtefact> optionalArtefact = m_symbolStorage.fetchProjectPartArtefact(filePathId);
 
@@ -136,4 +141,27 @@ bool SymbolIndexer::compilerMacrosOrIncludeSearchPathsAreDifferent(const V2::Pro
     return true;
 }
 
+FilePathIds SymbolIndexer::filterChangedFiles(const V2::ProjectPartContainer &projectPart) const
+{
+    FilePathIds ids;
+    ids.reserve(projectPart.sourcePathIds().size());
+
+    for (const FilePathId &sourceId : projectPart.sourcePathIds()) {
+        long long oldLastModified = m_symbolStorage.fetchLowestLastModifiedTime(sourceId);
+        long long currentLastModified =  m_fileStatusCache.lastModifiedTime(sourceId);
+        if (oldLastModified < currentLastModified)
+            ids.push_back(sourceId);
+    }
+
+    return ids;
+}
+
+FilePathIds SymbolIndexer::updatableFilePathIds(const V2::ProjectPartContainer &projectPart) const
+{
+    if (compilerMacrosOrIncludeSearchPathsAreDifferent(projectPart))
+        return projectPart.sourcePathIds();
+
+    return filterChangedFiles(projectPart);
+}
+
 } // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/symbolindexer.h

@@ -25,6 +25,7 @@
 
 #pragma once
 
+#include "filestatuscache.h"
 #include "symbolscollectorinterface.h"
 #include "symbolstorageinterface.h"
 #include "clangpathwatcher.h"
@@ -41,6 +42,7 @@ public:
                   SymbolStorageInterface &symbolStorage,
                   ClangPathWatcherInterface &pathWatcher,
                   FilePathCachingInterface &filePathCache,
+                  FileStatusCache &fileStatusCache,
                   Sqlite::TransactionInterface &transactionInterface);
 
     void updateProjectParts(V2::ProjectPartContainers &&projectParts,
@@ -55,11 +57,17 @@ public:
     bool compilerMacrosOrIncludeSearchPathsAreDifferent(
             const V2::ProjectPartContainer &projectPart) const;
 
+    FilePathIds filterChangedFiles(
+            const V2::ProjectPartContainer &projectPart) const;
+
+    FilePathIds updatableFilePathIds(const V2::ProjectPartContainer &projectPart) const;
+
 private:
     SymbolsCollectorInterface &m_symbolsCollector;
     SymbolStorageInterface &m_symbolStorage;
     ClangPathWatcherInterface &m_pathWatcher;
     FilePathCachingInterface &m_filePathCache;
+    FileStatusCache &m_fileStatusCache;
     Sqlite::TransactionInterface &m_transactionInterface;
 };
 

src/tools/clangrefactoringbackend/source/symbolindexing.h

@@ -75,7 +75,13 @@ private:
     StatementFactory m_statementFactory;
     Storage m_symbolStorage{m_statementFactory, m_filePathCache};
     ClangPathWatcher<QFileSystemWatcher, QTimer> m_sourceWatcher{m_filePathCache};
-    SymbolIndexer m_indexer{m_collector, m_symbolStorage, m_sourceWatcher, m_filePathCache, m_statementFactory.database};
+    FileStatusCache m_fileStatusCache{m_filePathCache};
+    SymbolIndexer m_indexer{m_collector,
+                            m_symbolStorage,
+                            m_sourceWatcher,
+                            m_filePathCache,
+                            m_fileStatusCache,
+                            m_statementFactory.database};
 };
 
 } // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/symbolstorage.h

@@ -106,6 +106,13 @@ public:
         return statement.template value<ProjectPartArtefact, 4>(projectPartName);
     }
 
+    long long fetchLowestLastModifiedTime(FilePathId sourceId) const override
+    {
+        ReadStatement &statement = m_statementFactory.getLowestLastModifiedTimeOfDependencies;
+
+        return statement.template value<long long>(sourceId.filePathId).value_or(0);
+    }
+
     void insertOrUpdateUsedMacros(const UsedMacros &usedMacros) override
     {
         WriteStatement &insertStatement = m_statementFactory.insertIntoNewUsedMacrosStatement;

src/tools/clangrefactoringbackend/source/symbolstorageinterface.h

@@ -62,6 +62,7 @@ public:
     virtual void insertOrUpdateSourceDependencies(const SourceDependencies &sourceDependencies) = 0;
     virtual Utils::optional<ProjectPartArtefact> fetchProjectPartArtefact(FilePathId sourceId) const = 0;
     virtual Utils::optional<ProjectPartArtefact> fetchProjectPartArtefact(Utils::SmallStringView projectPartName) const = 0;
+    virtual long long fetchLowestLastModifiedTime(FilePathId sourceId) const = 0;
 };
 
 } // namespace ClangBackEnd