Clang: Use PCHs for indexing

As generating the AST is quite expensive it would be very useful to cache
the not changed include. So we generate PCHs for include outside of a
project part. With this change this PCHs are used by the indexer.

For that they are save to the symbol database by the PCH manager and when
fetched by the symbol indexer.

Change-Id: I7a5b07cfb32d72d50dc52d2b108cd41727a7bfc7
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>

src/plugins/clangpchmanager/clangpchmanager-source.pri

@@ -12,7 +12,9 @@ HEADERS += \
     $$PWD/pchmanagerconnectionclient.h \
     $$PWD/clangpchmanager_global.h \
     $$PWD/projectupdater.h \
-    $$PWD/pchmanagerprojectupdater.h
+    $$PWD/pchmanagerprojectupdater.h \
+    $$PWD/precompiledheaderstorage.h \
+    $$PWD/precompiledheaderstorageinterface.h
 
 
 SOURCES += \
@@ -20,5 +22,6 @@ SOURCES += \
     $$PWD/pchmanagernotifierinterface.cpp \
     $$PWD/pchmanagerconnectionclient.cpp \
     $$PWD/projectupdater.cpp \
-    $$PWD/pchmanagerprojectupdater.cpp
+    $$PWD/pchmanagerprojectupdater.cpp \
+    $$PWD/precompiledheaderstorageinterface.cpp
 

src/plugins/clangpchmanager/clangpchmanagerplugin.cpp

@@ -27,6 +27,7 @@
 
 #include "pchmanagerconnectionclient.h"
 #include "pchmanagerclient.h"
+#include "precompiledheaderstorage.h"
 #include "qtcreatorprojectupdater.h"
 
 #include <filepathcaching.h>
@@ -57,7 +58,8 @@ public:
     Sqlite::Database database{Utils::PathString{Core::ICore::userResourcePath() + "/symbol-experimental-v1.db"}};
     ClangBackEnd::RefactoringDatabaseInitializer<Sqlite::Database> databaseInitializer{database};
     ClangBackEnd::FilePathCaching filePathCache{database};
-    PchManagerClient pchManagerClient;
+    PrecompiledHeaderStorage<> preCompiledHeaderStorage{database};
+    PchManagerClient pchManagerClient{preCompiledHeaderStorage};
     PchManagerConnectionClient connectionClient{&pchManagerClient};
     QtCreatorProjectUpdater<PchManagerProjectUpdater> projectUpdate{connectionClient.serverProxy(),
                                                                     pchManagerClient,

src/plugins/clangpchmanager/clangpchmanagerplugin.h

@@ -25,6 +25,8 @@
 
 #pragma once
 
+#include "clangpchmanager_global.h"
+
 #include <extensionsystem/iplugin.h>
 
 #include <memory>
@@ -34,7 +36,7 @@ namespace ClangPchManager {
 class ClangPchManagerPluginData;
 class PchManagerClient;
 
-class ClangPchManagerPlugin : public ExtensionSystem::IPlugin
+class CLANGPCHMANAGER_EXPORT ClangPchManagerPlugin : public ExtensionSystem::IPlugin
 {
     Q_OBJECT
     Q_PLUGIN_METADATA(IID "org.qt-project.Qt.QtCreatorPlugin" FILE "ClangPchManager.json")

src/plugins/clangpchmanager/pchmanagerclient.cpp

@@ -34,6 +34,11 @@
 
 namespace ClangPchManager {
 
+PchManagerClient::PchManagerClient(PrecompiledHeaderStorageInterface &precompiledHeaderStorage)
+    : m_precompiledHeaderStorage(precompiledHeaderStorage)
+{
+}
+
 void PchManagerClient::alive()
 {
     if (m_connectionClient)
@@ -42,17 +47,23 @@ void PchManagerClient::alive()
 
 void PchManagerClient::precompiledHeadersUpdated(ClangBackEnd::PrecompiledHeadersUpdatedMessage &&message)
 {
-    for (const ClangBackEnd::ProjectPartPch &projectPartPch : message.projectPartPchs()) {
-        precompiledHeaderUpdated(QString(projectPartPch.projectPartId),
-                                 QString(projectPartPch.pchPath),
-                                 projectPartPch.lastModified);
+    for (ClangBackEnd::ProjectPartPch &projectPartPch : message.takeProjectPartPchs()) {
+        const QString projectPartId{projectPartPch.projectPartId};
+        const QString pchPath{projectPartPch.pchPath};
+        addPchToDatabase(projectPartPch);
+        addProjectPartPch(std::move(projectPartPch));
+        precompiledHeaderUpdated(projectPartId, pchPath, projectPartPch.lastModified);
     }
 }
 
 void PchManagerClient::precompiledHeaderRemoved(const QString &projectPartId)
 {
-    for (auto notifier : m_notifiers)
+    for (auto notifier : m_notifiers) {
+        Utils::SmallString id(projectPartId);
+        removePchFromDatabase(id);
+        removeProjectPartPch(id);
         notifier->precompiledHeaderRemoved(projectPartId);
+    }
 }
 
 void PchManagerClient::setConnectionClient(PchManagerConnectionClient *connectionClient)
@@ -60,6 +71,21 @@ void PchManagerClient::setConnectionClient(PchManagerConnectionClient *connectio
     m_connectionClient = connectionClient;
 }
 
+Utils::optional<ClangBackEnd::ProjectPartPch> PchManagerClient::projectPartPch(Utils::SmallStringView projectPartId) const
+{
+    auto found = std::lower_bound(m_projectPartPchs.cbegin(),
+                                  m_projectPartPchs.cend(),
+                                  projectPartId,
+                                  [] (const auto &projectPartPch, auto projectPartId) {
+            return projectPartId < projectPartPch.projectPartId;
+    });
+
+    if (found != m_projectPartPchs.end() && found->projectPartId == projectPartId)
+        return *found;
+
+    return Utils::nullopt;
+}
+
 void PchManagerClient::attach(PchManagerNotifierInterface *notifier)
 {
     m_notifiers.push_back(notifier);
@@ -76,6 +102,48 @@ void PchManagerClient::detach(PchManagerNotifierInterface *notifierToBeDeleted)
     m_notifiers.erase(newEnd, m_notifiers.end());
 }
 
+void PchManagerClient::removeProjectPartPch(Utils::SmallStringView projectPartId)
+{
+    auto found = std::lower_bound(m_projectPartPchs.begin(),
+                                  m_projectPartPchs.end(),
+                                  projectPartId,
+                                  [] (const auto &projectPartPch, auto projectPartId) {
+            return projectPartId < projectPartPch.projectPartId;
+    });
+
+    if (found != m_projectPartPchs.end() && found->projectPartId == projectPartId) {
+        *found = std::move(m_projectPartPchs.back());
+        m_projectPartPchs.pop_back();
+    }
+}
+
+void PchManagerClient::addPchToDatabase(const ClangBackEnd::ProjectPartPch &projectPartPch)
+{
+    m_precompiledHeaderStorage.insertPrecompiledHeader(projectPartPch.projectPartId,
+                                                       projectPartPch.pchPath,
+                                                       projectPartPch.lastModified);
+}
+
+void PchManagerClient::removePchFromDatabase(const Utils::SmallStringView &projectPartId)
+{
+    m_precompiledHeaderStorage.deletePrecompiledHeader(projectPartId);
+}
+
+void PchManagerClient::addProjectPartPch(ClangBackEnd::ProjectPartPch &&projectPartPch)
+{
+    auto found = std::lower_bound(m_projectPartPchs.begin(),
+                                  m_projectPartPchs.end(),
+                                  projectPartPch.projectPartId,
+                                  [] (const auto &projectPartPch, auto projectPartId) {
+            return projectPartId < projectPartPch.projectPartId;
+    });
+
+    if (found != m_projectPartPchs.end() && found->projectPartId == projectPartPch.projectPartId)
+        *found = std::move(projectPartPch);
+    else
+        m_projectPartPchs.insert(found, std::move(projectPartPch));
+}
+
 const std::vector<PchManagerNotifierInterface *> &PchManagerClient::notifiers() const
 {
     return m_notifiers;

src/plugins/clangpchmanager/pchmanagerclient.h

@@ -25,7 +25,11 @@
 
 #pragma once
 
+#include "clangpchmanager_global.h"
+#include "precompiledheaderstorageinterface.h"
+
 #include <pchmanagerclientinterface.h>
+#include <projectpartpchproviderinterface.h>
 
 #include <vector>
 
@@ -34,10 +38,12 @@ class PchManagerConnectionClient;
 
 class PchManagerNotifierInterface;
 
-class PchManagerClient final : public ClangBackEnd::PchManagerClientInterface
+class CLANGPCHMANAGER_EXPORT PchManagerClient final : public ClangBackEnd::PchManagerClientInterface,
+                                                      public ClangBackEnd::ProjectPartPchProviderInterface
 {
     friend class PchManagerNotifierInterface;
 public:
+    PchManagerClient(PrecompiledHeaderStorageInterface &precompiledHeaderStorage);
     void alive() override;
     void precompiledHeadersUpdated(ClangBackEnd::PrecompiledHeadersUpdatedMessage &&message) override;
 
@@ -45,6 +51,14 @@ public:
 
     void setConnectionClient(PchManagerConnectionClient *connectionClient);
 
+    Utils::optional<ClangBackEnd::ProjectPartPch> projectPartPch(
+            Utils::SmallStringView projectPartId) const override;
+
+    const ClangBackEnd::ProjectPartPchs &projectPartPchs() const override
+    {
+        return m_projectPartPchs;
+    }
+
 unittest_public:
     const std::vector<PchManagerNotifierInterface*> &notifiers() const;
     void precompiledHeaderUpdated(const QString &projectPartId,
@@ -54,9 +68,17 @@ unittest_public:
     void attach(PchManagerNotifierInterface *notifier);
     void detach(PchManagerNotifierInterface *notifier);
 
+    void addProjectPartPch(ClangBackEnd::ProjectPartPch &&projectPartPch);
+    void removeProjectPartPch(Utils::SmallStringView projectPartId);
+
+    void addPchToDatabase(const ClangBackEnd::ProjectPartPch &projectPartPch);
+    void removePchFromDatabase(const Utils::SmallStringView &projectPartId);
+
 private:
+    ClangBackEnd::ProjectPartPchs m_projectPartPchs;
     std::vector<PchManagerNotifierInterface*> m_notifiers;
     PchManagerConnectionClient *m_connectionClient=nullptr;
+    PrecompiledHeaderStorageInterface &m_precompiledHeaderStorage;
 };
 
 } // namespace ClangPchManager

src/plugins/clangpchmanager/pchmanagernotifierinterface.h

@@ -25,7 +25,7 @@
 
 #pragma once
 
-#include <QtGlobal>
+#include "clangpchmanager_global.h"
 
 QT_FORWARD_DECLARE_CLASS(QString)
 
@@ -33,10 +33,10 @@ namespace ClangPchManager {
 
 class PchManagerClient;
 
-class PchManagerNotifierInterface
+class CLANGPCHMANAGER_EXPORT PchManagerNotifierInterface
 {
 public:
-    PchManagerNotifierInterface(PchManagerClient &pchManagerClientso);
+    PchManagerNotifierInterface(PchManagerClient &pchManagerClient);
 
     virtual ~PchManagerNotifierInterface();
     virtual void precompiledHeaderUpdated(const QString &projectPartId,
@@ -44,6 +44,9 @@ public:
                                           long long lastModified) = 0;
     virtual void precompiledHeaderRemoved(const QString &projectPartId) = 0;
 
+    PchManagerNotifierInterface(const PchManagerNotifierInterface &) = delete;
+    void operator=(const PchManagerNotifierInterface &) const = delete;
+
     PchManagerClient &m_pchManagerClient;
 };
 

src/plugins/clangpchmanager/precompiledheaderstorage.h

@@ -0,0 +1,80 @@
+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include "precompiledheaderstorageinterface.h"
+
+#include <sqlitetransaction.h>
+
+#include <utils/smallstringview.h>
+
+namespace ClangPchManager {
+
+template<typename Database=Sqlite::Database>
+class PrecompiledHeaderStorage final : public PrecompiledHeaderStorageInterface
+{
+    using ReadStatement = typename Database::ReadStatement;
+    using WriteStatement = typename Database::WriteStatement;
+public:
+    PrecompiledHeaderStorage(Database &database)
+        : m_transaction(database),
+          m_database(database)
+    {
+        m_transaction.commit();
+    }
+
+    void insertPrecompiledHeader(Utils::SmallStringView projectPartName,
+                                 Utils::SmallStringView pchPath,
+                                 long long pchBuildTime) override
+    {
+        m_insertProjectPartStatement.write(projectPartName);
+        m_insertPrecompiledHeaderStatement .write(projectPartName, pchPath, pchBuildTime);
+    }
+
+    void deletePrecompiledHeader(Utils::SmallStringView projectPartName) override
+    {
+        m_deletePrecompiledHeaderStatement.write(projectPartName);
+    }
+
+
+public:
+    Sqlite::ImmediateNonThrowingDestructorTransaction m_transaction;
+    Database &m_database;
+    WriteStatement m_insertPrecompiledHeaderStatement {
+        "INSERT OR REPLACE INTO precompiledHeaders(projectPartId, pchPath, pchBuildTime) VALUES((SELECT projectPartId FROM projectParts WHERE projectPartName = ?),?,?)",
+        m_database
+    };
+    WriteStatement m_insertProjectPartStatement{
+        "INSERT OR IGNORE INTO projectParts(projectPartName) VALUES (?)",
+        m_database
+    };
+    WriteStatement m_deletePrecompiledHeaderStatement{
+        "DELETE FROM precompiledHeaders WHERE projectPartId = (SELECT projectPartId FROM projectParts WHERE projectPartName = ?)",
+        m_database
+    };
+};
+
+}

src/plugins/clangpchmanager/precompiledheaderstorageinterface.cpp

@@ -0,0 +1,32 @@
+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "precompiledheaderstorageinterface.h"
+
+namespace ClangPchManager {
+
+PrecompiledHeaderStorageInterface::~PrecompiledHeaderStorageInterface() = default;
+
+} // namespace ClangPchManager

src/plugins/clangpchmanager/precompiledheaderstorageinterface.h

@@ -0,0 +1,48 @@
+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <utils/smallstringview.h>
+
+namespace ClangPchManager {
+
+class PrecompiledHeaderStorageInterface
+{
+public:
+    PrecompiledHeaderStorageInterface() = default;
+    virtual ~PrecompiledHeaderStorageInterface();
+
+    PrecompiledHeaderStorageInterface(const PrecompiledHeaderStorageInterface&) = delete;
+    PrecompiledHeaderStorageInterface &operator=(const PrecompiledHeaderStorageInterface&) = delete;
+
+    virtual void insertPrecompiledHeader(Utils::SmallStringView projectPartName,
+                                         Utils::SmallStringView pchPath,
+                                         long long pchBuildTime) = 0;
+
+    virtual void deletePrecompiledHeader(Utils::SmallStringView projectPartName) = 0;
+};
+
+} // namespace ClangPchManager

src/plugins/clangpchmanager/projectupdater.cpp

@@ -29,8 +29,8 @@
 
 #include <filepathid.h>
 #include <pchmanagerserverinterface.h>
-#include <removepchprojectpartsmessage.h>
-#include <updatepchprojectpartsmessage.h>
+#include <removeprojectpartsmessage.h>
+#include <updateprojectpartsmessage.h>
 
 #include <cpptools/compileroptionsbuilder.h>
 #include <cpptools/projectpart.h>
@@ -67,17 +67,17 @@ void ProjectUpdater::updateProjectParts(const std::vector<CppTools::ProjectPart
 {
     m_excludedPaths = createExcludedPaths(generatedFiles);
 
-    ClangBackEnd::UpdatePchProjectPartsMessage message{toProjectPartContainers(projectParts),
+    ClangBackEnd::UpdateProjectPartsMessage message{toProjectPartContainers(projectParts),
                                                        std::move(generatedFiles)};
 
-    m_server.updatePchProjectParts(std::move(message));
+    m_server.updateProjectParts(std::move(message));
 }
 
 void ProjectUpdater::removeProjectParts(const QStringList &projectPartIds)
 {
-    ClangBackEnd::RemovePchProjectPartsMessage message{Utils::SmallStringVector(projectPartIds)};
+    ClangBackEnd::RemoveProjectPartsMessage message{Utils::SmallStringVector(projectPartIds)};
 
-    m_server.removePchProjectParts(std::move(message));
+    m_server.removeProjectParts(std::move(message));
 }
 
 void ProjectUpdater::setExcludedPaths(Utils::PathStringVector &&excludedPaths)