ClangRefactoring: Ensure that database is written to log after indexing

Change-Id: Ic2473d9fe8dc9b41a7da728c9e1b5202524c1a79
Reviewed-by: Tim Jenssen <tim.jenssen@qt.io>

src/libs/sqlite/sqlite-lib.pri

@@ -29,6 +29,7 @@ SOURCES += \
 HEADERS += \
     $$PWD/createtablesqlstatementbuilder.h \
     $$PWD/sqlitedatabasebackend.h \
+    $$PWD/sqlitedatabaseinterface.h \
     $$PWD/sqliteexception.h \
     $$PWD/sqliteglobal.h \
     $$PWD/sqlitereadstatement.h \

src/libs/sqlite/sqlitedatabase.h

@@ -26,6 +26,7 @@
 #pragma once
 
 #include "sqlitedatabasebackend.h"
+#include "sqlitedatabaseinterface.h"
 #include "sqliteglobal.h"
 #include "sqlitetable.h"
 #include "sqlitetransaction.h"
@@ -43,7 +44,7 @@ using namespace std::chrono_literals;
 class ReadStatement;
 class WriteStatement;
 
-class SQLITE_EXPORT Database final : public TransactionInterface
+class SQLITE_EXPORT Database final : public TransactionInterface, public DatabaseInterface
 {
     template <typename Database>
     friend class Statement;
@@ -105,19 +106,18 @@ public:
         return m_databaseBackend.changesCount();
     }
 
-    int totalChangesCount()
+    int totalChangesCount() { return m_databaseBackend.totalChangesCount(); }
-    {
+
-        return m_databaseBackend.totalChangesCount();
+    void walCheckpointFull() override { m_databaseBackend.walCheckpointFull(); }
-    }
 
 private:
-    void deferredBegin();
+    void deferredBegin() override;
-    void immediateBegin();
+    void immediateBegin() override;
-    void exclusiveBegin();
+    void exclusiveBegin() override;
-    void commit();
+    void commit() override;
-    void rollback();
+    void rollback() override;
-    void lock();
+    void lock() override;
-    void unlock();
+    void unlock() override;
 
     void initializeTables();
     void registerTransactionStatements();

src/libs/sqlite/sqlitedatabasebackend.cpp

@@ -406,6 +406,27 @@ void DatabaseBackend::setBusyTimeout(std::chrono::milliseconds timeout)
     sqlite3_busy_timeout(m_databaseHandle, int(timeout.count()));
 }
 
+void DatabaseBackend::walCheckpointFull()
+{
+    int resultCode = sqlite3_wal_checkpoint_v2(m_databaseHandle,
+                                               nullptr,
+                                               SQLITE_CHECKPOINT_TRUNCATE,
+                                               nullptr,
+                                               nullptr);
+
+    switch (resultCode) {
+    case SQLITE_OK:
+        break;
+    case SQLITE_BUSY:
+        throw DatabaseIsBusy("DatabaseBackend::walCheckpointFull: Operation could not concluded "
+                             "because database is busy!");
+    case SQLITE_ERROR:
+        throwException("DatabaseBackend::walCheckpointFull: Error occurred!");
+    case SQLITE_MISUSE:
+        throwExceptionStatic("DatabaseBackend::walCheckpointFull: Misuse of database!");
+    }
+}
+
 void DatabaseBackend::throwExceptionStatic(const char *whatHasHappens)
 {
     throw Exception(whatHasHappens);

src/libs/sqlite/sqlitedatabasebackend.h

@@ -85,6 +85,8 @@ public:
 
     void setBusyTimeout(std::chrono::milliseconds timeout);
 
+    void walCheckpointFull();
+
 protected:
     bool databaseIsOpen() const;
 

src/libs/sqlite/sqlitedatabaseinterface.h

@@ -0,0 +1,37 @@
+/****************************************************************************
+**
+** Copyright (C) 2019 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+namespace Sqlite {
+class DatabaseInterface
+{
+public:
+    virtual void walCheckpointFull() = 0;
+
+protected:
+    ~DatabaseInterface() = default;
+};
+} // namespace Sqlite

src/tools/clangrefactoringbackend/source/symbolindexertaskqueue.h

@@ -30,6 +30,7 @@
 
 #include <filepathid.h>
 #include <progresscounter.h>
+#include <sqlitedatabase.h>
 #include <taskschedulerinterface.h>
 
 #include <utils/algorithm.h>
@@ -38,10 +39,6 @@
 #include <functional>
 #include <vector>
 
-namespace Sqlite {
-class TransactionInterface;
-}
-
 namespace ClangBackEnd {
 
 class SymbolsCollectorInterface;
@@ -53,9 +50,11 @@ public:
     using Task = SymbolIndexerTask::Callable;
 
     SymbolIndexerTaskQueue(TaskSchedulerInterface<Task> &symbolIndexerTaskScheduler,
-                           ProgressCounter &progressCounter)
+                           ProgressCounter &progressCounter,
-        : m_symbolIndexerScheduler(symbolIndexerTaskScheduler),
+                           Sqlite::DatabaseInterface &database)
-          m_progressCounter(progressCounter)
+        : m_symbolIndexerScheduler(symbolIndexerTaskScheduler)
+        , m_progressCounter(progressCounter)
+        , m_database(database)
     {}
 
     void addOrUpdateTasks(std::vector<SymbolIndexerTask> &&tasks)
@@ -94,12 +93,21 @@ public:
 
     void processEntries()
     {
-        uint taskCount = m_symbolIndexerScheduler.slotUsage().free;
+        auto slotUsage = m_symbolIndexerScheduler.slotUsage();
+        uint taskCount = slotUsage.free;
 
         auto newEnd = std::prev(m_tasks.end(), std::min<int>(int(taskCount), int(m_tasks.size())));
         m_symbolIndexerScheduler.addTasks({std::make_move_iterator(newEnd),
                                            std::make_move_iterator(m_tasks.end())});
         m_tasks.erase(newEnd, m_tasks.end());
+
+        if (m_tasks.empty() && slotUsage.used == 0) {
+            try {
+                m_database.walCheckpointFull();
+            } catch (Sqlite::Exception &exception) {
+                exception.printWarning();
+            }
+        }
     }
     void syncTasks();
 
@@ -108,6 +116,7 @@ private:
     std::vector<SymbolIndexerTask> m_tasks;
     TaskSchedulerInterface<Task> &m_symbolIndexerScheduler;
     ProgressCounter &m_progressCounter;
+    Sqlite::DatabaseInterface &m_database;
 };
 
 } // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/symbolindexing.h

@@ -105,6 +105,7 @@ public:
                     m_projectPartsStorage,
                     m_modifiedTimeChecker,
                     environment)
+        , m_indexerQueue(m_indexerScheduler, m_progressCounter, database)
         , m_indexerScheduler(m_collectorManger,
                              m_indexerQueue,
                              m_progressCounter,
@@ -152,7 +153,7 @@ private:
     ModifiedTimeChecker<ClangBackEnd::SourceTimeStamps> m_modifiedTimeChecker{getModifiedTime,
                                                                               m_filePathCache};
     SymbolIndexer m_indexer;
-    SymbolIndexerTaskQueue m_indexerQueue{m_indexerScheduler, m_progressCounter};
+    SymbolIndexerTaskQueue m_indexerQueue;
     SymbolIndexerTaskScheduler m_indexerScheduler;
 };
 

tests/unit/unittest/mocksqlitedatabase.h

@@ -31,12 +31,13 @@
 #include "mocksqlitetransactionbackend.h"
 #include "mocksqlitewritestatement.h"
 
+#include <sqlitedatabaseinterface.h>
 #include <sqlitetable.h>
 #include <sqlitetransaction.h>
 
 #include <utils/smallstringview.h>
 
-class MockSqliteDatabase : public MockSqliteTransactionBackend
+class MockSqliteDatabase : public MockSqliteTransactionBackend, public Sqlite::DatabaseInterface
 {
 public:
     using ReadStatement = NiceMock<MockSqliteReadStatement>;
@@ -56,5 +57,7 @@ public:
 
     MOCK_METHOD1(setIsInitialized,
                  void (bool));
+
+    MOCK_METHOD0(walCheckpointFull, void());
 };
 

tests/unit/unittest/symbolindexer-test.cpp

@@ -267,7 +267,8 @@ protected:
                                         mockProjectPartsStorage,
                                         mockModifiedTimeChecker,
                                         testEnvironment};
-    SymbolIndexerTaskQueue indexerQueue{indexerScheduler, progressCounter};
+    NiceMock<MockSqliteDatabase> mockSqliteDatabase;
+    SymbolIndexerTaskQueue indexerQueue{indexerScheduler, progressCounter, mockSqliteDatabase};
     Scheduler indexerScheduler{collectorManger,
                                indexerQueue,
                                progressCounter,

tests/unit/unittest/symbolindexertaskqueue-test.cpp

@@ -25,6 +25,7 @@
 
 #include "googletest.h"
 
+#include "mocksqlitedatabase.h"
 #include "mocktaskscheduler.h"
 
 #include <symbolindexertaskqueue.h>
@@ -54,7 +55,8 @@ protected:
     NiceMock<MockFunction<void(int, int)>> mockSetProgressCallback;
     ClangBackEnd::ProgressCounter progressCounter{mockSetProgressCallback.AsStdFunction()};
     NiceMock<MockTaskScheduler<Callable>> mockTaskScheduler;
-    ClangBackEnd::SymbolIndexerTaskQueue queue{mockTaskScheduler, progressCounter};
+    NiceMock<MockSqliteDatabase> mockSqliteDatabase;
+    ClangBackEnd::SymbolIndexerTaskQueue queue{mockTaskScheduler, progressCounter, mockSqliteDatabase};
 };
 
 TEST_F(SymbolIndexerTaskQueue, AddTasks)
@@ -208,4 +210,35 @@ TEST_F(SymbolIndexerTaskQueue, ProcessTasksRemovesProcessedTasks)
 
     ASSERT_THAT(queue.tasks(), SizeIs(1));
 }
+
+TEST_F(SymbolIndexerTaskQueue,
+       ProcessTasksWritesBackTheDatabaseLogIfTheQueueIsEmptyAndTheIndexerHasNothingToDo)
+{
+    InSequence s;
+
+    EXPECT_CALL(mockTaskScheduler, slotUsage()).WillRepeatedly(Return(SlotUsage{2, 0}));
+    EXPECT_CALL(mockSqliteDatabase, walCheckpointFull());
+
+    queue.processEntries();
 }
+
+TEST_F(SymbolIndexerTaskQueue, ProcessTasksDoesNotWritesBackTheDatabaseLogIfTheIndexerHasSomethingToDo)
+{
+    InSequence s;
+
+    EXPECT_CALL(mockTaskScheduler, slotUsage()).WillRepeatedly(Return(SlotUsage{1, 1}));
+    EXPECT_CALL(mockSqliteDatabase, walCheckpointFull()).Times(0);
+
+    queue.processEntries();
+}
+
+TEST_F(SymbolIndexerTaskQueue, HandleExeptionInWalCheckPoint)
+{
+    InSequence s;
+
+    EXPECT_CALL(mockTaskScheduler, slotUsage()).WillRepeatedly(Return(SlotUsage{2, 0}));
+    EXPECT_CALL(mockSqliteDatabase, walCheckpointFull()).WillOnce(Throw(Sqlite::DatabaseIsBusy{""}));
+
+    queue.processEntries();
+}
+} // namespace