Sqlite: Add session support

Session are captured by hooking in the sqlite changes. They are saved in
blobs and containing inserts, update and deletes. Because the are
semantically coupled to translactions we add a

Change-Id: Ie095558ebc50601fcaae32958ebeeb98b72d73b9
Reviewed-by: Tim Jenssen <tim.jenssen@qt.io>
This commit is contained in:
Marco Bubke
2020-05-27 23:48:03 +02:00
committed by Tim Jenssen
parent d6b1281a65
commit a61eff0079
29 changed files with 1253 additions and 48 deletions

View File

@@ -38,9 +38,14 @@ class Unique
friend bool operator==(Unique, Unique) { return true; }
};
enum class AutoIncrement { No, Yes };
class PrimaryKey
{
friend bool operator==(PrimaryKey, PrimaryKey) { return true; }
public:
AutoIncrement autoincrement = AutoIncrement::No;
};
class NotNull

View File

@@ -125,7 +125,12 @@ public:
void operator()(const Unique &) { columnDefinitionString.append(" UNIQUE"); }
void operator()(const PrimaryKey &) { columnDefinitionString.append(" PRIMARY KEY"); }
void operator()(const PrimaryKey &primaryKey)
{
columnDefinitionString.append(" PRIMARY KEY");
if (primaryKey.autoincrement == AutoIncrement::Yes)
columnDefinitionString.append(" AUTOINCREMENT");
}
void operator()(const ForeignKey &foreignKey)
{

View File

@@ -17,6 +17,8 @@ SOURCES += \
$$PWD/sqliteglobal.cpp \
$$PWD/sqlitereadstatement.cpp \
$$PWD/sqlitereadwritestatement.cpp \
$$PWD/sqlitesessionchangeset.cpp \
$$PWD/sqlitesessions.cpp \
$$PWD/sqlitewritestatement.cpp \
$$PWD/sqlstatementbuilder.cpp \
$$PWD/utf8string.cpp \
@@ -33,6 +35,8 @@ HEADERS += \
$$PWD/sqliteglobal.h \
$$PWD/sqlitereadstatement.h \
$$PWD/sqlitereadwritestatement.h \
$$PWD/sqlitesessionchangeset.h \
$$PWD/sqlitesessions.h \
$$PWD/sqlitetransaction.h \
$$PWD/sqlitevalue.h \
$$PWD/sqlitewritestatement.h \
@@ -46,16 +50,16 @@ HEADERS += \
$$PWD/sqliteindex.h \
$$PWD/sqlitebasestatement.h
DEFINES += SQLITE_THREADSAFE=2 SQLITE_ENABLE_FTS5 \
SQLITE_ENABLE_UNLOCK_NOTIFY SQLITE_ENABLE_JSON1 \
SQLITE_DEFAULT_FOREIGN_KEYS=1 SQLITE_TEMP_STORE=2 SQLITE_DEFAULT_PAGE_SIZE=32768 \
DEFINES += SQLITE_THREADSAFE=2 SQLITE_ENABLE_FTS5 SQLITE_ENABLE_UNLOCK_NOTIFY \
SQLITE_ENABLE_JSON1 SQLITE_DEFAULT_FOREIGN_KEYS=1 SQLITE_TEMP_STORE=2 \
SQLITE_DEFAULT_WAL_SYNCHRONOUS=1 SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_MEMSTATUS=0 \
SQLITE_OMIT_DEPRECATED SQLITE_OMIT_DECLTYPE \
SQLITE_MAX_EXPR_DEPTH=0 SQLITE_OMIT_SHARED_CACHE SQLITE_USE_ALLOCA \
SQLITE_ENABLE_MEMORY_MANAGEMENT SQLITE_ENABLE_NULL_TRIM SQLITE_OMIT_EXPLAIN \
SQLITE_OMIT_LOAD_EXTENSION SQLITE_OMIT_UTF16 SQLITE_DQS=0 \
SQLITE_ENABLE_STAT4 HAVE_ISNAN HAVE_FDATASYNC HAVE_MALLOC_USABLE_SIZE \
SQLITE_DEFAULT_MMAP_SIZE=268435456 SQLITE_CORE
SQLITE_DEFAULT_MMAP_SIZE=268435456 SQLITE_CORE SQLITE_ENABLE_SESSION SQLITE_ENABLE_PREUPDATE_HOOK \
SQLITE_LIKE_DOESNT_MATCH_BLOBS
CONFIG(debug, debug|release): DEFINES += SQLITE_ENABLE_API_ARMOR

View File

@@ -3,7 +3,5 @@ win32:QMAKE_CXXFLAGS_DEBUG += -O2
include(../../qtcreatorlibrary.pri)
win32:DEFINES += SQLITE_API=__declspec(dllexport)
unix:DEFINES += SQLITE_API=\"__attribute__((visibility(\\\"default\\\")))\"
include(sqlite-lib.pri)

View File

@@ -67,6 +67,8 @@ public:
return "REAL";
case ColumnType::Text:
return "TEXT";
case ColumnType::Blob:
return "BLOB";
}
Q_UNREACHABLE();

View File

@@ -25,9 +25,10 @@
#include "sqlitedatabase.h"
#include "sqlitereadwritestatement.h"
#include "sqlitesessions.h"
#include "sqlitetable.h"
#include "sqlitetransaction.h"
#include "sqlitereadwritestatement.h"
#include <QFileInfo>
@@ -51,6 +52,7 @@ public:
ReadWriteStatement exclusiveBegin{"BEGIN EXCLUSIVE", database};
ReadWriteStatement commitBegin{"COMMIT", database};
ReadWriteStatement rollbackBegin{"ROLLBACK", database};
Sessions sessions{database, "main", "databaseSessions"};
};
Database::Database()
@@ -60,17 +62,20 @@ Database::Database()
Database::Database(Utils::PathString &&databaseFilePath, JournalMode journalMode)
: Database(std::move(databaseFilePath), 1000ms, journalMode)
{
}
{}
Database::Database(Utils::PathString &&databaseFilePath,
std::chrono::milliseconds busyTimeout,
JournalMode journalMode)
: m_databaseBackend(*this),
m_busyTimeout(busyTimeout)
: m_databaseBackend(*this)
, m_busyTimeout(busyTimeout)
{
setJournalMode(journalMode);
open(std::move(databaseFilePath));
#ifndef QT_NO_DEBUG
execute("PRAGMA reverse_unordered_selects=1");
#endif
}
Database::~Database() = default;
@@ -136,6 +141,16 @@ void Database::setDatabaseFilePath(Utils::PathString &&databaseFilePath)
m_databaseFilePath = std::move(databaseFilePath);
}
void Database::setAttachedTables(const Utils::SmallStringVector &tables)
{
m_statements->sessions.setAttachedTables(tables);
}
void Database::applyAndUpdateSessions()
{
m_statements->sessions.applyAndUpdateSessions();
}
const Utils::PathString &Database::databaseFilePath() const
{
return m_databaseFilePath;
@@ -215,6 +230,22 @@ void Database::rollback()
m_statements->rollbackBegin.execute();
}
void Database::immediateSessionBegin()
{
m_statements->immediateBegin.execute();
m_statements->sessions.create();
}
void Database::sessionCommit()
{
m_statements->sessions.commit();
m_statements->commitBegin.execute();
}
void Database::sessionRollback()
{
m_statements->sessions.rollback();
m_statements->rollbackBegin.execute();
}
void Database::lock()
{
m_databaseMutex.lock();

View File

@@ -123,6 +123,9 @@ public:
void resetUpdateHook() { m_databaseBackend.resetUpdateHook(); }
void setAttachedTables(const Utils::SmallStringVector &tables);
void applyAndUpdateSessions();
private:
void deferredBegin() override;
void immediateBegin() override;
@@ -131,6 +134,9 @@ private:
void rollback() override;
void lock() override;
void unlock() override;
void immediateSessionBegin() override;
void sessionCommit() override;
void sessionRollback() override;
void initializeTables();
void registerTransactionStatements();

View File

@@ -25,7 +25,7 @@
#pragma once
#include <utils/smallstringview.h>
#include <utils/smallstringvector.h>
#include "sqliteglobal.h"
@@ -42,6 +42,8 @@ public:
virtual void execute(Utils::SmallStringView sqlStatement) = 0;
virtual void setUpdateHook(UpdateCallback &callback) = 0;
virtual void resetUpdateHook() = 0;
virtual void applyAndUpdateSessions() = 0;
virtual void setAttachedTables(const Utils::SmallStringVector &tables) = 0;
protected:
~DatabaseInterface() = default;

View File

@@ -43,7 +43,7 @@ public:
, m_sqliteErrorMessage(std::move(sqliteErrorMessage))
{}
const char *what() const noexcept override { return m_sqliteErrorMessage.data(); }
const char *what() const noexcept override { return m_whatErrorHasHappen; }
void printWarning() const;
@@ -283,4 +283,20 @@ public:
{}
};
class CannotApplyChangeSet : public Exception
{
public:
CannotApplyChangeSet(const char *whatErrorHasHappen)
: Exception(whatErrorHasHappen)
{}
};
class ChangeSetIsMisused : public Exception
{
public:
ChangeSetIsMisused(const char *whatErrorHasHappen)
: Exception(whatErrorHasHappen)
{}
};
} // namespace Sqlite

View File

@@ -39,14 +39,7 @@
namespace Sqlite {
enum class ColumnType : char
{
Numeric,
Integer,
Real,
Text,
None
};
enum class ColumnType : char { Numeric, Integer, Real, Text, Blob, None };
enum class ConstraintType : char { NoConstraint, PrimaryKey, Unique, ForeignKey };

View File

@@ -0,0 +1,72 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "sqlitesessionchangeset.h"
#include "sqlitesessions.h"
#include <utils/smallstringio.h>
#include <sqlite3ext.h>
namespace Sqlite {
namespace {
void checkResultCode(int resultCode)
{
switch (resultCode) {
case SQLITE_NOMEM:
throw std::bad_alloc();
}
if (resultCode != SQLITE_OK)
throw UnknowError("Unknow exception");
}
} // namespace
SessionChangeSet::SessionChangeSet(Utils::span<const byte> blob)
: data(sqlite3_malloc64(blob.size()))
, size(int(blob.size()))
{
std::memcpy(data, blob.data(), blob.size());
}
SessionChangeSet::SessionChangeSet(Sessions &session)
{
int resultCode = sqlite3session_changeset(session.session.get(), &size, &data);
checkResultCode(resultCode);
}
SessionChangeSet::~SessionChangeSet()
{
sqlite3_free(data);
}
Utils::span<const byte> SessionChangeSet::asSpan() const
{
return {static_cast<const byte *>(data), static_cast<std::size_t>(size)};
}
} // namespace Sqlite

View File

@@ -0,0 +1,80 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "sqliteglobal.h"
#include <utils/span.h>
#include <memory>
#include <vector>
#include <iosfwd>
namespace Sqlite {
class Sessions;
class SessionChangeSet
{
public:
SessionChangeSet(Utils::span<const byte> blob);
SessionChangeSet(Sessions &session);
~SessionChangeSet();
SessionChangeSet(const SessionChangeSet &) = delete;
void operator=(const SessionChangeSet &) = delete;
SessionChangeSet(SessionChangeSet &&other) noexcept
{
SessionChangeSet temp;
swap(temp, other);
swap(temp, *this);
}
void operator=(SessionChangeSet &);
Utils::span<const byte> asSpan() const;
friend void swap(SessionChangeSet &first, SessionChangeSet &second) noexcept
{
SessionChangeSet temp;
std::swap(temp.data, first.data);
std::swap(temp.size, first.size);
std::swap(first.data, second.data);
std::swap(first.size, second.size);
std::swap(temp.data, second.data);
std::swap(temp.size, second.size);
}
private:
SessionChangeSet() = default;
public:
void *data = nullptr;
int size = {};
};
using SessionChangeSets = std::vector<SessionChangeSet>;
} // namespace Sqlite

View File

@@ -0,0 +1,188 @@
/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "sqlitesessions.h"
#include "sqlitereadstatement.h"
#include "sqlitesessionchangeset.h"
#include "sqlitetable.h"
#include <sqlite3ext.h>
#include <memory>
namespace Sqlite {
namespace {
void checkResultCode(int resultCode)
{
switch (resultCode) {
case SQLITE_NOMEM:
throw std::bad_alloc();
case SQLITE_SCHEMA:
throw CannotApplyChangeSet("Cannot apply change set!");
case SQLITE_MISUSE:
throw ChangeSetIsMisused("Change set is misused!");
}
if (resultCode != SQLITE_OK)
throw UnknowError("Unknow exception");
}
int xConflict(void *, int conflict, sqlite3_changeset_iter *)
{
switch (conflict) {
case SQLITE_CHANGESET_DATA:
return SQLITE_CHANGESET_REPLACE;
case SQLITE_CHANGESET_NOTFOUND:
return SQLITE_CHANGESET_OMIT;
case SQLITE_CHANGESET_CONFLICT:
return SQLITE_CHANGESET_REPLACE;
case SQLITE_CHANGESET_CONSTRAINT:
return SQLITE_CHANGESET_OMIT;
case SQLITE_CHANGESET_FOREIGN_KEY:
return SQLITE_CHANGESET_OMIT;
}
return SQLITE_CHANGESET_ABORT;
}
} // namespace
void Sessions::attachTables(const Utils::SmallStringVector &tableNames)
{
for (Utils::SmallStringView tableName : tableNames) {
int resultCode = sqlite3session_attach(session.get(), tableName.data());
checkResultCode(resultCode);
}
}
Sessions::~Sessions() = default;
void Sessions::setAttachedTables(Utils::SmallStringVector tables)
{
tableNames = std::move(tables);
}
void Sessions::create()
{
sqlite3_session *newSession = nullptr;
int resultCode = sqlite3session_create(database.backend().sqliteDatabaseHandle(),
databaseName.data(),
&newSession);
session.reset(newSession);
checkResultCode(resultCode);
attachTables(tableNames);
}
void Sessions::commit()
{
if (session && !sqlite3session_isempty(session.get())) {
SessionChangeSet changeSet{*this};
insertSession.write(changeSet.asSpan());
}
session.reset();
}
void Sessions::rollback()
{
session.reset();
}
void Internal::SessionsBase::createSessionTable(Database &database)
{
Sqlite::Table table;
table.setUseIfNotExists(true);
table.setName(sessionsTableName);
table.addColumn("id", Sqlite::ColumnType::Integer, {Sqlite::PrimaryKey{AutoIncrement::Yes}});
table.addColumn("changeset", Sqlite::ColumnType::Blob);
table.initialize(database);
}
void Sessions::revert()
{
ReadStatement selectChangeSets{Utils::PathString{"SELECT changeset FROM ",
sessionsTableName,
" ORDER BY id DESC"},
database};
auto changeSets = selectChangeSets.values<SessionChangeSet>(1024);
for (auto &changeSet : changeSets) {
int resultCode = sqlite3changeset_apply_v2(database.backend().sqliteDatabaseHandle(),
changeSet.size,
changeSet.data,
nullptr,
xConflict,
nullptr,
nullptr,
nullptr,
SQLITE_CHANGESETAPPLY_INVERT
| SQLITE_CHANGESETAPPLY_NOSAVEPOINT);
checkResultCode(resultCode);
}
}
void Sessions::apply()
{
ReadStatement selectChangeSets{Utils::PathString{"SELECT changeset FROM ",
sessionsTableName,
" ORDER BY id"},
database};
auto changeSets = selectChangeSets.values<SessionChangeSet>(1024);
for (auto &changeSet : changeSets) {
int resultCode = sqlite3changeset_apply_v2(database.backend().sqliteDatabaseHandle(),
changeSet.size,
changeSet.data,
nullptr,
xConflict,
nullptr,
nullptr,
nullptr,
SQLITE_CHANGESETAPPLY_NOSAVEPOINT);
checkResultCode(resultCode);
}
}
void Sessions::applyAndUpdateSessions()
{
create();
apply();
deleteAll();
commit();
}
void Sessions::deleteAll()
{
WriteStatement{Utils::SmallString{"DELETE FROM ", sessionsTableName}, database}.execute();
}
} // namespace Sqlite

View File

@@ -0,0 +1,95 @@
/****************************************************************************
**
** Copyright (C) 2020 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "sqlitedatabase.h"
#include "sqlitewritestatement.h"
extern "C" {
typedef struct sqlite3_session sqlite3_session;
void sqlite3session_delete(sqlite3_session *pSession);
};
namespace Sqlite {
namespace Internal {
class SQLITE_EXPORT SessionsBase
{
public:
SessionsBase(Database &database, Utils::SmallStringView sessionsTableName)
: sessionsTableName(sessionsTableName)
{
createSessionTable(database);
}
void createSessionTable(Database &database);
public:
Utils::SmallString sessionsTableName;
};
} // namespace Internal
class SQLITE_EXPORT Sessions : public Internal::SessionsBase
{
public:
Sessions(Database &database,
Utils::SmallStringView databaseName,
Utils::SmallStringView sessionsTableName)
: SessionsBase(database, sessionsTableName)
, database(database)
, insertSession{Utils::PathString{"INSERT INTO ",
sessionsTableName,
"(changeset) VALUES(?)"},
database}
, databaseName(databaseName)
, session{nullptr, sqlite3session_delete}
{}
~Sessions();
void setAttachedTables(Utils::SmallStringVector tables);
void create();
void commit();
void rollback();
void revert();
void apply();
void applyAndUpdateSessions();
void deleteAll();
private:
void attachTables(const Utils::SmallStringVector &tables);
public:
Database &database;
WriteStatement insertSession;
Utils::SmallString databaseName;
Utils::SmallStringVector tableNames;
std::unique_ptr<sqlite3_session, decltype(&sqlite3session_delete)> session;
};
} // namespace Sqlite

View File

@@ -27,6 +27,8 @@
#include "sqliteglobal.h"
#include <utils/smallstringview.h>
#include <exception>
#include <mutex>
@@ -49,6 +51,9 @@ public:
virtual void rollback() = 0;
virtual void lock() = 0;
virtual void unlock() = 0;
virtual void immediateSessionBegin() = 0;
virtual void sessionCommit() = 0;
virtual void sessionRollback() = 0;
protected:
~TransactionInterface() = default;
@@ -82,6 +87,42 @@ protected:
bool m_rollback = false;
};
class AbstractThrowingSessionTransaction
{
public:
AbstractThrowingSessionTransaction(const AbstractTransaction &) = delete;
AbstractThrowingSessionTransaction &operator=(const AbstractTransaction &) = delete;
void commit()
{
m_interface.sessionCommit();
m_isAlreadyCommited = true;
m_locker.unlock();
}
~AbstractThrowingSessionTransaction() noexcept(false)
{
try {
if (m_rollback)
m_interface.sessionRollback();
} catch (...) {
if (!std::uncaught_exception())
throw;
}
}
protected:
AbstractThrowingSessionTransaction(TransactionInterface &interface)
: m_interface(interface)
{}
protected:
TransactionInterface &m_interface;
std::unique_lock<TransactionInterface> m_locker{m_interface};
bool m_isAlreadyCommited = false;
bool m_rollback = false;
};
class AbstractThrowingTransaction : public AbstractTransaction
{
public:
@@ -181,6 +222,23 @@ public:
};
using ExclusiveTransaction = BasicExclusiveTransaction<AbstractThrowingTransaction>;
using ExclusiveNonThrowingDestructorTransaction = BasicExclusiveTransaction<AbstractNonThrowingDestructorTransaction>;
using ExclusiveNonThrowingDestructorTransaction
= BasicExclusiveTransaction<AbstractNonThrowingDestructorTransaction>;
class ImmediateSessionTransaction final : public AbstractThrowingSessionTransaction
{
public:
ImmediateSessionTransaction(TransactionInterface &interface)
: AbstractThrowingSessionTransaction(interface)
{
interface.immediateSessionBegin();
}
~ImmediateSessionTransaction()
{
AbstractThrowingSessionTransaction::m_rollback
= !AbstractThrowingSessionTransaction::m_isAlreadyCommited;
}
};
} // namespace Sqlite

View File

@@ -30,6 +30,8 @@
#include <QVariant>
#include <cstddef>
#pragma once
namespace Sqlite {