From 7b330d3496c3c2083ee4fa212a3413f0b139f983 Mon Sep 17 00:00:00 2001 From: Marco Bubke Date: Tue, 18 May 2021 12:14:16 +0200 Subject: [PATCH] Sqlite: Add insertUpdateRemove An generic approach to merge values into a database. It is comparing two ranges. The first range is the Sqlite select range and the second one which are the new values. New values are inserted, change are updated and missing values are removed. Unchanged values are not touched. The compare function is comparing a key. Both ranges are binary ordered. Change-Id: I973c83677ea74f8fa62bd7ab8a73ed560c806562 Reviewed-by: Qt CI Bot Reviewed-by: Thomas Hartmann --- src/libs/sqlite/CMakeLists.txt | 1 + src/libs/sqlite/sqlite-lib.pri | 1 + src/libs/sqlite/sqlitealgorithms.h | 92 ++++++ tests/unit/unittest/CMakeLists.txt | 1 + tests/unit/unittest/sqlitealgorithms-test.cpp | 309 ++++++++++++++++++ tests/unit/unittest/unittest.pro | 1 + 6 files changed, 405 insertions(+) create mode 100644 src/libs/sqlite/sqlitealgorithms.h create mode 100644 tests/unit/unittest/sqlitealgorithms-test.cpp diff --git a/src/libs/sqlite/CMakeLists.txt b/src/libs/sqlite/CMakeLists.txt index 42de2ce9e99..09b294325f8 100644 --- a/src/libs/sqlite/CMakeLists.txt +++ b/src/libs/sqlite/CMakeLists.txt @@ -15,6 +15,7 @@ add_qtc_library(Sqlite constraints.h createtablesqlstatementbuilder.cpp createtablesqlstatementbuilder.h lastchangedrowid.h + sqlitealgorithms.h sqlitebasestatement.cpp sqlitebasestatement.h sqlitecolumn.h sqlitedatabase.cpp sqlitedatabase.h diff --git a/src/libs/sqlite/sqlite-lib.pri b/src/libs/sqlite/sqlite-lib.pri index 59d34a61c2f..f1248cdd7b2 100644 --- a/src/libs/sqlite/sqlite-lib.pri +++ b/src/libs/sqlite/sqlite-lib.pri @@ -25,6 +25,7 @@ SOURCES += \ $$PWD/sqlitebasestatement.cpp HEADERS += \ $$PWD/constraints.h \ + $$PWD/sqlitealgorithms.h \ $$PWD/sqliteblob.h \ $$PWD/sqlitelibraryinitializer.h \ $$PWD/sqlitetimestamp.h \ diff --git a/src/libs/sqlite/sqlitealgorithms.h b/src/libs/sqlite/sqlitealgorithms.h new file mode 100644 index 00000000000..0c83bd2f928 --- /dev/null +++ b/src/libs/sqlite/sqlitealgorithms.h @@ -0,0 +1,92 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#pragma once + +#include + +namespace Sqlite { + +constexpr int compare(Utils::SmallStringView first, Utils::SmallStringView second) noexcept +{ + auto difference = std::char_traits::compare(first.data(), + second.data(), + std::min(first.size(), second.size())); + + if (difference == 0) + return int(first.size() - second.size()); + + return difference; +} + +template +void insertUpdateDelete(SqliteRange &&sqliteRange, + Range &&values, + CompareKey compareKey, + InsertCallback insertCallback, + UpdateCallback updateCallback, + DeleteCallback deleteCallback) +{ + auto currentSqliteIterator = sqliteRange.begin(); + auto endSqliteIterator = sqliteRange.end(); + auto currentValueIterator = values.begin(); + auto endValueIterator = values.end(); + + while (true) { + bool hasMoreValues = currentValueIterator != endValueIterator; + bool hasMoreSqliteValues = currentSqliteIterator != endSqliteIterator; + if (hasMoreValues && hasMoreSqliteValues) { + const auto &sqliteValue = *currentSqliteIterator; + const auto &value = *currentValueIterator; + auto compare = compareKey(sqliteValue, value); + if (compare == 0) { + updateCallback(sqliteValue, value); + ++currentValueIterator; + ++currentSqliteIterator; + } else if (compare > 0) { + insertCallback(value); + ++currentValueIterator; + } else if (compare < 0) { + deleteCallback(sqliteValue); + ++currentSqliteIterator; + } + } else if (hasMoreValues) { + insertCallback(*currentValueIterator); + ++currentValueIterator; + } else if (hasMoreSqliteValues) { + deleteCallback(*currentSqliteIterator); + ++currentSqliteIterator; + } else { + break; + } + } +} + +} // namespace Sqlite diff --git a/tests/unit/unittest/CMakeLists.txt b/tests/unit/unittest/CMakeLists.txt index 49050ae2f37..14b1263563a 100644 --- a/tests/unit/unittest/CMakeLists.txt +++ b/tests/unit/unittest/CMakeLists.txt @@ -151,6 +151,7 @@ add_qtc_test(unittest GTEST sourcerangefilter-test.cpp sourcesmanager-test.cpp spydummy.cpp spydummy.h + sqlitealgorithms-test.cpp sqliteindex-test.cpp sqliteteststatement.h sqlitetransaction-test.cpp diff --git a/tests/unit/unittest/sqlitealgorithms-test.cpp b/tests/unit/unittest/sqlitealgorithms-test.cpp new file mode 100644 index 00000000000..6c376a2ad8b --- /dev/null +++ b/tests/unit/unittest/sqlitealgorithms-test.cpp @@ -0,0 +1,309 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qt Creator. +** +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +****************************************************************************/ + +#include "googletest.h" + +#include +#include +#include + +#include + +namespace { + +class KeyValueView +{ +public: + KeyValueView(Utils::SmallStringView key, long long value, long long rowid) + : key{key} + , value{value} + , rowid{rowid} + {} + +public: + Utils::SmallStringView key; + long long value = 0; + long long rowid = -1; +}; + +std::ostream &operator<<(std::ostream &out, KeyValueView keyValueView) +{ + return out << "(" << keyValueView.key << ", " << keyValueView.value << ")"; +} + +class KeyValue +{ +public: + KeyValue(Utils::SmallStringView key, long long value) + : key{key} + , value{value} + {} + + friend bool operator==(const KeyValue &first, const KeyValue &second) + { + return first.key == second.key && first.value == second.value; + } + + friend bool operator==(KeyValueView first, const KeyValue &second) + { + return first.key == second.key && first.value == second.value; + } + +public: + Utils::SmallString key; + long long value = 0; +}; + +std::ostream &operator<<(std::ostream &out, KeyValue keyValue) +{ + return out << "(" << keyValue.key << ", " << keyValue.value << ")"; +} + +using KeyValues = std::vector; + +MATCHER_P2(IsKeyValue, + key, + value, + std::string(negation ? "isn't " : "is ") + PrintToString(KeyValue{key, value})) +{ + return arg.key == key && arg.value == value; +} + +class SqliteAlgorithms : public testing::Test +{ +public: + ~SqliteAlgorithms() noexcept { transaction.commit(); } + + class Initializer + { + public: + Initializer(Sqlite::Database &database) + { + Sqlite::Table table; + table.setName("data"); + table.addColumn("key", Sqlite::ColumnType::None, {Sqlite::PrimaryKey{}}); + table.addColumn("value"); + + table.initialize(database); + } + }; + + auto select() { return selectViewsStatement.range(); } + + auto fetchKeyValues() { return selectValuesStatement.values(24); } + +protected: + Sqlite::Database database{":memory:", Sqlite::JournalMode::Memory}; + Sqlite::ImmediateTransaction transaction{database}; + Initializer initializer{database}; + Sqlite::ReadStatement<3> selectViewsStatement{"SELECT key, value, rowid FROM data ORDER BY key", + database}; + Sqlite::ReadStatement<2> selectValuesStatement{"SELECT key, value FROM data ORDER BY key", + database}; + Sqlite::WriteStatement insertStatement{"INSERT INTO data(key, value) VALUES (?1, ?2)", database}; + Sqlite::WriteStatement updateStatement{"UPDATE data SET value = ?2 WHERE rowid = ?1", database}; + Sqlite::WriteStatement deleteStatement{"DELETE FROM data WHERE rowid = ?", database}; + std::function insert{ + [&](const KeyValue &keyValue) { insertStatement.write(keyValue.key, keyValue.value); }}; + std::function update{ + [&](KeyValueView keyValueView, const KeyValue &keyValue) { + if (!(keyValueView == keyValue)) + updateStatement.write(keyValueView.rowid, keyValue.value); + }}; + std::function remove{ + [&](KeyValueView keyValueView) { deleteStatement.write(keyValueView.rowid); }}; +}; + +auto compareKey = [](KeyValueView keyValueView, const KeyValue &keyValue) { + return Sqlite::compare(keyValueView.key, keyValue.key); +}; + +TEST_F(SqliteAlgorithms, InsertValues) +{ + KeyValues keyValues = {{"one", 1}, {"oneone", 11}}; + + Sqlite::insertUpdateDelete(select(), keyValues, compareKey, insert, update, remove); + + ASSERT_THAT(fetchKeyValues(), UnorderedElementsAre(KeyValue{"one", 1}, KeyValue{"oneone", 11})); +} + +TEST_F(SqliteAlgorithms, InsertBeforeValues) +{ + KeyValues keyValues = {{"two", 2}, {"twotwo", 22}}; + Sqlite::insertUpdateDelete(select(), keyValues, compareKey, insert, update, remove); + KeyValues moreKeyValues = {{"one", 1}, {"oneone", 11}, {"two", 2}, {"twotwo", 22}}; + + Sqlite::insertUpdateDelete(select(), moreKeyValues, compareKey, insert, update, remove); + + ASSERT_THAT(fetchKeyValues(), + UnorderedElementsAre(KeyValue{"one", 1}, + KeyValue{"oneone", 11}, + KeyValue{"two", 2}, + KeyValue{"twotwo", 22})); +} + +TEST_F(SqliteAlgorithms, InsertInBetweenValues) +{ + KeyValues keyValues = {{"one", 1}, {"two", 2}}; + Sqlite::insertUpdateDelete(select(), keyValues, compareKey, insert, update, remove); + KeyValues moreKeyValues = {{"one", 1}, {"oneone", 11}, {"two", 2}, {"twotwo", 22}}; + + Sqlite::insertUpdateDelete(select(), moreKeyValues, compareKey, insert, update, remove); + + ASSERT_THAT(fetchKeyValues(), + UnorderedElementsAre(KeyValue{"one", 1}, + KeyValue{"oneone", 11}, + KeyValue{"two", 2}, + KeyValue{"twotwo", 22})); +} + +TEST_F(SqliteAlgorithms, InsertTrailingValues) +{ + KeyValues keyValues = {{"one", 1}, {"oneone", 11}}; + Sqlite::insertUpdateDelete(select(), keyValues, compareKey, insert, update, remove); + KeyValues moreKeyValues = {{"one", 1}, {"oneone", 11}, {"two", 2}, {"twotwo", 22}}; + + Sqlite::insertUpdateDelete(select(), moreKeyValues, compareKey, insert, update, remove); + + ASSERT_THAT(fetchKeyValues(), + UnorderedElementsAre(KeyValue{"one", 1}, + KeyValue{"oneone", 11}, + KeyValue{"two", 2}, + KeyValue{"twotwo", 22})); +} + +TEST_F(SqliteAlgorithms, UpdateValues) +{ + KeyValues keyValues = {{"one", 1}, {"oneone", 11}}; + Sqlite::insertUpdateDelete(select(), keyValues, compareKey, insert, update, remove); + KeyValues changedKeyValues = {{"one", 2}, {"oneone", 22}}; + + Sqlite::insertUpdateDelete(select(), changedKeyValues, compareKey, insert, update, remove); + + ASSERT_THAT(fetchKeyValues(), UnorderedElementsAre(KeyValue{"one", 2}, KeyValue{"oneone", 22})); +} + +TEST_F(SqliteAlgorithms, UpdateSomeValues) +{ + KeyValues keyValues = {{"one", 1}, {"oneone", 11}, {"two", 2}, {"twotwo", 22}}; + Sqlite::insertUpdateDelete(select(), keyValues, compareKey, insert, update, remove); + KeyValues moreKeyValues = {{"one", 101}, {"oneone", 11}, {"two", 202}, {"twotwo", 22}}; + + Sqlite::insertUpdateDelete(select(), moreKeyValues, compareKey, insert, update, remove); + + ASSERT_THAT(fetchKeyValues(), + UnorderedElementsAre(KeyValue{"one", 101}, + KeyValue{"oneone", 11}, + KeyValue{"two", 202}, + KeyValue{"twotwo", 22})); +} + +TEST_F(SqliteAlgorithms, DeleteBeforeSqliteEntries) +{ + KeyValues keyValues = {{"one", 1}, {"oneone", 11}, {"two", 2}, {"twotwo", 22}}; + Sqlite::insertUpdateDelete(select(), keyValues, compareKey, insert, update, remove); + KeyValues lessKeyValues = {{"two", 2}, {"twotwo", 22}}; + + Sqlite::insertUpdateDelete(select(), lessKeyValues, compareKey, insert, update, remove); + + ASSERT_THAT(fetchKeyValues(), UnorderedElementsAre(KeyValue{"two", 2}, KeyValue{"twotwo", 22})); +} + +TEST_F(SqliteAlgorithms, DeleteTrailingSqliteEntries2) +{ + KeyValues keyValues = {{"one", 1}, {"oneone", 11}, {"two", 2}, {"twotwo", 22}}; + Sqlite::insertUpdateDelete(select(), keyValues, compareKey, insert, update, remove); + KeyValues lessKeyValues = {{"one", 1}, {"oneone", 11}}; + + Sqlite::insertUpdateDelete(select(), lessKeyValues, compareKey, insert, update, remove); + + ASSERT_THAT(fetchKeyValues(), UnorderedElementsAre(KeyValue{"one", 1}, KeyValue{"oneone", 11})); +} + +TEST_F(SqliteAlgorithms, DeleteTrailingSqliteEntries) +{ + KeyValues keyValues = {{"one", 1}, {"oneone", 11}, {"two", 2}, {"twotwo", 22}}; + Sqlite::insertUpdateDelete(select(), keyValues, compareKey, insert, update, remove); + KeyValues lessKeyValues = {{"one", 1}, {"oneone", 11}}; + + Sqlite::insertUpdateDelete(select(), lessKeyValues, compareKey, insert, update, remove); + + ASSERT_THAT(fetchKeyValues(), UnorderedElementsAre(KeyValue{"one", 1}, KeyValue{"oneone", 11})); +} + +TEST_F(SqliteAlgorithms, DeleteSqliteEntries) +{ + KeyValues keyValues = {{"one", 1}, {"oneone", 11}, {"two", 2}, {"twotwo", 22}}; + Sqlite::insertUpdateDelete(select(), keyValues, compareKey, insert, update, remove); + KeyValues emptyKeyValues = {}; + + Sqlite::insertUpdateDelete(select(), emptyKeyValues, compareKey, insert, update, remove); + + ASSERT_THAT(fetchKeyValues(), IsEmpty()); +} + +TEST_F(SqliteAlgorithms, Synchonize) +{ + KeyValues keyValues = {{"one", 1}, {"oneone", 11}, {"two", 2}, {"twotwo", 22}}; + Sqlite::insertUpdateDelete(select(), keyValues, compareKey, insert, update, remove); + KeyValues emptyKeyValues = {{"oneone", 11}, {"three", 3}, {"twotwo", 202}}; + + Sqlite::insertUpdateDelete(select(), emptyKeyValues, compareKey, insert, update, remove); + + ASSERT_THAT(fetchKeyValues(), + UnorderedElementsAre(KeyValue{"oneone", 11}, + KeyValue{"three", 3}, + KeyValue{"twotwo", 202})); +} + +TEST_F(SqliteAlgorithms, CompareEqual) +{ + auto compare = Sqlite::compare("one", "one"); + + ASSERT_THAT(compare, Eq(0)); +} + +TEST_F(SqliteAlgorithms, CompareGreater) +{ + auto compare = Sqlite::compare("two", "one"); + + ASSERT_THAT(compare, Gt(0)); +} + +TEST_F(SqliteAlgorithms, CompareGreaterForTrailingText) +{ + auto compare = Sqlite::compare("oneone", "one"); + + ASSERT_THAT(compare, Gt(0)); +} + +TEST_F(SqliteAlgorithms, CompareLessForTrailingText) +{ + auto compare = Sqlite::compare("one", "oneone"); + + ASSERT_THAT(compare, Lt(0)); +} + +} // namespace diff --git a/tests/unit/unittest/unittest.pro b/tests/unit/unittest/unittest.pro index e6d33b6cbc0..d9f1ba95178 100644 --- a/tests/unit/unittest/unittest.pro +++ b/tests/unit/unittest/unittest.pro @@ -73,6 +73,7 @@ SOURCES += \ nodelistproperty-test.cpp \ projectstoragesqlitefunctionregistry-test.cpp \ storagecache-test.cpp \ + sqlitealgorithms-test.cpp \ synchronousimagecache-test.cpp \ imagecachegenerator-test.cpp \ imagecachestorage-test.cpp \