diff --git a/Builds/VisualStudio2012/beast.vcxproj b/Builds/VisualStudio2012/beast.vcxproj
index b53c3918..99c88c45 100644
--- a/Builds/VisualStudio2012/beast.vcxproj
+++ b/Builds/VisualStudio2012/beast.vcxproj
@@ -78,6 +78,12 @@
true
true
+
+ true
+ true
+ true
+ true
+
@@ -249,6 +255,8 @@
+
+
@@ -933,6 +941,13 @@
true
true
+
+
+ true
+ true
+ true
+ true
+
diff --git a/Builds/VisualStudio2012/beast.vcxproj.filters b/Builds/VisualStudio2012/beast.vcxproj.filters
index 2692e376..7f02acb9 100644
--- a/Builds/VisualStudio2012/beast.vcxproj.filters
+++ b/Builds/VisualStudio2012/beast.vcxproj.filters
@@ -36,6 +36,9 @@
beast_basics
+
+ beast_db
+
@@ -125,6 +128,12 @@
{1170f2bc-2456-410a-ab2b-c45f6ed37b9e}
+
+ {4834218f-f13f-41bc-a8a0-50314a3a99a3}
+
+
+ {15a98fee-1b52-45eb-9480-514b8750d755}
+
@@ -632,6 +641,12 @@
beast_core\memory
+
+ beast_db
+
+
+ beast_db\keyvalue
+
@@ -982,6 +997,12 @@
beast_core\diagnostic
+
+ beast_db
+
+
+ beast_db\keyvalue
+
diff --git a/modules/beast_db/beast_db.cpp b/modules/beast_db/beast_db.cpp
new file mode 100644
index 00000000..622c2afd
--- /dev/null
+++ b/modules/beast_db/beast_db.cpp
@@ -0,0 +1,31 @@
+//------------------------------------------------------------------------------
+/*
+ This file is part of Beast: https://github.com/vinniefalco/Beast
+ Copyright 2013, Vinnie Falco
+
+ Permission to use, copy, modify, and/or distribute this software for any
+ purpose with or without fee is hereby granted, provided that the above
+ copyright notice and this permission notice appear in all copies.
+
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "BeastConfig.h"
+
+#include "beast_db.h"
+
+#include "../beast_crypto/beast_crypto.h"
+
+namespace beast
+{
+
+#include "keyvalue/beast_KeyvaDB.cpp"
+
+}
diff --git a/modules/beast_db/beast_db.h b/modules/beast_db/beast_db.h
new file mode 100644
index 00000000..1612a178
--- /dev/null
+++ b/modules/beast_db/beast_db.h
@@ -0,0 +1,52 @@
+//------------------------------------------------------------------------------
+/*
+ This file is part of Beast: https://github.com/vinniefalco/Beast
+ Copyright 2013, Vinnie Falco
+
+ Permission to use, copy, modify, and/or distribute this software for any
+ purpose with or without fee is hereby granted, provided that the above
+ copyright notice and this permission notice appear in all copies.
+
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef BEAST_BEAST_DB_H_INCLUDED
+#define BEAST_BEAST_DB_H_INCLUDED
+
+//------------------------------------------------------------------------------
+
+/* If you fail to make sure that all your compile units are building Beast with
+ the same set of option flags, then there's a risk that different compile
+ units will treat the classes as having different memory layouts, leading to
+ very nasty memory corruption errors when they all get linked together.
+ That's why it's best to always include the BeastConfig.h file before any
+ beast headers.
+*/
+#ifndef BEAST_BEASTCONFIG_H_INCLUDED
+# ifdef _MSC_VER
+# pragma message ("Have you included your BeastConfig.h file before including the Beast headers?")
+# else
+# warning "Have you included your BeastConfig.h file before including the Beast headers?"
+# endif
+#endif
+
+#include "../beast_core/beast_core.h"
+#include "../beast_basics/beast_basics.h"
+
+//------------------------------------------------------------------------------
+
+namespace beast
+{
+
+#include "keyvalue/beast_KeyvaDB.h"
+
+}
+
+#endif
diff --git a/modules/beast_db/beast_db.mm b/modules/beast_db/beast_db.mm
new file mode 100644
index 00000000..2ae0b83c
--- /dev/null
+++ b/modules/beast_db/beast_db.mm
@@ -0,0 +1,20 @@
+//------------------------------------------------------------------------------
+/*
+ This file is part of Beast: https://github.com/vinniefalco/Beast
+ Copyright 2013, Vinnie Falco
+
+ Permission to use, copy, modify, and/or distribute this software for any
+ purpose with or without fee is hereby granted, provided that the above
+ copyright notice and this permission notice appear in all copies.
+
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "beast_db.cpp"
diff --git a/modules/beast_db/keyvalue/beast_KeyvaDB.cpp b/modules/beast_db/keyvalue/beast_KeyvaDB.cpp
new file mode 100644
index 00000000..7867292d
--- /dev/null
+++ b/modules/beast_db/keyvalue/beast_KeyvaDB.cpp
@@ -0,0 +1,861 @@
+//------------------------------------------------------------------------------
+/*
+ This file is part of Beast: https://github.com/vinniefalco/Beast
+ Copyright 2013, Vinnie Falco
+
+ Permission to use, copy, modify, and/or distribute this software for any
+ purpose with or without fee is hereby granted, provided that the above
+ copyright notice and this permission notice appear in all copies.
+
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+/*
+
+TODO
+
+- Check consistency / range checking on read
+
+- Cache top level tree nodes
+
+- Coalesce I/O in RandomAccessFile
+
+- Delete / file compaction
+
+*/
+
+class KeyvaDBImp : public KeyvaDB
+{
+private:
+ // These are stored in big endian format in the file.
+
+ // A file offset.
+ typedef int64 FileOffset;
+
+ // Index of a key.
+ //
+ // The value is broken up into two parts. The key block index,
+ // and a 1 based index within the keyblock corresponding to the
+ // internal key number.
+ //
+ typedef int32 KeyIndex;
+ typedef int32 KeyBlockIndex;
+
+ // Size of a value.
+ typedef uint32 ByteSize;
+
+private:
+ // returns the number of keys in a key block with the specified depth
+ static int calcKeysAtDepth (int depth)
+ {
+ return (1U << depth) - 1;
+ }
+
+ // returns the number of bytes in a key record
+ static int calcKeyRecordBytes (int keyBytes)
+ {
+ // This depends on the format of a serialized key record
+ return
+ sizeof (FileOffset) +
+ sizeof (ByteSize) +
+ sizeof (KeyIndex) +
+ sizeof (KeyIndex) +
+ keyBytes
+ ;
+ }
+
+ // returns the number of bytes in a key block
+ static int calcKeyBlockBytes (int depth, int keyBytes)
+ {
+ return calcKeysAtDepth (depth) * calcKeyRecordBytes (keyBytes);
+ }
+
+public:
+ enum
+ {
+ currentVersion = 1
+ };
+
+
+ //--------------------------------------------------------------------------
+
+ struct KeyAddress
+ {
+ // 1 based key block number
+ uint32 blockNumber;
+
+ // 1 based key index within the block, breadth-first left to right
+ uint32 keyNumber;
+ };
+
+ enum
+ {
+ // The size of the fixed area at the beginning of the key file.
+ // This is used to store some housekeeping information like the
+ // key size and version number.
+ //
+ masterHeaderBytes = 1000
+ };
+
+ // The master record is at the beginning of the key file
+ struct MasterRecord
+ {
+ // version number, starting from 1
+ int32 version;
+
+ KeyBlockIndex nextKeyBlockIndex;
+
+ void write (OutputStream& stream)
+ {
+ stream.writeTypeBigEndian (version);
+ }
+
+ void read (InputStream& stream)
+ {
+ stream.readTypeBigEndianInto (&version);
+ }
+ };
+
+ // Key records are indexed starting at one.
+ struct KeyRecord : Uncopyable
+ {
+ explicit KeyRecord (void* const keyStorage)
+ : key (keyStorage)
+ {
+ }
+
+ // Absolute byte FileOffset in the value file.
+ FileOffset valFileOffset;
+
+ // Size of the corresponding value, in bytes.
+ ByteSize valSize;
+
+ // Key record index of left node, or 0.
+ KeyIndex leftIndex;
+
+ // Key record index of right node, or 0.
+ KeyIndex rightIndex;
+
+ // Points to keyBytes storage of the key.
+ void* const key;
+ };
+
+ //--------------------------------------------------------------------------
+
+ // A complete keyblock. The contents of the memory for the key block
+ // are identical to the format on disk. Therefore it is necessary to
+ // use the serialization routines to extract or update the key records.
+ //
+ class KeyBlock : Uncopyable
+ {
+ public:
+ KeyBlock (int depth, int keyBytes)
+ : m_depth (depth)
+ , m_keyBytes (keyBytes)
+ , m_storage (calcKeyBlockBytes (depth, keyBytes))
+ {
+ }
+
+ void read (InputStream& stream)
+ {
+ stream.read (m_storage.getData (), calcKeyBlockBytes (m_depth, m_keyBytes));
+ }
+
+ void write (OutputStream& stream)
+ {
+ stream.write (m_storage.getData (), calcKeyBlockBytes (m_depth, m_keyBytes));
+ }
+
+ void readKeyRecord (KeyRecord* keyRecord, int keyIndex)
+ {
+ bassert (keyIndex >=1 && keyIndex <= calcKeysAtDepth (m_depth));
+
+ size_t const byteOffset = (keyIndex - 1) * calcKeyRecordBytes (m_keyBytes);
+
+ MemoryInputStream stream (
+ addBytesToPointer (m_storage.getData (), byteOffset),
+ calcKeyRecordBytes (m_keyBytes),
+ false);
+
+ stream.readTypeBigEndianInto (&keyRecord->valFileOffset);
+ stream.readTypeBigEndianInto (&keyRecord->valSize);
+ stream.readTypeBigEndianInto (&keyRecord->leftIndex);
+ stream.readTypeBigEndianInto (&keyRecord->rightIndex);
+ stream.read (keyRecord->key, m_keyBytes);
+ }
+
+#if 0
+ void writeKeyRecord (KeyRecord const& keyRecord, int keyIndex)
+ {
+ bassert (keyIndex >=1 && keyIndex <= calcKeysAtDepth (m_depth));
+
+#if 0
+ size_t const byteOffset = (keyIndex - 1) * calcKeyRecordBytes (m_keyBytes);
+
+ MemoryOutputStream stream (
+ addBytesToPointer (m_storage.getData (), byteOffset),
+ calcKeyRecordBytes (m_keyBytes));
+
+ stream.writeTypeBigEndian (keyRecord.valFileOffset);
+ stream.writeTypeBigEndian (keyRecord.valSize);
+ stream.writeTypeBigEndian (keyRecord.leftIndex);
+ stream.writeTypeBigEndian (keyRecord.rightIndex);
+ stream.write (keyRecord.key, m_keyBytes);
+#endif
+ }
+#endif
+
+ private:
+ int const m_depth;
+ int const m_keyBytes;
+ MemoryBlock m_storage;
+ };
+
+ //--------------------------------------------------------------------------
+
+ // Concurrent data
+ //
+ struct State
+ {
+ RandomAccessFile keyFile;
+ RandomAccessFile valFile;
+ MasterRecord masterRecord;
+ KeyIndex newKeyIndex;
+ FileOffset valFileSize;
+
+ bool hasKeys () const noexcept
+ {
+ return newKeyIndex > 1;
+ }
+ };
+
+ typedef SharedData SharedState;
+
+ //--------------------------------------------------------------------------
+
+ int const m_keyBytes;
+ int const m_keyBlockDepth;
+ SharedState m_state;
+ HeapBlock m_keyStorage;
+
+ //--------------------------------------------------------------------------
+
+ KeyvaDBImp (int keyBytes,
+ int keyBlockDepth,
+ File keyPath,
+ File valPath)
+ : m_keyBytes (keyBytes)
+ , m_keyBlockDepth (keyBlockDepth)
+ , m_keyStorage (keyBytes)
+ {
+ SharedState::WriteAccess state (m_state);
+
+ openFile (&state->keyFile, keyPath);
+
+ int64 const fileSize = state->keyFile.getFile ().getSize ();
+
+ if (fileSize == 0)
+ {
+ // VFALCO TODO Better error handling here
+ // initialize the key file
+ Result result = state->keyFile.setPosition (masterHeaderBytes - 1);
+ if (result.wasOk ())
+ {
+ char byte = 0;
+
+ result = state->keyFile.write (&byte, 1);
+
+ if (result.wasOk ())
+ {
+ state->keyFile.flush ();
+ }
+ }
+ }
+
+ state->newKeyIndex = 1 + (state->keyFile.getFile ().getSize () - masterHeaderBytes)
+ / calcKeyRecordBytes (m_keyBytes);
+
+ openFile (&state->valFile, valPath);
+
+ state->valFileSize = state->valFile.getFile ().getSize ();
+ }
+
+ ~KeyvaDBImp ()
+ {
+ SharedState::WriteAccess state (m_state);
+
+ flushInternal (state);
+ }
+
+ // Open a file for reading and writing.
+ // Creates the file if it doesn't exist.
+ static void openFile (RandomAccessFile* file, File path)
+ {
+ Result const result = file->open (path, RandomAccessFile::readWrite);
+
+ if (! result)
+ {
+ String s;
+ s << "KeyvaDB: Couldn't open " << path.getFileName () << " for writing.";
+ Throw (std::runtime_error (s.toStdString ()));
+ }
+ }
+
+ //--------------------------------------------------------------------------
+
+ Result createMasterRecord (SharedState::WriteAccess& state)
+ {
+ MemoryBlock buffer (masterHeaderBytes, true);
+
+ Result result = state->keyFile.setPosition (0);
+
+ if (result.wasOk ())
+ {
+ MasterRecord mr;
+
+ mr.version = 1;
+
+ result = state->keyFile.write (buffer.getData (), buffer.getSize ());
+ }
+
+ return result;
+ }
+
+ //--------------------------------------------------------------------------
+
+ FileOffset calcKeyRecordOffset (KeyIndex keyIndex)
+ {
+ bassert (keyIndex > 0);
+
+ FileOffset const byteOffset = masterHeaderBytes + (keyIndex - 1) * calcKeyRecordBytes (m_keyBytes);
+
+ return byteOffset;
+ }
+
+ // Read a key record into memory.
+ // VFALCO TODO Return a Result and do validity checking on all inputs
+ //
+ void readKeyRecord (KeyRecord* const keyRecord,
+ KeyIndex const keyIndex,
+ SharedState::WriteAccess& state)
+ {
+ FileOffset const byteOffset = calcKeyRecordOffset (keyIndex);
+
+ Result result = state->keyFile.setPosition (byteOffset);
+
+ if (result.wasOk ())
+ {
+ MemoryBlock data (calcKeyRecordBytes (m_keyBytes));
+
+ size_t bytesRead;
+
+ result = state->keyFile.read (data.getData (), calcKeyRecordBytes (m_keyBytes), &bytesRead);
+
+ if (result.wasOk ())
+ {
+ if (bytesRead == calcKeyRecordBytes (m_keyBytes))
+ {
+ MemoryInputStream stream (data, false);
+
+ // This defines the file format!
+ stream.readTypeBigEndianInto (&keyRecord->valFileOffset);
+ stream.readTypeBigEndianInto (&keyRecord->valSize);
+ stream.readTypeBigEndianInto (&keyRecord->leftIndex);
+ stream.readTypeBigEndianInto (&keyRecord->rightIndex);
+
+ // Grab the key
+ stream.read (keyRecord->key, m_keyBytes);
+ }
+ else
+ {
+ result = Result::fail ("KeyvaDB: amountRead != calcKeyRecordBytes()");
+ }
+ }
+ }
+
+ if (! result.wasOk ())
+ {
+ String s;
+ s << "KeyvaDB readKeyRecord failed in " << state->keyFile.getFile ().getFileName ();
+ Throw (std::runtime_error (s.toStdString ()));
+ }
+ }
+
+ // Write a key record from memory
+ void writeKeyRecord (KeyRecord const& keyRecord,
+ KeyIndex const keyIndex,
+ SharedState::WriteAccess& state,
+ bool includingKey)
+ {
+ FileOffset const byteOffset = calcKeyRecordOffset (keyIndex);
+
+ int const bytes = calcKeyRecordBytes (m_keyBytes) - (includingKey ? 0 : m_keyBytes);
+
+ // VFALCO TODO Recycle this buffer
+ MemoryBlock data (bytes);
+
+ {
+ MemoryOutputStream stream (data, false);
+
+ // This defines the file format!
+ stream.writeTypeBigEndian (keyRecord.valFileOffset);
+ stream.writeTypeBigEndian (keyRecord.valSize);
+ stream.writeTypeBigEndian (keyRecord.leftIndex);
+ stream.writeTypeBigEndian (keyRecord.rightIndex);
+
+ // Write the key
+ if (includingKey)
+ stream.write (keyRecord.key, m_keyBytes);
+ }
+
+ Result result = state->keyFile.setPosition (byteOffset);
+
+ if (result.wasOk ())
+ {
+ size_t bytesWritten;
+
+ result = state->keyFile.write (data.getData (), bytes, &bytesWritten);
+
+ if (result.wasOk ())
+ {
+ if (bytesWritten != bytes)
+ {
+ result = Result::fail ("KeyvaDB: bytesWritten != bytes");
+ }
+ }
+ }
+
+ if (!result.wasOk ())
+ {
+ String s;
+ s << "KeyvaDB: writeKeyRecord failed in " << state->keyFile.getFile ().getFileName ();
+ Throw (std::runtime_error (s.toStdString ()));
+ }
+ }
+
+ // Append a value to the value file.
+ // VFALCO TODO return a Result
+ void writeValue (void const* const value, ByteSize valueBytes, SharedState::WriteAccess& state)
+ {
+ Result result = state->valFile.setPosition (state->valFileSize);
+
+ if (result.wasOk ())
+ {
+ size_t bytesWritten;
+
+ result = state->valFile.write (value, valueBytes, &bytesWritten);
+
+ if (result.wasOk ())
+ {
+ if (bytesWritten == valueBytes)
+ {
+ state->valFileSize += valueBytes;
+ }
+ else
+ {
+ result = Result::fail ("KeyvaDB: bytesWritten != valueBytes");
+ }
+ }
+ }
+
+ if (! result.wasOk ())
+ {
+ String s;
+ s << "KeyvaDB: writeValue failed in " << state->valFile.getFile ().getFileName ();
+ Throw (std::runtime_error (s.toStdString ()));
+ }
+ }
+
+ //--------------------------------------------------------------------------
+
+ struct FindResult : Uncopyable
+ {
+ FindResult (void* const keyStorage)
+ : keyRecord (keyStorage)
+ {
+ }
+
+ int compare; // result of the last comparison
+ KeyIndex keyIndex; // index we looked at last
+ //KeyBlock keyBlock; // KeyBlock we looked at last
+ KeyRecord keyRecord; // KeyRecord we looked at last
+ };
+
+ // Find a key. If the key doesn't exist, enough information
+ // is left behind in the result to perform an insertion.
+ //
+ // Returns true if the key was found.
+ //
+ bool find (FindResult* findResult, void const* key, SharedState::WriteAccess& state)
+ {
+ // Not okay to call this with an empty key file!
+ bassert (state->hasKeys ());
+
+ // This performs a standard binary search
+
+ findResult->keyIndex = 1;
+
+ do
+ {
+ readKeyRecord (&findResult->keyRecord, findResult->keyIndex, state);
+
+ findResult->compare = memcmp (key, findResult->keyRecord.key, m_keyBytes);
+
+ if (findResult->compare < 0)
+ {
+ if (findResult->keyRecord.leftIndex != 0)
+ {
+ // Go left
+ findResult->keyIndex = findResult->keyRecord.leftIndex;
+ }
+ else
+ {
+ // Insert position is to the left
+ break;
+ }
+ }
+ else if (findResult->compare > 0)
+ {
+ if (findResult->keyRecord.rightIndex != 0)
+ {
+ // Go right
+ findResult->keyIndex = findResult->keyRecord.rightIndex;
+ }
+ else
+ {
+ // Insert position is to the right
+ break;
+ }
+ }
+ }
+ while (findResult->compare != 0);
+
+ return findResult->compare == 0;
+ }
+
+ //--------------------------------------------------------------------------
+
+ bool get (void const* key, GetCallback* callback)
+ {
+ FindResult findResult (m_keyStorage.getData ());
+
+ SharedState::WriteAccess state (m_state);
+
+ bool found = false;
+
+ if (state->hasKeys ())
+ {
+ found = find (&findResult, key, state);
+
+ if (found)
+ {
+ void* const destStorage = callback->getStorageForValue (findResult.keyRecord.valSize);
+
+ Result result = state->valFile.setPosition (findResult.keyRecord.valFileOffset);
+
+ if (result.wasOk ())
+ {
+ size_t bytesRead;
+
+ result = state->valFile.read (destStorage, findResult.keyRecord.valSize, &bytesRead);
+
+ if (result.wasOk ())
+ {
+ if (bytesRead != findResult.keyRecord.valSize)
+ {
+ result = Result::fail ("KeyvaDB: bytesRead != valSize");
+ }
+ }
+ }
+
+ if (! result.wasOk ())
+ {
+ String s;
+ s << "KeyvaDB: get in " << state->valFile.getFile ().getFileName ();
+ Throw (std::runtime_error (s.toStdString ()));
+ }
+ }
+ }
+
+ return found;
+ }
+
+ //--------------------------------------------------------------------------
+
+ // Write a key value pair. Does nothing if the key exists.
+ void put (void const* key, void const* value, int valueBytes)
+ {
+ bassert (valueBytes > 0);
+
+ SharedState::WriteAccess state (m_state);
+
+ if (state->hasKeys ())
+ {
+ // Search for the key
+
+ FindResult findResult (m_keyStorage.getData ());
+
+ bool const found = find (&findResult, key, state);
+
+ if (! found )
+ {
+ bassert (findResult.compare != 0);
+
+ // Binary tree insertion.
+ // Link the last key record to the new key
+ {
+ if (findResult.compare < 0)
+ {
+ findResult.keyRecord.leftIndex = state->newKeyIndex;
+ }
+ else
+ {
+ findResult.keyRecord.rightIndex = state->newKeyIndex;
+ }
+
+ writeKeyRecord (findResult.keyRecord, findResult.keyIndex, state, false);
+ }
+
+ // Write the new key
+ {
+ findResult.keyRecord.valFileOffset = state->valFileSize;
+ findResult.keyRecord.valSize = valueBytes;
+ findResult.keyRecord.leftIndex = 0;
+ findResult.keyRecord.rightIndex = 0;
+
+ memcpy (findResult.keyRecord.key, key, m_keyBytes);
+
+ writeKeyRecord (findResult.keyRecord, state->newKeyIndex, state, true);
+ }
+
+ // Key file has grown by one.
+ ++state->newKeyIndex;
+
+ // Write the value
+ writeValue (value, valueBytes, state);
+ }
+ else
+ {
+ // Key already exists, do nothing.
+ // We could check to make sure the payloads are the same.
+ }
+ }
+ else
+ {
+ //
+ // Write first key
+ //
+
+ KeyRecord keyRecord (m_keyStorage.getData ());
+
+ keyRecord.valFileOffset = state->valFileSize;
+ keyRecord.valSize = valueBytes;
+ keyRecord.leftIndex = 0;
+ keyRecord.rightIndex = 0;
+
+ memcpy (keyRecord.key, key, m_keyBytes);
+
+ writeKeyRecord (keyRecord, state->newKeyIndex, state, true);
+
+ // Key file has grown by one.
+ ++state->newKeyIndex;
+
+ //
+ // Write value
+ //
+
+ bassert (state->valFileSize == 0);
+
+ writeValue (value, valueBytes, state);
+ }
+ }
+
+ //--------------------------------------------------------------------------
+
+ void flush ()
+ {
+ SharedState::WriteAccess state (m_state);
+
+ flushInternal (state);
+ }
+
+ void flushInternal (SharedState::WriteAccess& state)
+ {
+ state->keyFile.flush ();
+ state->valFile.flush ();
+ }
+};
+
+KeyvaDB* KeyvaDB::New (int keyBytes, int keyBlockDepth, File keyPath, File valPath)
+{
+ return new KeyvaDBImp (keyBytes, keyBlockDepth, keyPath, valPath);
+}
+
+//------------------------------------------------------------------------------
+
+class KeyvaDBTests : public UnitTest
+{
+public:
+ enum
+ {
+ maxPayloadBytes = 8 * 1024
+ };
+
+ KeyvaDBTests () : UnitTest ("KeyvaDB", "ripple")
+ {
+ }
+
+ // Retrieval callback stores the value in a Payload object for comparison
+ struct PayloadGetCallback : KeyvaDB::GetCallback
+ {
+ UnitTestUtilities::Payload payload;
+
+ PayloadGetCallback () : payload (maxPayloadBytes)
+ {
+ }
+
+ void* getStorageForValue (int valueBytes)
+ {
+ bassert (valueBytes <= maxPayloadBytes);
+
+ payload.bytes = valueBytes;
+
+ return payload.data.getData ();
+ }
+ };
+
+ KeyvaDB* createDB (unsigned int keyBytes, File const& path)
+ {
+ File const keyPath = path.withFileExtension (".key");
+ File const valPath = path.withFileExtension (".val");
+
+ return KeyvaDB::New (keyBytes, 1, keyPath, valPath);
+ }
+
+ void deleteDBFiles (File const& path)
+ {
+ File const keyPath = path.withFileExtension (".key");
+ File const valPath = path.withFileExtension (".val");
+
+ keyPath.deleteFile ();
+ valPath.deleteFile ();
+ }
+
+ template
+ void testKeySize (unsigned int const maxItems)
+ {
+ using namespace UnitTestUtilities;
+
+ typedef UnsignedInteger KeyType;
+
+ int64 const seedValue = 50;
+
+ String s;
+
+ s << "keyBytes=" << String (uint64(KeyBytes)) << ", maxItems=" << String (maxItems);
+ beginTest (s);
+
+ // Set up the key and value files
+ File const path (File::createTempFile (""));
+
+ {
+ // open the db
+ ScopedPointer db (createDB (KeyBytes, path));
+
+ Payload payload (maxPayloadBytes);
+ Payload check (maxPayloadBytes);
+
+ {
+ // Create an array of ascending integers.
+ HeapBlock items (maxItems);
+ for (unsigned int i = 0; i < maxItems; ++i)
+ items [i] = i;
+
+ // Now shuffle it deterministically.
+ repeatableShuffle (maxItems, items, seedValue);
+
+ // Write all the keys of integers.
+ for (unsigned int i = 0; i < maxItems; ++i)
+ {
+ unsigned int keyIndex = items [i];
+
+ KeyType const key = KeyType::createFromInteger (keyIndex);
+
+ payload.repeatableRandomFill (1, maxPayloadBytes, keyIndex + seedValue);
+
+ db->put (key.cbegin (), payload.data.getData (), payload.bytes);
+
+ {
+ // VFALCO TODO Check what we just wrote?
+ //db->get (key.cbegin (), check.data.getData (), payload.bytes);
+ }
+ }
+ }
+
+ {
+ // Go through all of our keys and try to retrieve them.
+ // since this is done in ascending order, we should get
+ // random seeks at this point.
+ //
+ PayloadGetCallback cb;
+ for (unsigned int keyIndex = 0; keyIndex < maxItems; ++keyIndex)
+ {
+ KeyType const v = KeyType::createFromInteger (keyIndex);
+
+ bool const found = db->get (v.cbegin (), &cb);
+
+ expect (found, "Should be found");
+
+ if (found)
+ {
+ payload.repeatableRandomFill (1, maxPayloadBytes, keyIndex + seedValue);
+
+ expect (payload == cb.payload, "Should be equal");
+ }
+ }
+ }
+ }
+
+ {
+ // Re-open the database and confirm the data
+ ScopedPointer db (createDB (KeyBytes, path));
+
+ Payload payload (maxPayloadBytes);
+ Payload check (maxPayloadBytes);
+
+ PayloadGetCallback cb;
+ for (unsigned int keyIndex = 0; keyIndex < maxItems; ++keyIndex)
+ {
+ KeyType const v = KeyType::createFromInteger (keyIndex);
+
+ bool const found = db->get (v.cbegin (), &cb);
+
+ expect (found, "Should be found");
+
+ if (found)
+ {
+ payload.repeatableRandomFill (1, maxPayloadBytes, keyIndex + seedValue);
+
+ expect (payload == cb.payload, "Should be equal");
+ }
+ }
+ }
+
+ deleteDBFiles (path);
+ }
+
+ void runTest ()
+ {
+ testKeySize <4> (500);
+ testKeySize <32> (4000);
+ }
+};
+
+static KeyvaDBTests keyvaDBTests;
diff --git a/modules/beast_db/keyvalue/beast_KeyvaDB.h b/modules/beast_db/keyvalue/beast_KeyvaDB.h
new file mode 100644
index 00000000..20e4185f
--- /dev/null
+++ b/modules/beast_db/keyvalue/beast_KeyvaDB.h
@@ -0,0 +1,55 @@
+//------------------------------------------------------------------------------
+/*
+ This file is part of Beast: https://github.com/vinniefalco/Beast
+ Copyright 2013, Vinnie Falco
+
+ Permission to use, copy, modify, and/or distribute this software for any
+ purpose with or without fee is hereby granted, provided that the above
+ copyright notice and this permission notice appear in all copies.
+
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef BEAST_KEYVADB_H_INCLUDED
+#define BEAST_KEYVADB_H_INCLUDED
+
+/** Specialized Key/value database
+
+ Once written, a value can never be modified.
+*/
+class KeyvaDB : LeakChecked
+{
+public:
+ class GetCallback
+ {
+ public:
+ virtual void* getStorageForValue (int valueBytes) = 0;
+ };
+
+ static KeyvaDB* New (int keyBytes,
+ int keyBlockDepth,
+ File keyPath,
+ File valPath);
+
+ virtual ~KeyvaDB () { }
+
+ // VFALCO TODO Make the return value a Result so we can
+ // detect corruption and errors!
+ //
+ virtual bool get (void const* key, GetCallback* callback) = 0;
+
+ // VFALCO TODO Use Result for return value
+ //
+ virtual void put (void const* key, void const* value, int valueBytes) = 0;
+
+ virtual void flush () = 0;
+};
+
+#endif