forked from boostorg/beast
Import KeyvaDB to Beast
This commit is contained in:
Vinnie Falco committed
1 parent
ae0cd50840
commit
0f5b35f408
7 files changed
+1055
No files matched your search
@@ -78,6 +78,12 @@
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
|
||||
</None>
|
||||
<None Include="..\..\modules\beast_db\beast_db.mm">
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
|
||||
</None>
|
||||
<None Include="..\..\README.md" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
@@ -249,6 +255,8 @@
|
||||
<ClInclude Include="..\..\modules\beast_core\zip\zlib\zutil.h" />
|
||||
<ClInclude Include="..\..\modules\beast_crypto\beast_crypto.h" />
|
||||
<ClInclude Include="..\..\modules\beast_crypto\math\beast_UnsignedInteger.h" />
|
||||
<ClInclude Include="..\..\modules\beast_db\beast_db.h" />
|
||||
<ClInclude Include="..\..\modules\beast_db\keyvalue\beast_KeyvaDB.h" />
|
||||
<ClInclude Include="BeastConfig.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
@@ -933,6 +941,13 @@
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\modules\beast_db\beast_db.cpp" />
|
||||
<ClCompile Include="..\..\modules\beast_db\keyvalue\beast_KeyvaDB.cpp">
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
|
||||
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Text Include="..\..\TODO.txt" />
|
||||
|
||||
@@ -36,6 +36,9 @@
|
||||
<None Include="..\..\modules\beast_basics\beast_basics.mm">
|
||||
<Filter>beast_basics</Filter>
|
||||
</None>
|
||||
<None Include="..\..\modules\beast_db\beast_db.mm">
|
||||
<Filter>beast_db</Filter>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Filter Include="beast_core">
|
||||
@@ -125,6 +128,12 @@
|
||||
<Filter Include="beast_crypto\math">
|
||||
<UniqueIdentifier>{1170f2bc-2456-410a-ab2b-c45f6ed37b9e}</UniqueIdentifier>
|
||||
</Filter>
|
||||
<Filter Include="beast_db">
|
||||
<UniqueIdentifier>{4834218f-f13f-41bc-a8a0-50314a3a99a3}</UniqueIdentifier>
|
||||
</Filter>
|
||||
<Filter Include="beast_db\keyvalue">
|
||||
<UniqueIdentifier>{15a98fee-1b52-45eb-9480-514b8750d755}</UniqueIdentifier>
|
||||
</Filter>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\modules\beast_core\beast_core.h">
|
||||
@@ -632,6 +641,12 @@
|
||||
<ClInclude Include="..\..\modules\beast_core\memory\beast_RecycledObjectPool.h">
|
||||
<Filter>beast_core\memory</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\modules\beast_db\beast_db.h">
|
||||
<Filter>beast_db</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\modules\beast_db\keyvalue\beast_KeyvaDB.h">
|
||||
<Filter>beast_db\keyvalue</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\modules\beast_core\beast_core.cpp">
|
||||
@@ -982,6 +997,12 @@
|
||||
<ClCompile Include="..\..\modules\beast_core\diagnostic\beast_UnitTestUtilities.cpp">
|
||||
<Filter>beast_core\diagnostic</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\modules\beast_db\beast_db.cpp">
|
||||
<Filter>beast_db</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\modules\beast_db\keyvalue\beast_KeyvaDB.cpp">
|
||||
<Filter>beast_db\keyvalue</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Text Include="..\..\TODO.txt" />
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of Beast: https://github.com/vinniefalco/Beast
|
||||
Copyright 2013, Vinnie Falco <vinnie.falco@gmail.com>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include "BeastConfig.h"
|
||||
|
||||
#include "beast_db.h"
|
||||
|
||||
#include "../beast_crypto/beast_crypto.h"
|
||||
|
||||
namespace beast
|
||||
{
|
||||
|
||||
#include "keyvalue/beast_KeyvaDB.cpp"
|
||||
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of Beast: https://github.com/vinniefalco/Beast
|
||||
Copyright 2013, Vinnie Falco <vinnie.falco@gmail.com>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#ifndef BEAST_BEAST_DB_H_INCLUDED
|
||||
#define BEAST_BEAST_DB_H_INCLUDED
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
/* If you fail to make sure that all your compile units are building Beast with
|
||||
the same set of option flags, then there's a risk that different compile
|
||||
units will treat the classes as having different memory layouts, leading to
|
||||
very nasty memory corruption errors when they all get linked together.
|
||||
That's why it's best to always include the BeastConfig.h file before any
|
||||
beast headers.
|
||||
*/
|
||||
#ifndef BEAST_BEASTCONFIG_H_INCLUDED
|
||||
# ifdef _MSC_VER
|
||||
# pragma message ("Have you included your BeastConfig.h file before including the Beast headers?")
|
||||
# else
|
||||
# warning "Have you included your BeastConfig.h file before including the Beast headers?"
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#include "../beast_core/beast_core.h"
|
||||
#include "../beast_basics/beast_basics.h"
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
namespace beast
|
||||
{
|
||||
|
||||
#include "keyvalue/beast_KeyvaDB.h"
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,20 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of Beast: https://github.com/vinniefalco/Beast
|
||||
Copyright 2013, Vinnie Falco <vinnie.falco@gmail.com>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include "beast_db.cpp"
|
||||
@@ -0,0 +1,861 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of Beast: https://github.com/vinniefalco/Beast
|
||||
Copyright 2013, Vinnie Falco <vinnie.falco@gmail.com>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
/*
|
||||
|
||||
TODO
|
||||
|
||||
- Check consistency / range checking on read
|
||||
|
||||
- Cache top level tree nodes
|
||||
|
||||
- Coalesce I/O in RandomAccessFile
|
||||
|
||||
- Delete / file compaction
|
||||
|
||||
*/
|
||||
|
||||
class KeyvaDBImp : public KeyvaDB
|
||||
{
|
||||
private:
|
||||
// These are stored in big endian format in the file.
|
||||
|
||||
// A file offset.
|
||||
typedef int64 FileOffset;
|
||||
|
||||
// Index of a key.
|
||||
//
|
||||
// The value is broken up into two parts. The key block index,
|
||||
// and a 1 based index within the keyblock corresponding to the
|
||||
// internal key number.
|
||||
//
|
||||
typedef int32 KeyIndex;
|
||||
typedef int32 KeyBlockIndex;
|
||||
|
||||
// Size of a value.
|
||||
typedef uint32 ByteSize;
|
||||
|
||||
private:
|
||||
// returns the number of keys in a key block with the specified depth
|
||||
static int calcKeysAtDepth (int depth)
|
||||
{
|
||||
return (1U << depth) - 1;
|
||||
}
|
||||
|
||||
// returns the number of bytes in a key record
|
||||
static int calcKeyRecordBytes (int keyBytes)
|
||||
{
|
||||
// This depends on the format of a serialized key record
|
||||
return
|
||||
sizeof (FileOffset) +
|
||||
sizeof (ByteSize) +
|
||||
sizeof (KeyIndex) +
|
||||
sizeof (KeyIndex) +
|
||||
keyBytes
|
||||
;
|
||||
}
|
||||
|
||||
// returns the number of bytes in a key block
|
||||
static int calcKeyBlockBytes (int depth, int keyBytes)
|
||||
{
|
||||
return calcKeysAtDepth (depth) * calcKeyRecordBytes (keyBytes);
|
||||
}
|
||||
|
||||
public:
|
||||
enum
|
||||
{
|
||||
currentVersion = 1
|
||||
};
|
||||
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
struct KeyAddress
|
||||
{
|
||||
// 1 based key block number
|
||||
uint32 blockNumber;
|
||||
|
||||
// 1 based key index within the block, breadth-first left to right
|
||||
uint32 keyNumber;
|
||||
};
|
||||
|
||||
enum
|
||||
{
|
||||
// The size of the fixed area at the beginning of the key file.
|
||||
// This is used to store some housekeeping information like the
|
||||
// key size and version number.
|
||||
//
|
||||
masterHeaderBytes = 1000
|
||||
};
|
||||
|
||||
// The master record is at the beginning of the key file
|
||||
struct MasterRecord
|
||||
{
|
||||
// version number, starting from 1
|
||||
int32 version;
|
||||
|
||||
KeyBlockIndex nextKeyBlockIndex;
|
||||
|
||||
void write (OutputStream& stream)
|
||||
{
|
||||
stream.writeTypeBigEndian (version);
|
||||
}
|
||||
|
||||
void read (InputStream& stream)
|
||||
{
|
||||
stream.readTypeBigEndianInto (&version);
|
||||
}
|
||||
};
|
||||
|
||||
// Key records are indexed starting at one.
|
||||
struct KeyRecord : Uncopyable
|
||||
{
|
||||
explicit KeyRecord (void* const keyStorage)
|
||||
: key (keyStorage)
|
||||
{
|
||||
}
|
||||
|
||||
// Absolute byte FileOffset in the value file.
|
||||
FileOffset valFileOffset;
|
||||
|
||||
// Size of the corresponding value, in bytes.
|
||||
ByteSize valSize;
|
||||
|
||||
// Key record index of left node, or 0.
|
||||
KeyIndex leftIndex;
|
||||
|
||||
// Key record index of right node, or 0.
|
||||
KeyIndex rightIndex;
|
||||
|
||||
// Points to keyBytes storage of the key.
|
||||
void* const key;
|
||||
};
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
// A complete keyblock. The contents of the memory for the key block
|
||||
// are identical to the format on disk. Therefore it is necessary to
|
||||
// use the serialization routines to extract or update the key records.
|
||||
//
|
||||
class KeyBlock : Uncopyable
|
||||
{
|
||||
public:
|
||||
KeyBlock (int depth, int keyBytes)
|
||||
: m_depth (depth)
|
||||
, m_keyBytes (keyBytes)
|
||||
, m_storage (calcKeyBlockBytes (depth, keyBytes))
|
||||
{
|
||||
}
|
||||
|
||||
void read (InputStream& stream)
|
||||
{
|
||||
stream.read (m_storage.getData (), calcKeyBlockBytes (m_depth, m_keyBytes));
|
||||
}
|
||||
|
||||
void write (OutputStream& stream)
|
||||
{
|
||||
stream.write (m_storage.getData (), calcKeyBlockBytes (m_depth, m_keyBytes));
|
||||
}
|
||||
|
||||
void readKeyRecord (KeyRecord* keyRecord, int keyIndex)
|
||||
{
|
||||
bassert (keyIndex >=1 && keyIndex <= calcKeysAtDepth (m_depth));
|
||||
|
||||
size_t const byteOffset = (keyIndex - 1) * calcKeyRecordBytes (m_keyBytes);
|
||||
|
||||
MemoryInputStream stream (
|
||||
addBytesToPointer (m_storage.getData (), byteOffset),
|
||||
calcKeyRecordBytes (m_keyBytes),
|
||||
false);
|
||||
|
||||
stream.readTypeBigEndianInto (&keyRecord->valFileOffset);
|
||||
stream.readTypeBigEndianInto (&keyRecord->valSize);
|
||||
stream.readTypeBigEndianInto (&keyRecord->leftIndex);
|
||||
stream.readTypeBigEndianInto (&keyRecord->rightIndex);
|
||||
stream.read (keyRecord->key, m_keyBytes);
|
||||
}
|
||||
|
||||
#if 0
|
||||
void writeKeyRecord (KeyRecord const& keyRecord, int keyIndex)
|
||||
{
|
||||
bassert (keyIndex >=1 && keyIndex <= calcKeysAtDepth (m_depth));
|
||||
|
||||
#if 0
|
||||
size_t const byteOffset = (keyIndex - 1) * calcKeyRecordBytes (m_keyBytes);
|
||||
|
||||
MemoryOutputStream stream (
|
||||
addBytesToPointer (m_storage.getData (), byteOffset),
|
||||
calcKeyRecordBytes (m_keyBytes));
|
||||
|
||||
stream.writeTypeBigEndian (keyRecord.valFileOffset);
|
||||
stream.writeTypeBigEndian (keyRecord.valSize);
|
||||
stream.writeTypeBigEndian (keyRecord.leftIndex);
|
||||
stream.writeTypeBigEndian (keyRecord.rightIndex);
|
||||
stream.write (keyRecord.key, m_keyBytes);
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
private:
|
||||
int const m_depth;
|
||||
int const m_keyBytes;
|
||||
MemoryBlock m_storage;
|
||||
};
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
// Concurrent data
|
||||
//
|
||||
struct State
|
||||
{
|
||||
RandomAccessFile keyFile;
|
||||
RandomAccessFile valFile;
|
||||
MasterRecord masterRecord;
|
||||
KeyIndex newKeyIndex;
|
||||
FileOffset valFileSize;
|
||||
|
||||
bool hasKeys () const noexcept
|
||||
{
|
||||
return newKeyIndex > 1;
|
||||
}
|
||||
};
|
||||
|
||||
typedef SharedData <State> SharedState;
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
int const m_keyBytes;
|
||||
int const m_keyBlockDepth;
|
||||
SharedState m_state;
|
||||
HeapBlock <char> m_keyStorage;
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
KeyvaDBImp (int keyBytes,
|
||||
int keyBlockDepth,
|
||||
File keyPath,
|
||||
File valPath)
|
||||
: m_keyBytes (keyBytes)
|
||||
, m_keyBlockDepth (keyBlockDepth)
|
||||
, m_keyStorage (keyBytes)
|
||||
{
|
||||
SharedState::WriteAccess state (m_state);
|
||||
|
||||
openFile (&state->keyFile, keyPath);
|
||||
|
||||
int64 const fileSize = state->keyFile.getFile ().getSize ();
|
||||
|
||||
if (fileSize == 0)
|
||||
{
|
||||
// VFALCO TODO Better error handling here
|
||||
// initialize the key file
|
||||
Result result = state->keyFile.setPosition (masterHeaderBytes - 1);
|
||||
if (result.wasOk ())
|
||||
{
|
||||
char byte = 0;
|
||||
|
||||
result = state->keyFile.write (&byte, 1);
|
||||
|
||||
if (result.wasOk ())
|
||||
{
|
||||
state->keyFile.flush ();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
state->newKeyIndex = 1 + (state->keyFile.getFile ().getSize () - masterHeaderBytes)
|
||||
/ calcKeyRecordBytes (m_keyBytes);
|
||||
|
||||
openFile (&state->valFile, valPath);
|
||||
|
||||
state->valFileSize = state->valFile.getFile ().getSize ();
|
||||
}
|
||||
|
||||
~KeyvaDBImp ()
|
||||
{
|
||||
SharedState::WriteAccess state (m_state);
|
||||
|
||||
flushInternal (state);
|
||||
}
|
||||
|
||||
// Open a file for reading and writing.
|
||||
// Creates the file if it doesn't exist.
|
||||
static void openFile (RandomAccessFile* file, File path)
|
||||
{
|
||||
Result const result = file->open (path, RandomAccessFile::readWrite);
|
||||
|
||||
if (! result)
|
||||
{
|
||||
String s;
|
||||
s << "KeyvaDB: Couldn't open " << path.getFileName () << " for writing.";
|
||||
Throw (std::runtime_error (s.toStdString ()));
|
||||
}
|
||||
}
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
Result createMasterRecord (SharedState::WriteAccess& state)
|
||||
{
|
||||
MemoryBlock buffer (masterHeaderBytes, true);
|
||||
|
||||
Result result = state->keyFile.setPosition (0);
|
||||
|
||||
if (result.wasOk ())
|
||||
{
|
||||
MasterRecord mr;
|
||||
|
||||
mr.version = 1;
|
||||
|
||||
result = state->keyFile.write (buffer.getData (), buffer.getSize ());
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
FileOffset calcKeyRecordOffset (KeyIndex keyIndex)
|
||||
{
|
||||
bassert (keyIndex > 0);
|
||||
|
||||
FileOffset const byteOffset = masterHeaderBytes + (keyIndex - 1) * calcKeyRecordBytes (m_keyBytes);
|
||||
|
||||
return byteOffset;
|
||||
}
|
||||
|
||||
// Read a key record into memory.
|
||||
// VFALCO TODO Return a Result and do validity checking on all inputs
|
||||
//
|
||||
void readKeyRecord (KeyRecord* const keyRecord,
|
||||
KeyIndex const keyIndex,
|
||||
SharedState::WriteAccess& state)
|
||||
{
|
||||
FileOffset const byteOffset = calcKeyRecordOffset (keyIndex);
|
||||
|
||||
Result result = state->keyFile.setPosition (byteOffset);
|
||||
|
||||
if (result.wasOk ())
|
||||
{
|
||||
MemoryBlock data (calcKeyRecordBytes (m_keyBytes));
|
||||
|
||||
size_t bytesRead;
|
||||
|
||||
result = state->keyFile.read (data.getData (), calcKeyRecordBytes (m_keyBytes), &bytesRead);
|
||||
|
||||
if (result.wasOk ())
|
||||
{
|
||||
if (bytesRead == calcKeyRecordBytes (m_keyBytes))
|
||||
{
|
||||
MemoryInputStream stream (data, false);
|
||||
|
||||
// This defines the file format!
|
||||
stream.readTypeBigEndianInto (&keyRecord->valFileOffset);
|
||||
stream.readTypeBigEndianInto (&keyRecord->valSize);
|
||||
stream.readTypeBigEndianInto (&keyRecord->leftIndex);
|
||||
stream.readTypeBigEndianInto (&keyRecord->rightIndex);
|
||||
|
||||
// Grab the key
|
||||
stream.read (keyRecord->key, m_keyBytes);
|
||||
}
|
||||
else
|
||||
{
|
||||
result = Result::fail ("KeyvaDB: amountRead != calcKeyRecordBytes()");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (! result.wasOk ())
|
||||
{
|
||||
String s;
|
||||
s << "KeyvaDB readKeyRecord failed in " << state->keyFile.getFile ().getFileName ();
|
||||
Throw (std::runtime_error (s.toStdString ()));
|
||||
}
|
||||
}
|
||||
|
||||
// Write a key record from memory
|
||||
void writeKeyRecord (KeyRecord const& keyRecord,
|
||||
KeyIndex const keyIndex,
|
||||
SharedState::WriteAccess& state,
|
||||
bool includingKey)
|
||||
{
|
||||
FileOffset const byteOffset = calcKeyRecordOffset (keyIndex);
|
||||
|
||||
int const bytes = calcKeyRecordBytes (m_keyBytes) - (includingKey ? 0 : m_keyBytes);
|
||||
|
||||
// VFALCO TODO Recycle this buffer
|
||||
MemoryBlock data (bytes);
|
||||
|
||||
{
|
||||
MemoryOutputStream stream (data, false);
|
||||
|
||||
// This defines the file format!
|
||||
stream.writeTypeBigEndian (keyRecord.valFileOffset);
|
||||
stream.writeTypeBigEndian (keyRecord.valSize);
|
||||
stream.writeTypeBigEndian (keyRecord.leftIndex);
|
||||
stream.writeTypeBigEndian (keyRecord.rightIndex);
|
||||
|
||||
// Write the key
|
||||
if (includingKey)
|
||||
stream.write (keyRecord.key, m_keyBytes);
|
||||
}
|
||||
|
||||
Result result = state->keyFile.setPosition (byteOffset);
|
||||
|
||||
if (result.wasOk ())
|
||||
{
|
||||
size_t bytesWritten;
|
||||
|
||||
result = state->keyFile.write (data.getData (), bytes, &bytesWritten);
|
||||
|
||||
if (result.wasOk ())
|
||||
{
|
||||
if (bytesWritten != bytes)
|
||||
{
|
||||
result = Result::fail ("KeyvaDB: bytesWritten != bytes");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!result.wasOk ())
|
||||
{
|
||||
String s;
|
||||
s << "KeyvaDB: writeKeyRecord failed in " << state->keyFile.getFile ().getFileName ();
|
||||
Throw (std::runtime_error (s.toStdString ()));
|
||||
}
|
||||
}
|
||||
|
||||
// Append a value to the value file.
|
||||
// VFALCO TODO return a Result
|
||||
void writeValue (void const* const value, ByteSize valueBytes, SharedState::WriteAccess& state)
|
||||
{
|
||||
Result result = state->valFile.setPosition (state->valFileSize);
|
||||
|
||||
if (result.wasOk ())
|
||||
{
|
||||
size_t bytesWritten;
|
||||
|
||||
result = state->valFile.write (value, valueBytes, &bytesWritten);
|
||||
|
||||
if (result.wasOk ())
|
||||
{
|
||||
if (bytesWritten == valueBytes)
|
||||
{
|
||||
state->valFileSize += valueBytes;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = Result::fail ("KeyvaDB: bytesWritten != valueBytes");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (! result.wasOk ())
|
||||
{
|
||||
String s;
|
||||
s << "KeyvaDB: writeValue failed in " << state->valFile.getFile ().getFileName ();
|
||||
Throw (std::runtime_error (s.toStdString ()));
|
||||
}
|
||||
}
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
struct FindResult : Uncopyable
|
||||
{
|
||||
FindResult (void* const keyStorage)
|
||||
: keyRecord (keyStorage)
|
||||
{
|
||||
}
|
||||
|
||||
int compare; // result of the last comparison
|
||||
KeyIndex keyIndex; // index we looked at last
|
||||
//KeyBlock keyBlock; // KeyBlock we looked at last
|
||||
KeyRecord keyRecord; // KeyRecord we looked at last
|
||||
};
|
||||
|
||||
// Find a key. If the key doesn't exist, enough information
|
||||
// is left behind in the result to perform an insertion.
|
||||
//
|
||||
// Returns true if the key was found.
|
||||
//
|
||||
bool find (FindResult* findResult, void const* key, SharedState::WriteAccess& state)
|
||||
{
|
||||
// Not okay to call this with an empty key file!
|
||||
bassert (state->hasKeys ());
|
||||
|
||||
// This performs a standard binary search
|
||||
|
||||
findResult->keyIndex = 1;
|
||||
|
||||
do
|
||||
{
|
||||
readKeyRecord (&findResult->keyRecord, findResult->keyIndex, state);
|
||||
|
||||
findResult->compare = memcmp (key, findResult->keyRecord.key, m_keyBytes);
|
||||
|
||||
if (findResult->compare < 0)
|
||||
{
|
||||
if (findResult->keyRecord.leftIndex != 0)
|
||||
{
|
||||
// Go left
|
||||
findResult->keyIndex = findResult->keyRecord.leftIndex;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Insert position is to the left
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (findResult->compare > 0)
|
||||
{
|
||||
if (findResult->keyRecord.rightIndex != 0)
|
||||
{
|
||||
// Go right
|
||||
findResult->keyIndex = findResult->keyRecord.rightIndex;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Insert position is to the right
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
while (findResult->compare != 0);
|
||||
|
||||
return findResult->compare == 0;
|
||||
}
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
bool get (void const* key, GetCallback* callback)
|
||||
{
|
||||
FindResult findResult (m_keyStorage.getData ());
|
||||
|
||||
SharedState::WriteAccess state (m_state);
|
||||
|
||||
bool found = false;
|
||||
|
||||
if (state->hasKeys ())
|
||||
{
|
||||
found = find (&findResult, key, state);
|
||||
|
||||
if (found)
|
||||
{
|
||||
void* const destStorage = callback->getStorageForValue (findResult.keyRecord.valSize);
|
||||
|
||||
Result result = state->valFile.setPosition (findResult.keyRecord.valFileOffset);
|
||||
|
||||
if (result.wasOk ())
|
||||
{
|
||||
size_t bytesRead;
|
||||
|
||||
result = state->valFile.read (destStorage, findResult.keyRecord.valSize, &bytesRead);
|
||||
|
||||
if (result.wasOk ())
|
||||
{
|
||||
if (bytesRead != findResult.keyRecord.valSize)
|
||||
{
|
||||
result = Result::fail ("KeyvaDB: bytesRead != valSize");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (! result.wasOk ())
|
||||
{
|
||||
String s;
|
||||
s << "KeyvaDB: get in " << state->valFile.getFile ().getFileName ();
|
||||
Throw (std::runtime_error (s.toStdString ()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
// Write a key value pair. Does nothing if the key exists.
|
||||
void put (void const* key, void const* value, int valueBytes)
|
||||
{
|
||||
bassert (valueBytes > 0);
|
||||
|
||||
SharedState::WriteAccess state (m_state);
|
||||
|
||||
if (state->hasKeys ())
|
||||
{
|
||||
// Search for the key
|
||||
|
||||
FindResult findResult (m_keyStorage.getData ());
|
||||
|
||||
bool const found = find (&findResult, key, state);
|
||||
|
||||
if (! found )
|
||||
{
|
||||
bassert (findResult.compare != 0);
|
||||
|
||||
// Binary tree insertion.
|
||||
// Link the last key record to the new key
|
||||
{
|
||||
if (findResult.compare < 0)
|
||||
{
|
||||
findResult.keyRecord.leftIndex = state->newKeyIndex;
|
||||
}
|
||||
else
|
||||
{
|
||||
findResult.keyRecord.rightIndex = state->newKeyIndex;
|
||||
}
|
||||
|
||||
writeKeyRecord (findResult.keyRecord, findResult.keyIndex, state, false);
|
||||
}
|
||||
|
||||
// Write the new key
|
||||
{
|
||||
findResult.keyRecord.valFileOffset = state->valFileSize;
|
||||
findResult.keyRecord.valSize = valueBytes;
|
||||
findResult.keyRecord.leftIndex = 0;
|
||||
findResult.keyRecord.rightIndex = 0;
|
||||
|
||||
memcpy (findResult.keyRecord.key, key, m_keyBytes);
|
||||
|
||||
writeKeyRecord (findResult.keyRecord, state->newKeyIndex, state, true);
|
||||
}
|
||||
|
||||
// Key file has grown by one.
|
||||
++state->newKeyIndex;
|
||||
|
||||
// Write the value
|
||||
writeValue (value, valueBytes, state);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Key already exists, do nothing.
|
||||
// We could check to make sure the payloads are the same.
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
//
|
||||
// Write first key
|
||||
//
|
||||
|
||||
KeyRecord keyRecord (m_keyStorage.getData ());
|
||||
|
||||
keyRecord.valFileOffset = state->valFileSize;
|
||||
keyRecord.valSize = valueBytes;
|
||||
keyRecord.leftIndex = 0;
|
||||
keyRecord.rightIndex = 0;
|
||||
|
||||
memcpy (keyRecord.key, key, m_keyBytes);
|
||||
|
||||
writeKeyRecord (keyRecord, state->newKeyIndex, state, true);
|
||||
|
||||
// Key file has grown by one.
|
||||
++state->newKeyIndex;
|
||||
|
||||
//
|
||||
// Write value
|
||||
//
|
||||
|
||||
bassert (state->valFileSize == 0);
|
||||
|
||||
writeValue (value, valueBytes, state);
|
||||
}
|
||||
}
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
|
||||
void flush ()
|
||||
{
|
||||
SharedState::WriteAccess state (m_state);
|
||||
|
||||
flushInternal (state);
|
||||
}
|
||||
|
||||
void flushInternal (SharedState::WriteAccess& state)
|
||||
{
|
||||
state->keyFile.flush ();
|
||||
state->valFile.flush ();
|
||||
}
|
||||
};
|
||||
|
||||
KeyvaDB* KeyvaDB::New (int keyBytes, int keyBlockDepth, File keyPath, File valPath)
|
||||
{
|
||||
return new KeyvaDBImp (keyBytes, keyBlockDepth, keyPath, valPath);
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
class KeyvaDBTests : public UnitTest
|
||||
{
|
||||
public:
|
||||
enum
|
||||
{
|
||||
maxPayloadBytes = 8 * 1024
|
||||
};
|
||||
|
||||
KeyvaDBTests () : UnitTest ("KeyvaDB", "ripple")
|
||||
{
|
||||
}
|
||||
|
||||
// Retrieval callback stores the value in a Payload object for comparison
|
||||
struct PayloadGetCallback : KeyvaDB::GetCallback
|
||||
{
|
||||
UnitTestUtilities::Payload payload;
|
||||
|
||||
PayloadGetCallback () : payload (maxPayloadBytes)
|
||||
{
|
||||
}
|
||||
|
||||
void* getStorageForValue (int valueBytes)
|
||||
{
|
||||
bassert (valueBytes <= maxPayloadBytes);
|
||||
|
||||
payload.bytes = valueBytes;
|
||||
|
||||
return payload.data.getData ();
|
||||
}
|
||||
};
|
||||
|
||||
KeyvaDB* createDB (unsigned int keyBytes, File const& path)
|
||||
{
|
||||
File const keyPath = path.withFileExtension (".key");
|
||||
File const valPath = path.withFileExtension (".val");
|
||||
|
||||
return KeyvaDB::New (keyBytes, 1, keyPath, valPath);
|
||||
}
|
||||
|
||||
void deleteDBFiles (File const& path)
|
||||
{
|
||||
File const keyPath = path.withFileExtension (".key");
|
||||
File const valPath = path.withFileExtension (".val");
|
||||
|
||||
keyPath.deleteFile ();
|
||||
valPath.deleteFile ();
|
||||
}
|
||||
|
||||
template <size_t KeyBytes>
|
||||
void testKeySize (unsigned int const maxItems)
|
||||
{
|
||||
using namespace UnitTestUtilities;
|
||||
|
||||
typedef UnsignedInteger <KeyBytes> KeyType;
|
||||
|
||||
int64 const seedValue = 50;
|
||||
|
||||
String s;
|
||||
|
||||
s << "keyBytes=" << String (uint64(KeyBytes)) << ", maxItems=" << String (maxItems);
|
||||
beginTest (s);
|
||||
|
||||
// Set up the key and value files
|
||||
File const path (File::createTempFile (""));
|
||||
|
||||
{
|
||||
// open the db
|
||||
ScopedPointer <KeyvaDB> db (createDB (KeyBytes, path));
|
||||
|
||||
Payload payload (maxPayloadBytes);
|
||||
Payload check (maxPayloadBytes);
|
||||
|
||||
{
|
||||
// Create an array of ascending integers.
|
||||
HeapBlock <unsigned int> items (maxItems);
|
||||
for (unsigned int i = 0; i < maxItems; ++i)
|
||||
items [i] = i;
|
||||
|
||||
// Now shuffle it deterministically.
|
||||
repeatableShuffle (maxItems, items, seedValue);
|
||||
|
||||
// Write all the keys of integers.
|
||||
for (unsigned int i = 0; i < maxItems; ++i)
|
||||
{
|
||||
unsigned int keyIndex = items [i];
|
||||
|
||||
KeyType const key = KeyType::createFromInteger (keyIndex);
|
||||
|
||||
payload.repeatableRandomFill (1, maxPayloadBytes, keyIndex + seedValue);
|
||||
|
||||
db->put (key.cbegin (), payload.data.getData (), payload.bytes);
|
||||
|
||||
{
|
||||
// VFALCO TODO Check what we just wrote?
|
||||
//db->get (key.cbegin (), check.data.getData (), payload.bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// Go through all of our keys and try to retrieve them.
|
||||
// since this is done in ascending order, we should get
|
||||
// random seeks at this point.
|
||||
//
|
||||
PayloadGetCallback cb;
|
||||
for (unsigned int keyIndex = 0; keyIndex < maxItems; ++keyIndex)
|
||||
{
|
||||
KeyType const v = KeyType::createFromInteger (keyIndex);
|
||||
|
||||
bool const found = db->get (v.cbegin (), &cb);
|
||||
|
||||
expect (found, "Should be found");
|
||||
|
||||
if (found)
|
||||
{
|
||||
payload.repeatableRandomFill (1, maxPayloadBytes, keyIndex + seedValue);
|
||||
|
||||
expect (payload == cb.payload, "Should be equal");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// Re-open the database and confirm the data
|
||||
ScopedPointer <KeyvaDB> db (createDB (KeyBytes, path));
|
||||
|
||||
Payload payload (maxPayloadBytes);
|
||||
Payload check (maxPayloadBytes);
|
||||
|
||||
PayloadGetCallback cb;
|
||||
for (unsigned int keyIndex = 0; keyIndex < maxItems; ++keyIndex)
|
||||
{
|
||||
KeyType const v = KeyType::createFromInteger (keyIndex);
|
||||
|
||||
bool const found = db->get (v.cbegin (), &cb);
|
||||
|
||||
expect (found, "Should be found");
|
||||
|
||||
if (found)
|
||||
{
|
||||
payload.repeatableRandomFill (1, maxPayloadBytes, keyIndex + seedValue);
|
||||
|
||||
expect (payload == cb.payload, "Should be equal");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
deleteDBFiles (path);
|
||||
}
|
||||
|
||||
void runTest ()
|
||||
{
|
||||
testKeySize <4> (500);
|
||||
testKeySize <32> (4000);
|
||||
}
|
||||
};
|
||||
|
||||
static KeyvaDBTests keyvaDBTests;
|
||||
@@ -0,0 +1,55 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of Beast: https://github.com/vinniefalco/Beast
|
||||
Copyright 2013, Vinnie Falco <vinnie.falco@gmail.com>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#ifndef BEAST_KEYVADB_H_INCLUDED
|
||||
#define BEAST_KEYVADB_H_INCLUDED
|
||||
|
||||
/** Specialized Key/value database
|
||||
|
||||
Once written, a value can never be modified.
|
||||
*/
|
||||
class KeyvaDB : LeakChecked <KeyvaDB>
|
||||
{
|
||||
public:
|
||||
class GetCallback
|
||||
{
|
||||
public:
|
||||
virtual void* getStorageForValue (int valueBytes) = 0;
|
||||
};
|
||||
|
||||
static KeyvaDB* New (int keyBytes,
|
||||
int keyBlockDepth,
|
||||
File keyPath,
|
||||
File valPath);
|
||||
|
||||
virtual ~KeyvaDB () { }
|
||||
|
||||
// VFALCO TODO Make the return value a Result so we can
|
||||
// detect corruption and errors!
|
||||
//
|
||||
virtual bool get (void const* key, GetCallback* callback) = 0;
|
||||
|
||||
// VFALCO TODO Use Result for return value
|
||||
//
|
||||
virtual void put (void const* key, void const* value, int valueBytes) = 0;
|
||||
|
||||
virtual void flush () = 0;
|
||||
};
|
||||
|
||||
#endif
|
||||
Reference in new issue
Block a user