Store object members with two slots: one for the key and one for the value

This commit is contained in:
Benoit Blanchon
2024-08-23 15:31:47 +02:00
parent a2b09bfbd2
commit 09c89dcacf
24 changed files with 229 additions and 173 deletions

View File

@ -1,6 +1,11 @@
ArduinoJson: change log
=======================
HEAD
----
* Store object members with two slots: one for the key and one for the value
v7.1.0 (2024-06-27)
------

View File

@ -10,7 +10,7 @@ static_assert(ARDUINOJSON_LITTLE_ENDIAN == 1, "ARDUINOJSON_LITTLE_ENDIAN");
static_assert(ARDUINOJSON_USE_DOUBLE == 1, "ARDUINOJSON_USE_DOUBLE");
static_assert(sizeof(ArduinoJson::detail::VariantSlot) == 8,
static_assert(sizeof(ArduinoJson::detail::VariantSlot) == 6,
"sizeof(VariantSlot)");
void setup() {}

View File

@ -8,7 +8,7 @@ static_assert(ARDUINOJSON_LITTLE_ENDIAN == 1, "ARDUINOJSON_LITTLE_ENDIAN");
static_assert(ARDUINOJSON_USE_DOUBLE == 1, "ARDUINOJSON_USE_DOUBLE");
static_assert(sizeof(ArduinoJson::detail::VariantSlot) == 16,
static_assert(sizeof(ArduinoJson::detail::VariantSlot) == 12,
"sizeof(VariantSlot)");
int main() {}

View File

@ -8,7 +8,7 @@ static_assert(ARDUINOJSON_LITTLE_ENDIAN == 1, "ARDUINOJSON_LITTLE_ENDIAN");
static_assert(ARDUINOJSON_USE_DOUBLE == 1, "ARDUINOJSON_USE_DOUBLE");
static_assert(sizeof(ArduinoJson::detail::VariantSlot) == 24,
static_assert(sizeof(ArduinoJson::detail::VariantSlot) == 16,
"sizeof(VariantSlot)");
int main() {}

View File

@ -8,7 +8,7 @@ static_assert(ARDUINOJSON_LITTLE_ENDIAN == 1, "ARDUINOJSON_LITTLE_ENDIAN");
static_assert(ARDUINOJSON_USE_DOUBLE == 1, "ARDUINOJSON_USE_DOUBLE");
static_assert(sizeof(ArduinoJson::detail::VariantSlot) == 24,
static_assert(sizeof(ArduinoJson::detail::VariantSlot) == 16,
"sizeof(VariantSlot)");
int main() {}

View File

@ -68,6 +68,12 @@ TEST_CASE("JsonArray::remove()") {
REQUIRE(array[1] == 2);
}
SECTION("remove end()") {
array.remove(array.end());
REQUIRE(3 == array.size());
}
SECTION("In a loop") {
for (JsonArray::iterator it = array.begin(); it != array.end(); ++it) {
if (*it == 2)

View File

@ -308,12 +308,12 @@ TEST_CASE("deserialize JSON object") {
REQUIRE(doc["a"] == 2);
}
SECTION("NUL in keys") { // we don't support NULs in keys
SECTION("NUL in keys") {
DeserializationError err =
deserializeJson(doc, "{\"x\\u0000a\":1,\"x\\u0000b\":2}");
REQUIRE(err == DeserializationError::Ok);
REQUIRE(doc.as<std::string>() == "{\"x\":2}");
REQUIRE(doc.as<std::string>() == "{\"x\\u0000a\":1,\"x\\u0000b\":2}");
}
}

View File

@ -345,12 +345,12 @@ TEST_CASE("Deduplicate keys") {
}
TEST_CASE("MemberProxy under memory constraints") {
KillswitchAllocator killswitch;
SpyingAllocator spy(&killswitch);
TimebombAllocator timebomb(1);
SpyingAllocator spy(&timebomb);
JsonDocument doc(&spy);
SECTION("key allocation fails") {
killswitch.on();
SECTION("key slot allocation fails") {
timebomb.setCountdown(0);
doc["hello"_s] = "world";
@ -361,4 +361,36 @@ TEST_CASE("MemberProxy under memory constraints") {
AllocateFail(sizeofPool()),
});
}
SECTION("value slot allocation fails") {
timebomb.setCountdown(1);
// fill the pool entirely, but leave one slot for the key
doc["foo"][ARDUINOJSON_POOL_CAPACITY - 4] = 1;
REQUIRE(doc.overflowed() == false);
doc["hello"_s] = "world";
REQUIRE(doc.is<JsonObject>());
REQUIRE(doc.size() == 1);
REQUIRE(doc.overflowed() == true);
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
AllocateFail(sizeofPool()),
});
}
SECTION("key string allocation fails") {
timebomb.setCountdown(1);
doc["hello"_s] = "world";
REQUIRE(doc.is<JsonObject>());
REQUIRE(doc.size() == 0);
REQUIRE(doc.overflowed() == true);
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
AllocateFail(sizeofString("hello")),
});
}
}

View File

@ -178,7 +178,7 @@ TEST_CASE("JsonDocument::shrinkToFit()") {
AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofString("abcdefg")),
Reallocate(sizeofPool(), sizeofPool(1)),
Reallocate(sizeofPool(), sizeofPool(2)),
});
}
}

View File

@ -10,9 +10,7 @@ ARDUINOJSON_BEGIN_PRIVATE_NAMESPACE
class ArrayData : public CollectionData {
public:
VariantData* addElement(ResourceManager* resources) {
return addSlot(resources).data();
}
VariantData* addElement(ResourceManager* resources);
static VariantData* addElement(ArrayData* array, ResourceManager* resources) {
if (!array)
@ -51,6 +49,16 @@ class ArrayData : public CollectionData {
array->removeElement(index, resources);
}
void remove(iterator it, ResourceManager* resources) {
CollectionData::removeOne(it, resources);
}
static void remove(ArrayData* array, iterator it,
ResourceManager* resources) {
if (array)
return array->remove(it, resources);
}
private:
iterator at(size_t index, const ResourceManager* resources) const;
};

View File

@ -19,6 +19,14 @@ inline ArrayData::iterator ArrayData::at(
return it;
}
inline VariantData* ArrayData::addElement(ResourceManager* resources) {
auto slot = resources->allocSlot();
if (!slot)
return nullptr;
CollectionData::appendOne(slot, resources);
return slot->data();
}
inline VariantData* ArrayData::getOrAddElement(size_t index,
ResourceManager* resources) {
auto it = createIterator(resources);
@ -58,7 +66,7 @@ inline bool ArrayData::addValue(T&& value, ResourceManager* resources) {
resources->freeSlot(slot);
return false;
}
addSlot(slot, resources);
CollectionData::appendOne(slot, resources);
return true;
}

View File

@ -49,12 +49,6 @@ class CollectionIterator {
return *data();
}
const char* key() const;
bool ownsKey() const;
void setKey(StringNode*);
void setKey(const char*);
VariantData* data() {
return reinterpret_cast<VariantData*>(slot_);
}
@ -99,22 +93,17 @@ class CollectionData {
collection->clear(resources);
}
void remove(iterator it, ResourceManager* resources);
static void remove(CollectionData* collection, iterator it,
ResourceManager* resources) {
if (collection)
return collection->remove(it, resources);
}
SlotId head() const {
return head_;
}
void addSlot(SlotWithId slot, ResourceManager* resources);
protected:
iterator addSlot(ResourceManager*);
void appendOne(SlotWithId slot, const ResourceManager* resources);
void appendPair(SlotWithId key, SlotWithId value,
const ResourceManager* resources);
void removeOne(iterator it, ResourceManager* resources);
void removePair(iterator it, ResourceManager* resources);
private:
SlotWithId getPreviousSlot(VariantSlot*, const ResourceManager*) const;

View File

@ -17,28 +17,6 @@ inline CollectionIterator::CollectionIterator(VariantSlot* slot, SlotId slotId)
nextId_ = slot_ ? slot_->next() : NULL_SLOT;
}
inline const char* CollectionIterator::key() const {
ARDUINOJSON_ASSERT(slot_ != nullptr);
return slot_->key();
}
inline void CollectionIterator::setKey(const char* s) {
ARDUINOJSON_ASSERT(slot_ != nullptr);
ARDUINOJSON_ASSERT(s != nullptr);
return slot_->setKey(s);
}
inline void CollectionIterator::setKey(StringNode* s) {
ARDUINOJSON_ASSERT(slot_ != nullptr);
ARDUINOJSON_ASSERT(s != nullptr);
return slot_->setKey(s);
}
inline bool CollectionIterator::ownsKey() const {
ARDUINOJSON_ASSERT(slot_ != nullptr);
return slot_->ownsKey();
}
inline void CollectionIterator::next(const ResourceManager* resources) {
ARDUINOJSON_ASSERT(currentId_ != NULL_SLOT);
slot_ = resources->getSlot(nextId_);
@ -47,11 +25,8 @@ inline void CollectionIterator::next(const ResourceManager* resources) {
nextId_ = slot_->next();
}
inline CollectionData::iterator CollectionData::addSlot(
ResourceManager* resources) {
auto slot = resources->allocSlot();
if (!slot)
return {};
inline void CollectionData::appendOne(SlotWithId slot,
const ResourceManager* resources) {
if (tail_ != NULL_SLOT) {
auto tail = resources->getSlot(tail_);
tail->setNext(slot.id());
@ -60,18 +35,19 @@ inline CollectionData::iterator CollectionData::addSlot(
head_ = slot.id();
tail_ = slot.id();
}
return iterator(slot, slot.id());
}
inline void CollectionData::addSlot(SlotWithId slot,
ResourceManager* resources) {
inline void CollectionData::appendPair(SlotWithId key, SlotWithId value,
const ResourceManager* resources) {
key->setNext(value.id());
if (tail_ != NULL_SLOT) {
auto tail = resources->getSlot(tail_);
tail->setNext(slot.id());
tail_ = slot.id();
tail->setNext(key.id());
tail_ = value.id();
} else {
head_ = slot.id();
tail_ = slot.id();
head_ = key.id();
tail_ = value.id();
}
}
@ -95,14 +71,14 @@ inline SlotWithId CollectionData::getPreviousSlot(
while (currentId != NULL_SLOT) {
auto currentSlot = resources->getSlot(currentId);
if (currentSlot == target)
return prev;
break;
prev = SlotWithId(currentSlot, currentId);
currentId = currentSlot->next();
}
return SlotWithId();
return prev;
}
inline void CollectionData::remove(iterator it, ResourceManager* resources) {
inline void CollectionData::removeOne(iterator it, ResourceManager* resources) {
if (it.done())
return;
auto curr = it.slot_;
@ -117,6 +93,24 @@ inline void CollectionData::remove(iterator it, ResourceManager* resources) {
resources->freeSlot({it.slot_, it.currentId_});
}
inline void CollectionData::removePair(ObjectData::iterator it,
ResourceManager* resources) {
if (it.done())
return;
auto keySlot = it.slot_;
auto valueId = it.nextId_;
auto valueSlot = resources->getSlot(valueId);
// remove value slot
keySlot->setNext(valueSlot->next());
resources->freeSlot({valueSlot, valueId});
// remove key slot
removeOne(it, resources);
}
inline size_t CollectionData::nesting(const ResourceManager* resources) const {
size_t maxChildNesting = 0;
for (auto it = createIterator(resources); !it.done(); it.next(resources)) {

View File

@ -44,17 +44,18 @@ class JsonSerializer : public VariantDataVisitor<size_t> {
auto slotId = object.head();
bool isKey = true;
while (slotId != NULL_SLOT) {
auto slot = resources_->getSlot(slotId);
formatter_.writeString(slot->key());
write(':');
slot->data()->accept(*this);
slotId = slot->next();
if (slotId != NULL_SLOT)
write(',');
write(isKey ? ':' : ',');
isKey = !isKey;
}
write('}');

View File

@ -45,14 +45,17 @@ class PrettyJsonSerializer : public JsonSerializer<TWriter> {
if (!it.done()) {
base::write("{\r\n");
nesting_++;
bool isKey = true;
while (!it.done()) {
indent();
base::visit(it.key());
base::write(": ");
if (isKey)
indent();
it->accept(*this);
it.next(base::resources_);
base::write(it.done() ? "\r\n" : ",\r\n");
if (isKey)
base::write(": ");
else
base::write(it.done() ? "\r\n" : ",\r\n");
isKey = !isKey;
}
nesting_--;
indent();

View File

@ -11,8 +11,6 @@
ARDUINOJSON_BEGIN_PRIVATE_NAMESPACE
inline void ResourceManager::freeSlot(SlotWithId slot) {
if (slot->ownsKey())
dereferenceString(slot->key());
slot->data()->setNull(this);
variantPools_.freeSlot(slot);
}

View File

@ -84,7 +84,6 @@ class MsgPackSerializer : public VariantDataVisitor<size_t> {
auto slotId = object.head();
while (slotId != NULL_SLOT) {
auto slot = resources_->getSlot(slotId);
visit(slot->key());
slot->data()->accept(*this);
slotId = slot->next();
}

View File

@ -34,7 +34,8 @@ class JsonObjectIterator {
}
JsonObjectIterator& operator++() {
iterator_.next(resources_);
iterator_.next(resources_); // key
iterator_.next(resources_); // value
return *this;
}
@ -69,7 +70,8 @@ class JsonObjectConstIterator {
}
JsonObjectConstIterator& operator++() {
iterator_.next(resources_);
iterator_.next(resources_); // key
iterator_.next(resources_); // value
return *this;
}

View File

@ -16,27 +16,27 @@ class JsonPair {
public:
// INTERNAL USE ONLY
JsonPair(detail::ObjectData::iterator iterator,
detail::ResourceManager* resources)
: iterator_(iterator), resources_(resources) {}
detail::ResourceManager* resources) {
if (!iterator.done()) {
key_ = iterator->asString();
iterator.next(resources);
value_ = JsonVariant(iterator.data(), resources);
}
}
// Returns the key.
JsonString key() const {
if (!iterator_.done())
return JsonString(iterator_.key(), iterator_.ownsKey()
? JsonString::Copied
: JsonString::Linked);
else
return JsonString();
return key_;
}
// Returns the value.
JsonVariant value() {
return JsonVariant(iterator_.data(), resources_);
return value_;
}
private:
detail::ObjectData::iterator iterator_;
detail::ResourceManager* resources_;
JsonString key_;
JsonVariant value_;
};
// A read-only key-value pair.
@ -44,27 +44,27 @@ class JsonPair {
class JsonPairConst {
public:
JsonPairConst(detail::ObjectData::iterator iterator,
const detail::ResourceManager* resources)
: iterator_(iterator), resources_(resources) {}
const detail::ResourceManager* resources) {
if (!iterator.done()) {
key_ = iterator->asString();
iterator.next(resources);
value_ = JsonVariantConst(iterator.data(), resources);
}
}
// Returns the key.
JsonString key() const {
if (!iterator_.done())
return JsonString(iterator_.key(), iterator_.ownsKey()
? JsonString::Copied
: JsonString::Linked);
else
return JsonString();
return key_;
}
// Returns the value.
JsonVariantConst value() const {
return JsonVariantConst(iterator_.data(), resources_);
return value_;
}
private:
detail::ObjectData::iterator iterator_;
const detail::ResourceManager* resources_;
JsonString key_;
JsonVariantConst value_;
};
ARDUINOJSON_END_PUBLIC_NAMESPACE

View File

@ -10,34 +10,8 @@ ARDUINOJSON_BEGIN_PRIVATE_NAMESPACE
class ObjectData : public CollectionData {
public:
VariantData* addMember(StringNode* key, ResourceManager* resources) {
ARDUINOJSON_ASSERT(key != nullptr);
auto it = addSlot(resources);
if (it.done())
return nullptr;
it.setKey(key);
return it.data();
}
template <typename TAdaptedString>
VariantData* addMember(TAdaptedString key, ResourceManager* resources) {
ARDUINOJSON_ASSERT(!key.isNull());
if (key.isLinked()) {
auto it = addSlot(resources);
if (!it.done())
it.setKey(key.data());
return it.data();
} else {
auto storedKey = resources->saveString(key);
if (!storedKey)
return nullptr;
auto it = addSlot(resources);
if (!it.done())
it.setKey(storedKey);
return it.data();
}
}
template <typename TAdaptedString> // also works with StringNode*
VariantData* addMember(TAdaptedString key, ResourceManager* resources);
template <typename TAdaptedString>
VariantData* getOrAddMember(TAdaptedString key, ResourceManager* resources);
@ -65,6 +39,27 @@ class ObjectData : public CollectionData {
obj->removeMember(key, resources);
}
void remove(iterator it, ResourceManager* resources) {
CollectionData::removePair(it, resources);
}
static void remove(ObjectData* obj, ObjectData::iterator it,
ResourceManager* resources) {
if (!obj)
return;
obj->remove(it, resources);
}
size_t size(const ResourceManager* resources) const {
return CollectionData::size(resources) / 2;
}
static size_t size(const ObjectData* obj, const ResourceManager* resources) {
if (!obj)
return 0;
return obj->size(resources);
}
private:
template <typename TAdaptedString>
iterator findKey(TAdaptedString key, const ResourceManager* resources) const;

View File

@ -12,15 +12,19 @@ ARDUINOJSON_BEGIN_PRIVATE_NAMESPACE
template <typename TAdaptedString>
inline VariantData* ObjectData::getMember(
TAdaptedString key, const ResourceManager* resources) const {
return findKey(key, resources).data();
auto it = findKey(key, resources);
if (it.done())
return nullptr;
it.next(resources);
return it.data();
}
template <typename TAdaptedString>
VariantData* ObjectData::getOrAddMember(TAdaptedString key,
ResourceManager* resources) {
auto it = findKey(key, resources);
if (!it.done())
return it.data();
auto data = getMember(key, resources);
if (data)
return data;
return addMember(key, resources);
}
@ -29,9 +33,11 @@ inline ObjectData::iterator ObjectData::findKey(
TAdaptedString key, const ResourceManager* resources) const {
if (key.isNull())
return iterator();
bool isKey = true;
for (auto it = createIterator(resources); !it.done(); it.next(resources)) {
if (stringEquals(key, adaptString(it.key())))
if (isKey && stringEquals(key, adaptString(it->asString())))
return it;
isKey = !isKey;
}
return iterator();
}
@ -42,4 +48,23 @@ inline void ObjectData::removeMember(TAdaptedString key,
remove(findKey(key, resources), resources);
}
template <typename TAdaptedString>
inline VariantData* ObjectData::addMember(TAdaptedString key,
ResourceManager* resources) {
auto keySlot = resources->allocSlot();
if (!keySlot)
return nullptr;
auto valueSlot = resources->allocSlot();
if (!valueSlot)
return nullptr;
if (!keySlot->data()->setString(key, resources))
return nullptr;
CollectionData::appendPair(keySlot, valueSlot, resources);
return valueSlot->data();
}
ARDUINOJSON_END_PRIVATE_NAMESPACE

View File

@ -14,8 +14,6 @@
ARDUINOJSON_BEGIN_PRIVATE_NAMESPACE
enum {
VALUE_MASK = 0x7F,
OWNED_VALUE_BIT = 0x01,
VALUE_IS_NULL = 0,
VALUE_IS_RAW_STRING = 0x03,
@ -34,8 +32,6 @@ enum {
COLLECTION_MASK = 0x60,
VALUE_IS_OBJECT = 0x20,
VALUE_IS_ARRAY = 0x40,
OWNED_KEY_BIT = 0x80
};
union VariantContent {

View File

@ -410,20 +410,29 @@ class VariantData {
}
template <typename TAdaptedString>
void setString(TAdaptedString value, ResourceManager* resources) {
bool setString(TAdaptedString value, ResourceManager* resources) {
setNull(resources);
if (value.isNull())
return;
return false;
if (value.isLinked()) {
setLinkedString(value.data());
return;
return true;
}
auto dup = resources->saveString(value);
if (dup)
if (dup) {
setOwnedString(dup);
return true;
}
return false;
}
bool setString(StringNode* s, ResourceManager*) {
setOwnedString(s);
return true;
}
template <typename TAdaptedString>
@ -447,7 +456,13 @@ class VariantData {
}
size_t size(const ResourceManager* resources) const {
return isCollection() ? content_.asCollection.size(resources) : 0;
if (isObject())
return content_.asObject.size(resources);
if (isArray())
return content_.asArray.size(resources);
return 0;
}
static size_t size(const VariantData* var, const ResourceManager* resources) {
@ -489,7 +504,7 @@ class VariantData {
}
uint8_t type() const {
return flags_ & VALUE_MASK;
return flags_;
}
private:
@ -503,8 +518,7 @@ class VariantData {
}
void setType(uint8_t t) {
flags_ &= OWNED_KEY_BIT;
flags_ |= t;
flags_ = t;
}
};

View File

@ -20,7 +20,6 @@ class VariantSlot {
VariantContent content_;
uint8_t flags_;
SlotId next_;
const char* key_;
public:
// Placement new
@ -30,7 +29,9 @@ class VariantSlot {
static void operator delete(void*, void*) noexcept {}
VariantSlot() : flags_(0), next_(NULL_SLOT), key_(0) {}
VariantSlot() : flags_(0), next_(NULL_SLOT) {
(void)flags_; // HACK: suppress Clang warning "private field is not used"
}
VariantData* data() {
return reinterpret_cast<VariantData*>(&content_);
@ -47,26 +48,6 @@ class VariantSlot {
void setNext(SlotId slot) {
next_ = slot;
}
void setKey(const char* k) {
ARDUINOJSON_ASSERT(k);
flags_ &= VALUE_MASK;
key_ = k;
}
void setKey(StringNode* k) {
ARDUINOJSON_ASSERT(k);
flags_ |= OWNED_KEY_BIT;
key_ = k->data;
}
const char* key() const {
return key_;
}
bool ownsKey() const {
return (flags_ & OWNED_KEY_BIT) != 0;
}
};
inline VariantData* slotData(VariantSlot* slot) {
@ -80,7 +61,7 @@ constexpr size_t sizeofArray(size_t n) {
// Returns the size (in bytes) of an object with n members.
constexpr size_t sizeofObject(size_t n) {
return n * sizeof(VariantSlot);
return 2 * n * sizeof(VariantSlot);
}
ARDUINOJSON_END_PRIVATE_NAMESPACE