feedc0de/ArduinoJson
1
0
Fork 0
forked from bblanchon/ArduinoJson
Code Pull Requests Activity

Store static string in a dedicated pool

Browse Source
This commit is contained in:
Benoit Blanchon
2025-02-24 17:48:37 +01:00
parent 907bb43481
commit 08b2400592
25 changed files with 133 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
extras/tests/Deprecated/BasicJsonDocument.cpp
View File

@ -54,7 +54,7 @@ TEST_CASE("BasicJsonDocument") {
doc["hello"] = "world";
auto copy = doc;
REQUIRE(copy.as<std::string>() == "{\"hello\":\"world\"}");
REQUIRE(allocatorLog == "AA");
REQUIRE(allocatorLog == "AAAA");
}
SECTION("capacity") {

6
extras/tests/Helpers/Allocators.hpp
View File

@ -275,6 +275,12 @@ inline size_t sizeofPool(
return MemoryPool<VariantData>::slotsToBytes(n);
}
inline size_t sizeofStaticStringPool(
ArduinoJson::detail::SlotCount n = ARDUINOJSON_POOL_CAPACITY) {
using namespace ArduinoJson::detail;
return MemoryPool<const char*>::slotsToBytes(n);
}
inline size_t sizeofStringBuffer(size_t iteration = 1) {
// returns 31, 63, 127, 255, etc.
auto capacity = ArduinoJson::detail::StringBuilder::initialCapacity;

1
extras/tests/JsonArray/add.cpp
View File

@ -56,6 +56,7 @@ TEST_CASE("JsonArray::add(T)") {
REQUIRE(array[0].is<int>() == false);
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
});
}

4
extras/tests/JsonDeserializer/destination_types.cpp
View File

@ -104,6 +104,8 @@ TEST_CASE("deserializeJson(MemberProxy)") {
REQUIRE(err == DeserializationError::Ok);
REQUIRE(doc.as<std::string>() == "{\"hello\":\"world\",\"value\":[42]}");
REQUIRE(spy.log() == AllocatorLog{});
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofStaticStringPool()),
});
}
}

8
extras/tests/JsonDeserializer/filter.cpp
View File

@ -825,7 +825,9 @@ TEST_CASE("shrink filter") {
deserializeJson(doc, "{}", DeserializationOption::Filter(filter));
REQUIRE(spy.log() == AllocatorLog{
Reallocate(sizeofPool(), sizeofObject(1)),
});
REQUIRE(spy.log() ==
AllocatorLog{
Reallocate(sizeofPool(), sizeofObject(1)),
Reallocate(sizeofStaticStringPool(), sizeofStaticStringPool(1)),
});
}

1
extras/tests/JsonDocument/ElementProxy.cpp
View File

@ -31,6 +31,7 @@ TEST_CASE("ElementProxy::add()") {
REQUIRE(doc.as<std::string>() == "[[\"world\"]]");
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
});
}

9
extras/tests/JsonDocument/MemberProxy.cpp
View File

@ -25,6 +25,7 @@ TEST_CASE("MemberProxy::add()") {
REQUIRE(doc.as<std::string>() == "{\"hello\":[42]}");
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
});
}
@ -34,6 +35,7 @@ TEST_CASE("MemberProxy::add()") {
REQUIRE(doc.as<std::string>() == "{\"hello\":[\"world\"]}");
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
});
}
@ -44,6 +46,7 @@ TEST_CASE("MemberProxy::add()") {
REQUIRE(doc.as<std::string>() == "{\"hello\":[\"world\"]}");
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
Allocate(sizeofString("world")),
});
}
@ -55,8 +58,8 @@ TEST_CASE("MemberProxy::add()") {
REQUIRE(doc.as<std::string>() == "{\"hello\":[\"world\"]}");
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
Allocate(sizeofString("world")),
});
}
@ -71,6 +74,7 @@ TEST_CASE("MemberProxy::add()") {
REQUIRE(doc.as<std::string>() == "{\"hello\":[\"world\"]}");
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
Allocate(sizeofString("world")),
});
}
@ -399,7 +403,7 @@ TEST_CASE("MemberProxy under memory constraints") {
}
SECTION("value slot allocation fails") {
timebomb.setCountdown(1);
timebomb.setCountdown(2);
// fill the pool entirely, but leave one slot for the key
doc["foo"][ARDUINOJSON_POOL_CAPACITY - 4] = 1;
@ -412,6 +416,7 @@ TEST_CASE("MemberProxy under memory constraints") {
REQUIRE(doc.overflowed() == true);
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
AllocateFail(sizeofPool()),
});
}

1
extras/tests/JsonDocument/add.cpp
View File

@ -32,6 +32,7 @@ TEST_CASE("JsonDocument::add(T)") {
REQUIRE(doc.as<std::string>() == "[\"hello\"]");
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
});
}

2
extras/tests/JsonDocument/constructor.cpp
View File

@ -62,6 +62,7 @@ TEST_CASE("JsonDocument constructor") {
REQUIRE(doc2.as<std::string>() == "{\"hello\":\"world\"}");
REQUIRE(spyingAllocator.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
});
}
@ -85,6 +86,7 @@ TEST_CASE("JsonDocument constructor") {
REQUIRE(doc2.as<std::string>() == "[\"hello\"]");
REQUIRE(spyingAllocator.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
});
}

4
extras/tests/JsonDocument/set.cpp
View File

@ -37,7 +37,9 @@ TEST_CASE("JsonDocument::set()") {
doc.set("example");
REQUIRE(doc.as<const char*>() == "example"_s);
REQUIRE(spy.log() == AllocatorLog{});
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofStaticStringPool()),
});
}
SECTION("const char*") {

19
extras/tests/JsonDocument/shrinkToFit.cpp
View File

@ -75,7 +75,11 @@ TEST_CASE("JsonDocument::shrinkToFit()") {
doc.shrinkToFit();
REQUIRE(doc.as<std::string>() == "hello");
REQUIRE(spyingAllocator.log() == AllocatorLog{});
REQUIRE(spyingAllocator.log() ==
AllocatorLog{
Allocate(sizeofStaticStringPool()),
Reallocate(sizeofStaticStringPool(), sizeofStaticStringPool(1)),
});
}
SECTION("owned string") {
@ -110,7 +114,9 @@ TEST_CASE("JsonDocument::shrinkToFit()") {
REQUIRE(spyingAllocator.log() ==
AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
Reallocate(sizeofPool(), sizeofObject(1)),
Reallocate(sizeofStaticStringPool(), sizeofStaticStringPool(1)),
});
}
@ -137,7 +143,9 @@ TEST_CASE("JsonDocument::shrinkToFit()") {
REQUIRE(spyingAllocator.log() ==
AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
Reallocate(sizeofPool(), sizeofArray(1)),
Reallocate(sizeofStaticStringPool(), sizeofStaticStringPool(1)),
});
}
@ -164,20 +172,23 @@ TEST_CASE("JsonDocument::shrinkToFit()") {
REQUIRE(spyingAllocator.log() ==
AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
Reallocate(sizeofPool(), sizeofObject(1)),
Reallocate(sizeofStaticStringPool(), sizeofStaticStringPool(2)),
});
}
SECTION("owned string in object") {
doc["key"] = "abcdefg"_s;
doc["key"_s] = "value"_s;
doc.shrinkToFit();
REQUIRE(doc.as<std::string>() == "{\"key\":\"abcdefg\"}");
REQUIRE(doc.as<std::string>() == "{\"key\":\"value\"}");
REQUIRE(spyingAllocator.log() ==
AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofString("abcdefg")),
Allocate(sizeofString("key")),
Allocate(sizeofString("value")),
Reallocate(sizeofPool(), sizeofPool(2)),
});
}

1
extras/tests/JsonDocument/subscript.cpp
View File

@ -112,6 +112,7 @@ TEST_CASE("JsonDocument::operator[] key storage") {
REQUIRE(doc.as<std::string>() == "{\"hello\":0}");
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
});
}

22
extras/tests/JsonObject/set.cpp
View File

@ -26,25 +26,12 @@ TEST_CASE("JsonObject::set()") {
REQUIRE(obj2["hello"] == "world"_s);
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
});
}
SECTION("copy local string value") {
obj1["hello"] = "world"_s;
spy.clearLog();
bool success = obj2.set(obj1);
REQUIRE(success == true);
REQUIRE(obj2["hello"] == "world"_s);
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofString("world")),
});
}
SECTION("copy local key") {
obj1["hello"_s] = "world";
SECTION("copy local string key and value") {
obj1["hello"_s] = "world"_s;
spy.clearLog();
bool success = obj2.set(obj1);
@ -54,6 +41,7 @@ TEST_CASE("JsonObject::set()") {
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofString("hello")),
Allocate(sizeofString("world")),
});
}
@ -110,7 +98,7 @@ TEST_CASE("JsonObject::set()") {
}
SECTION("copy fails in the middle of an array") {
TimebombAllocator timebomb(1);
TimebombAllocator timebomb(2);
JsonDocument doc3(&timebomb);
JsonObject obj3 = doc3.to<JsonObject>();

18
extras/tests/JsonObject/subscript.cpp
View File

@ -102,21 +102,25 @@ TEST_CASE("JsonObject::operator[]") {
REQUIRE(42 == obj[key]);
}
SECTION("should not duplicate const char*") {
SECTION("string literals") {
obj["hello"] = "world";
REQUIRE(spy.log() == AllocatorLog{Allocate(sizeofPool())});
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
});
}
SECTION("should duplicate char* value") {
obj["hello"] = const_cast<char*>("world");
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
Allocate(sizeofString("world")),
});
}
SECTION("should duplicate char* key") {
obj[const_cast<char*>("hello")] = "world";
obj[const_cast<char*>("hello")] = 42;
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofString("hello")),
@ -136,12 +140,13 @@ TEST_CASE("JsonObject::operator[]") {
obj["hello"] = "world"_s;
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
Allocate(sizeofString("world")),
});
}
SECTION("should duplicate std::string key") {
obj["hello"_s] = "world";
obj["hello"_s] = 42;
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofString("hello")),
@ -158,7 +163,7 @@ TEST_CASE("JsonObject::operator[]") {
}
SECTION("should duplicate a non-static JsonString key") {
obj[JsonString("hello", false)] = "world";
obj[JsonString("hello", false)] = 42;
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofString("hello")),
@ -166,9 +171,10 @@ TEST_CASE("JsonObject::operator[]") {
}
SECTION("should not duplicate a static JsonString key") {
obj[JsonString("hello", true)] = "world";
obj[JsonString("hello", true)] = 42;
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofPool()),
Allocate(sizeofStaticStringPool()),
});
}

6
extras/tests/JsonVariant/copy.cpp
View File

@ -38,13 +38,15 @@ TEST_CASE("JsonVariant::set(JsonVariant)") {
REQUIRE(var1.as<std::string>() == "{\"value\":[42]}");
}
SECTION("stores const char* by reference") {
SECTION("stores string literals by pointer") {
var1.set("hello!!");
spyingAllocator.clearLog();
var2.set(var1);
REQUIRE(spyingAllocator.log() == AllocatorLog{});
REQUIRE(spyingAllocator.log() == AllocatorLog{
Allocate(sizeofStaticStringPool()),
});
}
SECTION("stores char* by copy") {

8
extras/tests/JsonVariant/set.cpp
View File

@ -23,7 +23,9 @@ TEST_CASE("JsonVariant::set() when there is enough memory") {
REQUIRE(result == true);
CHECK(variant ==
"hello"_s); // linked string cannot contain '\0' at the moment
CHECK(spy.log() == AllocatorLog{});
CHECK(spy.log() == AllocatorLog{
Allocate(sizeofStaticStringPool()),
});
}
SECTION("const char*") {
@ -137,7 +139,9 @@ TEST_CASE("JsonVariant::set() when there is enough memory") {
REQUIRE(result == true);
REQUIRE(variant == "world"); // stores by pointer
REQUIRE(spy.log() == AllocatorLog{});
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofStaticStringPool()),
});
}
SECTION("non-static JsonString") {

4
extras/tests/MsgPackDeserializer/destination_types.cpp
View File

@ -104,6 +104,8 @@ TEST_CASE("deserializeMsgPack(MemberProxy)") {
REQUIRE(err == DeserializationError::Ok);
REQUIRE(doc.as<std::string>() == "{\"hello\":\"world\",\"value\":[42]}");
REQUIRE(spy.log() == AllocatorLog{});
REQUIRE(spy.log() == AllocatorLog{
Allocate(sizeofStaticStringPool()),
});
}
}

5
src/ArduinoJson/Memory/MemoryPool.hpp
View File

@ -34,6 +34,11 @@ class Slot {
return ptr_;
}
T& operator*() const {
ARDUINOJSON_ASSERT(ptr_ != nullptr);
return *ptr_;
}
T* operator->() const {
ARDUINOJSON_ASSERT(ptr_ != nullptr);
return ptr_;

17
src/ArduinoJson/Memory/ResourceManager.hpp
View File

@ -34,6 +34,7 @@ class ResourceManager {
~ResourceManager() {
stringPool_.clear(allocator_);
variantPools_.clear(allocator_);
staticStringsPools_.clear(allocator_);
}
ResourceManager(const ResourceManager&) = delete;
@ -42,6 +43,7 @@ class ResourceManager {
friend void swap(ResourceManager& a, ResourceManager& b) {
swap(a.stringPool_, b.stringPool_);
swap(a.variantPools_, b.variantPools_);
swap(a.staticStringsPools_, b.staticStringsPools_);
swap_(a.allocator_, b.allocator_);
swap_(a.overflowed_, b.overflowed_);
}
@ -111,14 +113,28 @@ class ResourceManager {
stringPool_.dereference(s, allocator_);
}
SlotId saveStaticString(const char* s) {
auto slot = staticStringsPools_.allocSlot(allocator_);
if (!slot)
return NULL_SLOT;
*slot = s;
return slot.id();
}
const char* getStaticString(SlotId id) const {
return *staticStringsPools_.getSlot(id);
}
void clear() {
variantPools_.clear(allocator_);
overflowed_ = false;
stringPool_.clear(allocator_);
staticStringsPools_.clear(allocator_);
}
void shrinkToFit() {
variantPools_.shrinkToFit(allocator_);
staticStringsPools_.shrinkToFit(allocator_);
}
private:
@ -126,6 +142,7 @@ class ResourceManager {
bool overflowed_;
StringPool stringPool_;
MemoryPoolList<SlotData> variantPools_;
MemoryPoolList<const char*> staticStringsPools_;
};
ARDUINOJSON_END_PRIVATE_NAMESPACE

4
src/ArduinoJson/Object/JsonPair.hpp
View File

@ -18,7 +18,7 @@ class JsonPair {
JsonPair(detail::ObjectData::iterator iterator,
detail::ResourceManager* resources) {
if (!iterator.done()) {
key_ = iterator->asString();
key_ = iterator->asString(resources);
iterator.next(resources);
value_ = JsonVariant(iterator.data(), resources);
}
@ -46,7 +46,7 @@ class JsonPairConst {
JsonPairConst(detail::ObjectData::iterator iterator,
const detail::ResourceManager* resources) {
if (!iterator.done()) {
key_ = iterator->asString();
key_ = iterator->asString(resources);
iterator.next(resources);
value_ = JsonVariantConst(iterator.data(), resources);
}

2
src/ArduinoJson/Object/ObjectImpl.hpp
View File

@ -36,7 +36,7 @@ inline ObjectData::iterator ObjectData::findKey(
return iterator();
bool isKey = true;
for (auto it = createIterator(resources); !it.done(); it.next(resources)) {
if (isKey && stringEquals(key, adaptString(it->asString())))
if (isKey && stringEquals(key, adaptString(it->asString(resources))))
return it;
isKey = !isKey;
}

4
src/ArduinoJson/Variant/ConverterImpl.hpp
View File

@ -160,7 +160,7 @@ struct Converter<const char*> : private detail::VariantAttorney {
static const char* fromJson(JsonVariantConst src) {
auto data = getData(src);
return data ? data->asString().c_str() : 0;
return data ? data->asString(getResourceManager(src)).c_str() : 0;
}
static bool checkJson(JsonVariantConst src) {
@ -178,7 +178,7 @@ struct Converter<JsonString> : private detail::VariantAttorney {
static JsonString fromJson(JsonVariantConst src) {
auto data = getData(src);
return data ? data->asString() : JsonString();
return data ? data->asString(getResourceManager(src)) : JsonString();
}
static bool checkJson(JsonVariantConst src) {

3
src/ArduinoJson/Variant/VariantContent.hpp
View File

@ -53,13 +53,10 @@ union VariantContent {
bool asBoolean;
uint32_t asUint32;
int32_t asInt32;
#if ARDUINOJSON_USE_EXTENSIONS
SlotId asSlotId;
#endif
ArrayData asArray;
ObjectData asObject;
CollectionData asCollection;
const char* asLinkedString;
struct StringNode* asOwnedString;
};

19
src/ArduinoJson/Variant/VariantData.hpp
View File

@ -64,7 +64,7 @@ class VariantData {
return visit.visit(content_.asObject);
case VariantType::LinkedString:
return visit.visit(JsonString(content_.asLinkedString, true));
return visit.visit(JsonString(asLinkedString(resources), true));
case VariantType::OwnedString:
return visit.visit(JsonString(content_.asOwnedString->data,
@ -200,7 +200,7 @@ class VariantData {
return static_cast<T>(extension->asInt64);
#endif
case VariantType::LinkedString:
str = content_.asLinkedString;
str = asLinkedString(resources);
break;
case VariantType::OwnedString:
str = content_.asOwnedString->data;
@ -242,7 +242,7 @@ class VariantData {
return convertNumber<T>(extension->asInt64);
#endif
case VariantType::LinkedString:
str = content_.asLinkedString;
str = asLinkedString(resources);
break;
case VariantType::OwnedString:
str = content_.asOwnedString->data;
@ -279,10 +279,12 @@ class VariantData {
}
}
JsonString asString() const {
const char* asLinkedString(const ResourceManager* resources) const;
JsonString asString(const ResourceManager* resources) const {
switch (type_) {
case VariantType::LinkedString:
return JsonString(content_.asLinkedString, true);
return JsonString(asLinkedString(resources), true);
case VariantType::OwnedString:
return JsonString(content_.asOwnedString->data,
content_.asOwnedString->length);
@ -502,12 +504,7 @@ class VariantData {
var->setString(value, resources);
}
void setLinkedString(const char* s) {
ARDUINOJSON_ASSERT(type_ == VariantType::Null); // must call clear() first
ARDUINOJSON_ASSERT(s);
type_ = VariantType::LinkedString;
content_.asLinkedString = s;
}
bool setLinkedString(const char* s, ResourceManager* resources);
void setOwnedString(StringNode* s) {
ARDUINOJSON_ASSERT(type_ == VariantType::Null); // must call clear() first

26
src/ArduinoJson/Variant/VariantImpl.hpp
View File

@ -18,6 +18,20 @@ inline void VariantData::setRawString(SerializedValue<T> value,
setRawString(dup);
}
inline bool VariantData::setLinkedString(const char* s,
ResourceManager* resources) {
ARDUINOJSON_ASSERT(type_ == VariantType::Null); // must call clear() first
ARDUINOJSON_ASSERT(s);
auto slotId = resources->saveStaticString(s);
if (slotId == NULL_SLOT)
return false;
type_ = VariantType::LinkedString;
content_.asSlotId = slotId;
return true;
}
template <typename TAdaptedString>
inline bool VariantData::setString(TAdaptedString value,
ResourceManager* resources) {
@ -26,10 +40,8 @@ inline bool VariantData::setString(TAdaptedString value,
if (value.isNull())
return false;
if (value.isStatic()) {
setLinkedString(value.data());
return true;
}
if (value.isStatic())
return setLinkedString(value.data(), resources);
auto dup = resources->saveString(value);
if (dup) {
@ -65,6 +77,12 @@ inline const VariantExtension* VariantData::getExtension(
}
#endif
inline const char* VariantData::asLinkedString(
const ResourceManager* resources) const {
ARDUINOJSON_ASSERT(type_ == VariantType::LinkedString);
return resources->getStaticString(content_.asSlotId);
}
template <typename T>
enable_if_t<sizeof(T) == 8, bool> VariantData::setFloat(
T value, ResourceManager* resources) {