From d8f3058efae200eb6f05dcf653f3f8a250b89f25 Mon Sep 17 00:00:00 2001 From: Benoit Blanchon Date: Tue, 11 Apr 2023 10:03:47 +0200 Subject: [PATCH] Store the strings in the heap --- CHANGELOG.md | 1 + README.md | 1 - extras/tests/JsonDeserializer/string.cpp | 61 ++++++--- extras/tests/JsonDocument/assignment.cpp | 88 +++++++------ extras/tests/JsonDocument/constructor.cpp | 29 +++-- extras/tests/JsonDocument/garbageCollect.cpp | 13 +- extras/tests/JsonDocument/overflowed.cpp | 13 +- extras/tests/JsonDocument/shrinkToFit.cpp | 33 ++--- extras/tests/JsonObject/copy.cpp | 9 +- extras/tests/JsonVariant/misc.cpp | 5 + extras/tests/JsonVariant/set.cpp | 4 +- extras/tests/MemoryPool/StringCopier.cpp | 79 +++++++++--- extras/tests/MemoryPool/saveString.cpp | 53 +------- extras/tests/Misc/printable.cpp | 71 +++++++--- .../deserializeVariant.cpp | 108 +++++++--------- idf_component.yml | 2 +- library.json | 2 +- library.properties | 2 +- src/ArduinoJson/Collection/CollectionData.hpp | 2 +- src/ArduinoJson/Collection/CollectionImpl.hpp | 7 +- src/ArduinoJson/Json/JsonDeserializer.hpp | 1 - src/ArduinoJson/Memory/MemoryPool.hpp | 121 +++++++++++------- .../MsgPack/MsgPackDeserializer.hpp | 1 - .../StringStorage/StringCopier.hpp | 53 ++++---- src/ArduinoJson/Variant/ConverterImpl.hpp | 34 ++--- src/ArduinoJson/Variant/VariantData.hpp | 10 +- src/ArduinoJson/Variant/VariantSlot.hpp | 8 +- 27 files changed, 434 insertions(+), 377 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef0be16e..6ba30c69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,3 +11,4 @@ HEAD * Remove `JSON_ARRAY_SIZE()`, `JSON_OBJECT_SIZE()`, and `JSON_STRING_SIZE()` * Remove `ARDUINOJSON_ENABLE_STRING_DEDUPLICATION` (string deduplication cannot be enabled anymore) * Remove `JsonDocument::capacity()` +* Store the strings in the heap diff --git a/README.md b/README.md index fb658d12..8748c143 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,6 @@ ArduinoJson is a C++ JSON library for Arduino and IoT (Internet Of Things). * [Twice smaller than the "official" Arduino_JSON library](https://arduinojson.org/2019/11/19/arduinojson-vs-arduino_json/) * [Almost 10% faster than the "official" Arduino_JSON library](https://arduinojson.org/2019/11/19/arduinojson-vs-arduino_json/) * [Consumes roughly 10% less RAM than the "official" Arduino_JSON library](https://arduinojson.org/2019/11/19/arduinojson-vs-arduino_json/) - * [Fixed memory allocation, no heap fragmentation](https://arduinojson.org/v6/api/jsondocument/) * [Deduplicates strings](https://arduinojson.org/news/2020/08/01/version-6-16-0/) * Versatile * Supports [custom allocators (to use external RAM chip, for example)](https://arduinojson.org/v6/how-to/use-external-ram-on-esp32/) diff --git a/extras/tests/JsonDeserializer/string.cpp b/extras/tests/JsonDeserializer/string.cpp index e20af40d..521f4d1c 100644 --- a/extras/tests/JsonDeserializer/string.cpp +++ b/extras/tests/JsonDeserializer/string.cpp @@ -6,6 +6,8 @@ #include #include +#include "Allocators.hpp" + using ArduinoJson::detail::sizeofArray; using ArduinoJson::detail::sizeofObject; using ArduinoJson::detail::sizeofString; @@ -96,42 +98,67 @@ TEST_CASE("Invalid JSON string") { } } -TEST_CASE("Not enough room to save the key") { - JsonDocument doc(sizeofObject(1) + sizeofString(7)); +TEST_CASE("Allocation of the key fails") { + TimebombAllocator timebombAllocator(1); + SpyingAllocator spyingAllocator(&timebombAllocator); + JsonDocument doc(1024, &spyingAllocator); - SECTION("Quoted string") { + SECTION("Quoted string, first member") { REQUIRE(deserializeJson(doc, "{\"example\":1}") == - DeserializationError::Ok); - REQUIRE(deserializeJson(doc, "{\"accuracy\":1}") == DeserializationError::NoMemory); - REQUIRE(deserializeJson(doc, "{\"hello\":1,\"world\"}") == - DeserializationError::NoMemory); // fails in the second string + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(1024) + << AllocatorLog::AllocateFail(sizeofString(31))); } - SECTION("Non-quoted string") { - REQUIRE(deserializeJson(doc, "{example:1}") == DeserializationError::Ok); - REQUIRE(deserializeJson(doc, "{accuracy:1}") == + SECTION("Quoted string, second member") { + timebombAllocator.setCountdown(2); + REQUIRE(deserializeJson(doc, "{\"hello\":1,\"world\"}") == DeserializationError::NoMemory); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(1024) + << AllocatorLog::Allocate(sizeofString(31)) + << AllocatorLog::Reallocate(sizeofString(31), + sizeofString(5)) + << AllocatorLog::AllocateFail(sizeofString(31))); + } + + SECTION("Non-Quoted string, first member") { + REQUIRE(deserializeJson(doc, "{example:1}") == + DeserializationError::NoMemory); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(1024) + << AllocatorLog::AllocateFail(sizeofString(31))); + } + + SECTION("Non-Quoted string, second member") { + timebombAllocator.setCountdown(2); REQUIRE(deserializeJson(doc, "{hello:1,world}") == - DeserializationError::NoMemory); // fails in the second string + DeserializationError::NoMemory); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(1024) + << AllocatorLog::Allocate(sizeofString(31)) + << AllocatorLog::Reallocate(sizeofString(31), + sizeofString(5)) + << AllocatorLog::AllocateFail(sizeofString(31))); } } -TEST_CASE("Empty memory pool") { - // NOLINTNEXTLINE(clang-analyzer-optin.portability.UnixAPI) - JsonDocument doc(0); +TEST_CASE("String allocation fails") { + SpyingAllocator spyingAllocator(FailingAllocator::instance()); + JsonDocument doc(0, &spyingAllocator); SECTION("Input is const char*") { REQUIRE(deserializeJson(doc, "\"hello\"") == DeserializationError::NoMemory); - REQUIRE(deserializeJson(doc, "\"\"") == DeserializationError::NoMemory); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::AllocateFail(sizeofString(31))); } SECTION("Input is const char*") { char hello[] = "\"hello\""; REQUIRE(deserializeJson(doc, hello) == DeserializationError::Ok); - char empty[] = "\"hello\""; - REQUIRE(deserializeJson(doc, empty) == DeserializationError::Ok); + REQUIRE(spyingAllocator.log() == AllocatorLog()); } } diff --git a/extras/tests/JsonDocument/assignment.cpp b/extras/tests/JsonDocument/assignment.cpp index b8d40cdd..f3b440ed 100644 --- a/extras/tests/JsonDocument/assignment.cpp +++ b/extras/tests/JsonDocument/assignment.cpp @@ -9,63 +9,59 @@ using ArduinoJson::detail::sizeofArray; using ArduinoJson::detail::sizeofObject; +using ArduinoJson::detail::sizeofString; TEST_CASE("JsonDocument assignment") { SpyingAllocator spyingAllocator; SECTION("Copy assignment same capacity") { - { - JsonDocument doc1(1024, &spyingAllocator); - deserializeJson(doc1, "{\"hello\":\"world\"}"); - JsonDocument doc2(1024, &spyingAllocator); + JsonDocument doc1(1024, &spyingAllocator); + deserializeJson(doc1, "{\"hello\":\"world\"}"); + JsonDocument doc2(1024, &spyingAllocator); + spyingAllocator.clearLog(); - doc2 = doc1; + doc2 = doc1; - REQUIRE(doc2.as() == "{\"hello\":\"world\"}"); - } - REQUIRE(spyingAllocator.log() == AllocatorLog() - << AllocatorLog::Allocate(1024) - << AllocatorLog::Allocate(1024) - << AllocatorLog::Deallocate(1024) - << AllocatorLog::Deallocate(1024)); + REQUIRE(doc2.as() == "{\"hello\":\"world\"}"); + + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(sizeofString(5)) // hello + << AllocatorLog::Allocate(sizeofString(5)) // world + ); } SECTION("Copy assignment reallocates when capacity is smaller") { - { - JsonDocument doc1(4096, &spyingAllocator); - deserializeJson(doc1, "{\"hello\":\"world\"}"); - JsonDocument doc2(8, &spyingAllocator); + JsonDocument doc1(4096, &spyingAllocator); + deserializeJson(doc1, "{\"hello\":\"world\"}"); + JsonDocument doc2(8, &spyingAllocator); + spyingAllocator.clearLog(); - doc2 = doc1; + doc2 = doc1; - REQUIRE(doc2.as() == "{\"hello\":\"world\"}"); - } - REQUIRE(spyingAllocator.log() == AllocatorLog() - << AllocatorLog::Allocate(4096) - << AllocatorLog::Allocate(8) - << AllocatorLog::Deallocate(8) - << AllocatorLog::Allocate(4096) - << AllocatorLog::Deallocate(4096) - << AllocatorLog::Deallocate(4096)); + REQUIRE(doc2.as() == "{\"hello\":\"world\"}"); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Deallocate(8) + << AllocatorLog::Allocate(4096) + << AllocatorLog::Allocate(sizeofString(5)) // hello + << AllocatorLog::Allocate(sizeofString(5)) // world + ); } SECTION("Copy assignment reallocates when capacity is larger") { - { - JsonDocument doc1(1024, &spyingAllocator); - deserializeJson(doc1, "{\"hello\":\"world\"}"); - JsonDocument doc2(4096, &spyingAllocator); + JsonDocument doc1(1024, &spyingAllocator); + deserializeJson(doc1, "{\"hello\":\"world\"}"); + JsonDocument doc2(4096, &spyingAllocator); + spyingAllocator.clearLog(); - doc2 = doc1; + doc2 = doc1; - REQUIRE(doc2.as() == "{\"hello\":\"world\"}"); - } - REQUIRE(spyingAllocator.log() == AllocatorLog() - << AllocatorLog::Allocate(1024) - << AllocatorLog::Allocate(4096) - << AllocatorLog::Deallocate(4096) - << AllocatorLog::Allocate(1024) - << AllocatorLog::Deallocate(1024) - << AllocatorLog::Deallocate(1024)); + REQUIRE(doc2.as() == "{\"hello\":\"world\"}"); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Deallocate(4096) + << AllocatorLog::Allocate(1024) + << AllocatorLog::Allocate(sizeofString(5)) // hello + << AllocatorLog::Allocate(sizeofString(5)) // world + ); } SECTION("Move assign") { @@ -79,11 +75,13 @@ TEST_CASE("JsonDocument assignment") { REQUIRE(doc2.as() == "The size of this string is 32!!"); REQUIRE(doc1.as() == "null"); } - REQUIRE(spyingAllocator.log() == AllocatorLog() - << AllocatorLog::Allocate(4096) - << AllocatorLog::Allocate(8) - << AllocatorLog::Deallocate(8) - << AllocatorLog::Deallocate(4096)); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(4096) + << AllocatorLog::Allocate(sizeofString(31)) + << AllocatorLog::Allocate(8) + << AllocatorLog::Deallocate(8) + << AllocatorLog::Deallocate(sizeofString(31)) + << AllocatorLog::Deallocate(4096)); } SECTION("Assign from JsonObject") { diff --git a/extras/tests/JsonDocument/constructor.cpp b/extras/tests/JsonDocument/constructor.cpp index a325a82b..841ed952 100644 --- a/extras/tests/JsonDocument/constructor.cpp +++ b/extras/tests/JsonDocument/constructor.cpp @@ -8,6 +8,7 @@ #include "Allocators.hpp" using ArduinoJson::detail::addPadding; +using ArduinoJson::detail::sizeofString; TEST_CASE("JsonDocument constructor") { SpyingAllocator spyingAllocator; @@ -29,11 +30,15 @@ TEST_CASE("JsonDocument constructor") { REQUIRE(doc1.as() == "The size of this string is 32!!"); REQUIRE(doc2.as() == "The size of this string is 32!!"); } - REQUIRE(spyingAllocator.log() == AllocatorLog() - << AllocatorLog::Allocate(4096) - << AllocatorLog::Allocate(4096) - << AllocatorLog::Deallocate(4096) - << AllocatorLog::Deallocate(4096)); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(4096) + << AllocatorLog::Allocate(sizeofString(31)) + << AllocatorLog::Allocate(4096) + << AllocatorLog::Allocate(sizeofString(31)) + << AllocatorLog::Deallocate(sizeofString(31)) + << AllocatorLog::Deallocate(4096) + << AllocatorLog::Deallocate(sizeofString(31)) + << AllocatorLog::Deallocate(4096)); } SECTION("JsonDocument(JsonDocument&&)") { @@ -46,9 +51,11 @@ TEST_CASE("JsonDocument constructor") { REQUIRE(doc2.as() == "The size of this string is 32!!"); REQUIRE(doc1.as() == "null"); } - REQUIRE(spyingAllocator.log() == AllocatorLog() - << AllocatorLog::Allocate(4096) - << AllocatorLog::Deallocate(4096)); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(4096) + << AllocatorLog::Allocate(sizeofString(31)) + << AllocatorLog::Deallocate(sizeofString(31)) + << AllocatorLog::Deallocate(4096)); } SECTION("JsonDocument(JsonObject)") { @@ -82,7 +89,9 @@ TEST_CASE("JsonDocument constructor") { JsonDocument doc2(doc1.as(), &spyingAllocator); REQUIRE(doc2.as() == "hello"); - REQUIRE(spyingAllocator.log() == AllocatorLog() << AllocatorLog::Allocate( - addPadding(doc1.memoryUsage()))); + REQUIRE( + spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(addPadding(doc1.memoryUsage())) + << AllocatorLog::Allocate(sizeofString(5))); } } diff --git a/extras/tests/JsonDocument/garbageCollect.cpp b/extras/tests/JsonDocument/garbageCollect.cpp index 6b25272b..5bf759b6 100644 --- a/extras/tests/JsonDocument/garbageCollect.cpp +++ b/extras/tests/JsonDocument/garbageCollect.cpp @@ -21,16 +21,19 @@ TEST_CASE("JsonDocument::garbageCollect()") { deserializeJson(doc, "{\"blanket\":1,\"dancing\":2}"); REQUIRE(doc.memoryUsage() == sizeofObject(2) + 2 * sizeofString(7)); doc.remove("blanket"); + spyingAllocator.clearLog(); bool result = doc.garbageCollect(); REQUIRE(result == true); REQUIRE(doc.memoryUsage() == sizeofObject(1) + sizeofString(7)); REQUIRE(doc.as() == "{\"dancing\":2}"); - REQUIRE(spyingAllocator.log() == AllocatorLog() - << AllocatorLog::Allocate(4096) - << AllocatorLog::Allocate(4096) - << AllocatorLog::Deallocate(4096)); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(4096) + << AllocatorLog::Allocate(sizeofString(7)) + << AllocatorLog::Deallocate(sizeofString(7)) + << AllocatorLog::Deallocate(sizeofString(7)) + << AllocatorLog::Deallocate(4096)); } SECTION("when allocation fails") { @@ -38,6 +41,7 @@ TEST_CASE("JsonDocument::garbageCollect()") { REQUIRE(doc.memoryUsage() == sizeofObject(2) + 2 * sizeofString(7)); doc.remove("blanket"); controllableAllocator.disable(); + spyingAllocator.clearLog(); bool result = doc.garbageCollect(); @@ -46,7 +50,6 @@ TEST_CASE("JsonDocument::garbageCollect()") { REQUIRE(doc.as() == "{\"dancing\":2}"); REQUIRE(spyingAllocator.log() == AllocatorLog() - << AllocatorLog::Allocate(4096) << AllocatorLog::AllocateFail(4096)); } } diff --git a/extras/tests/JsonDocument/overflowed.cpp b/extras/tests/JsonDocument/overflowed.cpp index fd66160f..a02d4ed5 100644 --- a/extras/tests/JsonDocument/overflowed.cpp +++ b/extras/tests/JsonDocument/overflowed.cpp @@ -5,8 +5,9 @@ #include #include +#include "Allocators.hpp" + using ArduinoJson::detail::sizeofArray; -using ArduinoJson::detail::sizeofString; TEST_CASE("JsonDocument::overflowed()") { SECTION("returns false on a fresh object") { @@ -27,13 +28,15 @@ TEST_CASE("JsonDocument::overflowed()") { } SECTION("returns true after a failed string copy") { - JsonDocument doc(sizeofArray(1)); + ControllableAllocator allocator; + JsonDocument doc(sizeofArray(1), &allocator); + allocator.disable(); doc.add(std::string("example")); CHECK(doc.overflowed() == true); } SECTION("returns false after a successful string copy") { - JsonDocument doc(sizeofArray(1) + sizeofString(7)); + JsonDocument doc(sizeofArray(1)); doc.add(std::string("example")); CHECK(doc.overflowed() == false); } @@ -46,12 +49,12 @@ TEST_CASE("JsonDocument::overflowed()") { SECTION("returns true after a failed deserialization") { JsonDocument doc(sizeofArray(1)); - deserializeJson(doc, "[\"example\"]"); + deserializeJson(doc, "[1, 2]"); CHECK(doc.overflowed() == true); } SECTION("returns false after a successful deserialization") { - JsonDocument doc(sizeofArray(1) + sizeofString(7)); + JsonDocument doc(sizeofArray(1)); deserializeJson(doc, "[\"example\"]"); CHECK(doc.overflowed() == false); } diff --git a/extras/tests/JsonDocument/shrinkToFit.cpp b/extras/tests/JsonDocument/shrinkToFit.cpp index bc25f8e9..e10dacab 100644 --- a/extras/tests/JsonDocument/shrinkToFit.cpp +++ b/extras/tests/JsonDocument/shrinkToFit.cpp @@ -86,13 +86,15 @@ TEST_CASE("JsonDocument::shrinkToFit()") { SECTION("owned string") { doc.set(std::string("abcdefg")); + REQUIRE(doc.as() == "abcdefg"); doc.shrinkToFit(); REQUIRE(doc.as() == "abcdefg"); REQUIRE(spyingAllocator.log() == AllocatorLog() << AllocatorLog::Allocate(4096) - << AllocatorLog::Reallocate(4096, sizeofString(7))); + << AllocatorLog::Allocate(sizeofString(7)) + << AllocatorLog::Reallocate(4096, 0)); } SECTION("linked raw") { @@ -114,7 +116,8 @@ TEST_CASE("JsonDocument::shrinkToFit()") { REQUIRE(doc.as() == "[{},12]"); REQUIRE(spyingAllocator.log() == AllocatorLog() << AllocatorLog::Allocate(4096) - << AllocatorLog::Reallocate(4096, sizeofString(7))); + << AllocatorLog::Allocate(sizeofString(7)) + << AllocatorLog::Reallocate(4096, 0)); } SECTION("linked key") { @@ -136,8 +139,8 @@ TEST_CASE("JsonDocument::shrinkToFit()") { REQUIRE(doc.as() == "{\"abcdefg\":42}"); REQUIRE(spyingAllocator.log() == AllocatorLog() << AllocatorLog::Allocate(4096) - << AllocatorLog::Reallocate( - 4096, sizeofObject(1) + sizeofString(7))); + << AllocatorLog::Allocate(sizeofString(7)) + << AllocatorLog::Reallocate(4096, sizeofObject(1))); } SECTION("linked string in array") { @@ -159,8 +162,8 @@ TEST_CASE("JsonDocument::shrinkToFit()") { REQUIRE(doc.as() == "[\"abcdefg\"]"); REQUIRE(spyingAllocator.log() == AllocatorLog() << AllocatorLog::Allocate(4096) - << AllocatorLog::Reallocate( - 4096, sizeofArray(1) + sizeofString(7))); + << AllocatorLog::Allocate(sizeofString(7)) + << AllocatorLog::Reallocate(4096, sizeofArray(1))); } SECTION("linked string in object") { @@ -182,21 +185,7 @@ TEST_CASE("JsonDocument::shrinkToFit()") { REQUIRE(doc.as() == "{\"key\":\"abcdefg\"}"); REQUIRE(spyingAllocator.log() == AllocatorLog() << AllocatorLog::Allocate(4096) - << AllocatorLog::Reallocate( - 4096, sizeofObject(1) + sizeofString(7))); - } - - SECTION("unaligned") { - doc.add(std::string("?")); // two bytes in the string pool - REQUIRE(doc.memoryUsage() == sizeofObject(1) + sizeofString(1)); - - doc.shrinkToFit(); - - // the new capacity should be padded to align the pointers - REQUIRE(doc[0] == "?"); - REQUIRE(spyingAllocator.log() == - AllocatorLog() << AllocatorLog::Allocate(4096) - << AllocatorLog::Reallocate( - 4096, sizeofArray(1) + sizeof(void*))); + << AllocatorLog::Allocate(sizeofString(7)) + << AllocatorLog::Reallocate(4096, sizeofObject(1))); } } diff --git a/extras/tests/JsonObject/copy.cpp b/extras/tests/JsonObject/copy.cpp index b2a0837e..85de6bfc 100644 --- a/extras/tests/JsonObject/copy.cpp +++ b/extras/tests/JsonObject/copy.cpp @@ -77,24 +77,25 @@ TEST_CASE("JsonObject::set()") { JsonDocument doc3(sizeofObject(1)); JsonObject obj3 = doc3.to(); - obj1[std::string("hello")] = "world"; + obj1["a"] = 1; + obj1["b"] = 2; bool success = obj3.set(obj1); REQUIRE(success == false); - REQUIRE(doc3.as() == "{}"); + REQUIRE(doc3.as() == "{\"a\":1}"); } SECTION("destination too small to store the value") { JsonDocument doc3(sizeofObject(1)); JsonObject obj3 = doc3.to(); - obj1["hello"] = std::string("world"); + obj1["hello"][1] = "world"; bool success = obj3.set(obj1); REQUIRE(success == false); - REQUIRE(doc3.as() == "{\"hello\":null}"); + REQUIRE(doc3.as() == "{\"hello\":[]}"); } SECTION("destination is null") { diff --git a/extras/tests/JsonVariant/misc.cpp b/extras/tests/JsonVariant/misc.cpp index fbbb4b7b..2f61582f 100644 --- a/extras/tests/JsonVariant/misc.cpp +++ b/extras/tests/JsonVariant/misc.cpp @@ -10,6 +10,11 @@ TEST_CASE("VariantData") { true); } +TEST_CASE("StringNode") { + REQUIRE(std::is_standard_layout::value == + true); +} + TEST_CASE("JsonVariant from JsonArray") { SECTION("JsonArray is null") { JsonArray arr; diff --git a/extras/tests/JsonVariant/set.cpp b/extras/tests/JsonVariant/set.cpp index e48f6379..adc49b00 100644 --- a/extras/tests/JsonVariant/set.cpp +++ b/extras/tests/JsonVariant/set.cpp @@ -5,6 +5,8 @@ #include #include +#include "Allocators.hpp" + enum ErrorCode { ERROR_01 = 1, ERROR_10 = 10 }; TEST_CASE("JsonVariant::set() when there is enough memory") { @@ -128,7 +130,7 @@ TEST_CASE("JsonVariant::set() when there is enough memory") { } TEST_CASE("JsonVariant::set() with not enough memory") { - JsonDocument doc(1); + JsonDocument doc(1, FailingAllocator::instance()); JsonVariant v = doc.to(); diff --git a/extras/tests/MemoryPool/StringCopier.cpp b/extras/tests/MemoryPool/StringCopier.cpp index bd28b4fd..2988584c 100644 --- a/extras/tests/MemoryPool/StringCopier.cpp +++ b/extras/tests/MemoryPool/StringCopier.cpp @@ -5,11 +5,30 @@ #include #include +#include "Allocators.hpp" + using namespace ArduinoJson::detail; TEST_CASE("StringCopier") { - SECTION("Works when buffer is big enough") { - MemoryPool pool(addPadding(sizeofString(5))); + ControllableAllocator controllableAllocator; + SpyingAllocator spyingAllocator(&controllableAllocator); + MemoryPool pool(0, &spyingAllocator); + + SECTION("Empty string") { + StringCopier str(&pool); + + str.startString(); + str.save(); + + REQUIRE(pool.size() == sizeofString(0)); + REQUIRE(pool.overflowed() == false); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(sizeofString(31)) + << AllocatorLog::Reallocate(sizeofString(31), + sizeofString(0))); + } + + SECTION("Short string fits in first allocation") { StringCopier str(&pool); str.startString(); @@ -18,38 +37,60 @@ TEST_CASE("StringCopier") { REQUIRE(str.isValid() == true); REQUIRE(str.str() == "hello"); REQUIRE(pool.overflowed() == false); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(sizeofString(31))); } - SECTION("Returns null when too small") { - MemoryPool pool(sizeof(void*)); + SECTION("Long string needs reallocation") { StringCopier str(&pool); str.startString(); - str.append("hello world!"); + str.append( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua."); - REQUIRE(str.isValid() == false); - REQUIRE(pool.overflowed() == true); - } - - SECTION("Increases size of memory pool") { - MemoryPool pool(addPadding(sizeofString(6))); - StringCopier str(&pool); - - str.startString(); - str.save(); - - REQUIRE(1 == pool.size()); + REQUIRE(str.isValid() == true); + REQUIRE(str.str() == + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua."); REQUIRE(pool.overflowed() == false); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(sizeofString(31)) + << AllocatorLog::Reallocate(sizeofString(31), + sizeofString(63)) + << AllocatorLog::Reallocate(sizeofString(63), + sizeofString(127))); } - SECTION("Works when memory pool is 0 bytes") { - MemoryPool pool(0); + SECTION("Realloc fails") { StringCopier str(&pool); str.startString(); + controllableAllocator.disable(); + str.append( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua."); + + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(sizeofString(31)) + << AllocatorLog::ReallocateFail(sizeofString(31), + sizeofString(63)) + << AllocatorLog::Deallocate(sizeofString(31))); REQUIRE(str.isValid() == false); REQUIRE(pool.overflowed() == true); } + + SECTION("Initial allocation fails") { + StringCopier str(&pool); + + controllableAllocator.disable(); + str.startString(); + + REQUIRE(str.isValid() == false); + REQUIRE(pool.overflowed() == true); + REQUIRE(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::AllocateFail(sizeofString(31))); + } } static const char* addStringToPool(MemoryPool& pool, const char* s) { diff --git a/extras/tests/MemoryPool/saveString.cpp b/extras/tests/MemoryPool/saveString.cpp index e7c704fa..5ee88617 100644 --- a/extras/tests/MemoryPool/saveString.cpp +++ b/extras/tests/MemoryPool/saveString.cpp @@ -39,13 +39,6 @@ TEST_CASE("MemoryPool::saveString()") { const char* a = saveString(pool, "hello\0world", 11); const char* b = saveString(pool, "hello\0world", 11); REQUIRE(a == b); - } - - SECTION("Reuse part of a string if it ends with NUL") { - const char* a = saveString(pool, "hello\0world", 11); - const char* b = saveString(pool, "hello"); - REQUIRE(a == b); - REQUIRE(pool.size() == 12); REQUIRE(pool.size() == sizeofString(11)); } @@ -56,52 +49,8 @@ TEST_CASE("MemoryPool::saveString()") { REQUIRE(pool.size() == sizeofString(5) + sizeofString(11)); } - SECTION("Returns NULL when full") { - REQUIRE(pool.capacity() == 32); - - const void* p1 = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - REQUIRE(p1 != 0); - REQUIRE(pool.size() == 32); - - const void* p2 = saveString(pool, "b"); - REQUIRE(p2 == 0); - } - - SECTION("Returns NULL when pool is too small") { - const void* p = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - REQUIRE(0 == p); - } - - SECTION("Returns NULL when buffer is NULL") { + SECTION("Returns NULL when allocation fails") { MemoryPool pool2(32, FailingAllocator::instance()); REQUIRE(0 == saveString(pool2, "a")); } - - SECTION("Returns NULL when capacity is 0") { - MemoryPool pool2(0); - REQUIRE(0 == saveString(pool2, "a")); - } - - SECTION("Returns same address after clear()") { - const void* a = saveString(pool, "hello"); - pool.clear(); - const void* b = saveString(pool, "world"); - - REQUIRE(a == b); - } - - SECTION("Can use full capacity when fresh") { - const void* a = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - - REQUIRE(a != 0); - } - - SECTION("Can use full capacity after clear") { - saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - pool.clear(); - - const void* a = saveString(pool, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); - - REQUIRE(a != 0); - } } diff --git a/extras/tests/Misc/printable.cpp b/extras/tests/Misc/printable.cpp index f39e2183..0adabbee 100644 --- a/extras/tests/Misc/printable.cpp +++ b/extras/tests/Misc/printable.cpp @@ -8,6 +8,8 @@ #define ARDUINOJSON_ENABLE_ARDUINO_STREAM 1 #include +#include "Allocators.hpp" + using ArduinoJson::detail::sizeofArray; using ArduinoJson::detail::sizeofString; @@ -52,7 +54,7 @@ struct PrintableString : public Printable { TEST_CASE("Printable") { SECTION("Doesn't overflow") { JsonDocument doc(8); - const char* value = "example"; // == 7 chars + const char* value = "example"; doc.set(666); // to make sure we override the value @@ -77,53 +79,82 @@ TEST_CASE("Printable") { } } - SECTION("Overflows early") { - JsonDocument doc(8); - const char* value = "hello world"; // > 8 chars + SECTION("First allocation fails") { + SpyingAllocator spyingAllocator(FailingAllocator::instance()); + JsonDocument doc(0, &spyingAllocator); + const char* value = "hello world"; doc.set(666); // to make sure we override the value SECTION("Via Print::write(char)") { PrintableString printable(value); - CHECK(doc.set(printable) == false); - CHECK(doc.isNull()); - CHECK(printable.totalBytesWritten() == 8); - CHECK(doc.overflowed() == true); - CHECK(doc.memoryUsage() == 0); - } - SECTION("Via Print::write(const char*, size_t)") { - PrintableString printable(value); - CHECK(doc.set(printable) == false); + bool success = doc.set(printable); + + CHECK(success == false); CHECK(doc.isNull()); CHECK(printable.totalBytesWritten() == 0); CHECK(doc.overflowed() == true); CHECK(doc.memoryUsage() == 0); + CHECK(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::AllocateFail(sizeofString(31))); + } + + SECTION("Via Print::write(const char*, size_t)") { + PrintableString printable(value); + + bool success = doc.set(printable); + + CHECK(success == false); + CHECK(doc.isNull()); + CHECK(printable.totalBytesWritten() == 0); + CHECK(doc.overflowed() == true); + CHECK(doc.memoryUsage() == 0); + CHECK(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::AllocateFail(sizeofString(31))); } } - SECTION("Overflows adding terminator") { - JsonDocument doc(8); - const char* value = "overflow"; // == 8 chars + SECTION("Reallocation fails") { + TimebombAllocator timebombAllocator(1); + SpyingAllocator spyingAllocator(&timebombAllocator); + JsonDocument doc(0, &spyingAllocator); + const char* value = "Lorem ipsum dolor sit amet, cons"; // > 31 chars doc.set(666); // to make sure we override the value SECTION("Via Print::write(char)") { PrintableString printable(value); - CHECK(doc.set(printable) == false); + + bool success = doc.set(printable); + + CHECK(success == false); CHECK(doc.isNull()); - CHECK(printable.totalBytesWritten() == 8); + CHECK(printable.totalBytesWritten() == 31); CHECK(doc.overflowed() == true); CHECK(doc.memoryUsage() == 0); + CHECK(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(sizeofString(31)) + << AllocatorLog::ReallocateFail(sizeofString(31), + sizeofString(63)) + << AllocatorLog::Deallocate(sizeofString(31))); } SECTION("Via Print::write(const char*, size_t)") { PrintableString printable(value); - CHECK(doc.set(printable) == false); + + bool success = doc.set(printable); + + CHECK(success == false); CHECK(doc.isNull()); - CHECK(printable.totalBytesWritten() == 0); + CHECK(printable.totalBytesWritten() == 31); CHECK(doc.overflowed() == true); CHECK(doc.memoryUsage() == 0); + CHECK(spyingAllocator.log() == + AllocatorLog() << AllocatorLog::Allocate(sizeofString(31)) + << AllocatorLog::ReallocateFail(sizeofString(31), + sizeofString(63)) + << AllocatorLog::Deallocate(sizeofString(31))); } } diff --git a/extras/tests/MsgPackDeserializer/deserializeVariant.cpp b/extras/tests/MsgPackDeserializer/deserializeVariant.cpp index 0e142b4d..0656f3c1 100644 --- a/extras/tests/MsgPackDeserializer/deserializeVariant.cpp +++ b/extras/tests/MsgPackDeserializer/deserializeVariant.cpp @@ -5,6 +5,8 @@ #include #include +#include "Allocators.hpp" + using ArduinoJson::detail::sizeofArray; using ArduinoJson::detail::sizeofObject; using ArduinoJson::detail::sizeofString; @@ -20,9 +22,10 @@ static void checkValue(const char* input, T expected) { REQUIRE(doc.as() == expected); } -static void checkError(size_t capacity, const char* input, - DeserializationError expected) { - JsonDocument doc(capacity); +static void checkError(size_t capacity, size_t timebombCountDown, + const char* input, DeserializationError expected) { + TimebombAllocator timebombAllocator(timebombCountDown); + JsonDocument doc(capacity, &timebombAllocator); DeserializationError error = deserializeMsgPack(doc, input); @@ -144,133 +147,120 @@ TEST_CASE("deserialize MsgPack value") { TEST_CASE("deserializeMsgPack() under memory constaints") { SECTION("single values always fit") { - checkError(0, "\xc0", DeserializationError::Ok); // nil - checkError(0, "\xc2", DeserializationError::Ok); // false - checkError(0, "\xc3", DeserializationError::Ok); // true - checkError(0, "\xcc\x00", DeserializationError::Ok); // uint 8 - checkError(0, "\xcd\x30\x39", DeserializationError::Ok); // uint 16 - checkError(0, "\xCE\x12\x34\x56\x78", DeserializationError::Ok); // uint 32 + checkError(0, 0, "\xc0", DeserializationError::Ok); // nil + checkError(0, 0, "\xc2", DeserializationError::Ok); // false + checkError(0, 0, "\xc3", DeserializationError::Ok); // true + checkError(0, 0, "\xcc\x00", DeserializationError::Ok); // uint 8 + checkError(0, 0, "\xcd\x30\x39", DeserializationError::Ok); // uint 16 + checkError(0, 0, "\xCE\x12\x34\x56\x78", + DeserializationError::Ok); // uint 32 } SECTION("fixstr") { - checkError(8, "\xA0", DeserializationError::Ok); - checkError(8, "\xA7ZZZZZZZ", DeserializationError::Ok); - checkError(8, "\xA8ZZZZZZZZ", DeserializationError::NoMemory); - checkError(16, "\xAFZZZZZZZZZZZZZZZ", DeserializationError::Ok); - checkError(16, "\xB0ZZZZZZZZZZZZZZZZ", DeserializationError::NoMemory); + checkError(0, 2, "\xA7ZZZZZZZ", DeserializationError::Ok); + checkError(0, 0, "\xA7ZZZZZZZ", DeserializationError::NoMemory); } SECTION("str 8") { - checkError(8, "\xD9\x00", DeserializationError::Ok); - checkError(8, "\xD9\x07ZZZZZZZ", DeserializationError::Ok); - checkError(8, "\xD9\x08ZZZZZZZZ", DeserializationError::NoMemory); - checkError(16, "\xD9\x0FZZZZZZZZZZZZZZZ", DeserializationError::Ok); - checkError(16, "\xD9\x10ZZZZZZZZZZZZZZZZ", DeserializationError::NoMemory); + checkError(0, 2, "\xD9\x07ZZZZZZZ", DeserializationError::Ok); + checkError(0, 0, "\xD9\x07ZZZZZZZ", DeserializationError::NoMemory); } SECTION("str 16") { - checkError(8, "\xDA\x00\x00", DeserializationError::Ok); - checkError(8, "\xDA\x00\x07ZZZZZZZ", DeserializationError::Ok); - checkError(8, "\xDA\x00\x08ZZZZZZZZ", DeserializationError::NoMemory); - checkError(16, "\xDA\x00\x0FZZZZZZZZZZZZZZZ", DeserializationError::Ok); - checkError(16, "\xDA\x00\x10ZZZZZZZZZZZZZZZZ", - DeserializationError::NoMemory); + checkError(0, 2, "\xDA\x00\x07ZZZZZZZ", DeserializationError::Ok); + checkError(0, 0, "\xDA\x00\x07ZZZZZZZ", DeserializationError::NoMemory); } SECTION("str 32") { - checkError(8, "\xDB\x00\x00\x00\x00", DeserializationError::Ok); - checkError(8, "\xDB\x00\x00\x00\x07ZZZZZZZ", DeserializationError::Ok); - checkError(8, "\xDB\x00\x00\x00\x08ZZZZZZZZ", - DeserializationError::NoMemory); - checkError(16, "\xDB\x00\x00\x00\x0FZZZZZZZZZZZZZZZ", - DeserializationError::Ok); - checkError(16, "\xDB\x00\x00\x00\x10ZZZZZZZZZZZZZZZZ", + checkError(0, 2, "\xDB\x00\x00\x00\x07ZZZZZZZ", DeserializationError::Ok); + checkError(0, 0, "\xDB\x00\x00\x00\x07ZZZZZZZ", DeserializationError::NoMemory); } SECTION("fixarray") { - checkError(sizeofArray(0), "\x90", DeserializationError::Ok); // [] - checkError(sizeofArray(0), "\x91\x01", + checkError(sizeofArray(0), 1, "\x90", DeserializationError::Ok); // [] + checkError(sizeofArray(0), 1, "\x91\x01", DeserializationError::NoMemory); // [1] - checkError(sizeofArray(1), "\x91\x01", + checkError(sizeofArray(1), 1, "\x91\x01", DeserializationError::Ok); // [1] - checkError(sizeofArray(1), "\x92\x01\x02", + checkError(sizeofArray(1), 1, "\x92\x01\x02", DeserializationError::NoMemory); // [1,2] } SECTION("array 16") { - checkError(sizeofArray(0), "\xDC\x00\x00", DeserializationError::Ok); - checkError(sizeofArray(0), "\xDC\x00\x01\x01", + checkError(sizeofArray(0), 1, "\xDC\x00\x00", DeserializationError::Ok); + checkError(sizeofArray(0), 1, "\xDC\x00\x01\x01", DeserializationError::NoMemory); - checkError(sizeofArray(1), "\xDC\x00\x01\x01", DeserializationError::Ok); - checkError(sizeofArray(1), "\xDC\x00\x02\x01\x02", + checkError(sizeofArray(1), 1, "\xDC\x00\x01\x01", DeserializationError::Ok); + checkError(sizeofArray(1), 1, "\xDC\x00\x02\x01\x02", DeserializationError::NoMemory); } SECTION("array 32") { - checkError(sizeofArray(0), "\xDD\x00\x00\x00\x00", + checkError(sizeofArray(0), 1, "\xDD\x00\x00\x00\x00", DeserializationError::Ok); - checkError(sizeofArray(0), "\xDD\x00\x00\x00\x01\x01", + checkError(sizeofArray(0), 1, "\xDD\x00\x00\x00\x01\x01", DeserializationError::NoMemory); - checkError(sizeofArray(1), "\xDD\x00\x00\x00\x01\x01", + checkError(sizeofArray(1), 1, "\xDD\x00\x00\x00\x01\x01", DeserializationError::Ok); - checkError(sizeofArray(1), "\xDD\x00\x00\x00\x02\x01\x02", + checkError(sizeofArray(1), 1, "\xDD\x00\x00\x00\x02\x01\x02", DeserializationError::NoMemory); } SECTION("fixmap") { SECTION("{}") { - checkError(sizeofObject(0), "\x80", DeserializationError::Ok); + checkError(sizeofObject(0), 0, "\x80", DeserializationError::Ok); } SECTION("{H:1}") { - checkError(sizeofObject(0), "\x81\xA1H\x01", + checkError(sizeofObject(0), 0, "\x81\xA1H\x01", DeserializationError::NoMemory); - checkError(sizeofObject(1) + sizeofString(2), "\x81\xA1H\x01", + checkError(sizeofObject(1) + sizeofString(2), 3, "\x81\xA1H\x01", DeserializationError::Ok); } SECTION("{H:1,W:2}") { - checkError(sizeofObject(1) + sizeofString(2), "\x82\xA1H\x01\xA1W\x02", + checkError(sizeofObject(1) + sizeofString(2), 3, "\x82\xA1H\x01\xA1W\x02", DeserializationError::NoMemory); - checkError(sizeofObject(2) + 2 * sizeofString(2), + checkError(sizeofObject(2) + 2 * sizeofString(2), 5, "\x82\xA1H\x01\xA1W\x02", DeserializationError::Ok); } } SECTION("map 16") { SECTION("{}") { - checkError(sizeofObject(0), "\xDE\x00\x00", DeserializationError::Ok); + checkError(sizeofObject(0), 0, "\xDE\x00\x00", DeserializationError::Ok); } SECTION("{H:1}") { - checkError(sizeofObject(0), "\xDE\x00\x01\xA1H\x01", + checkError(sizeofObject(1) + sizeofString(2), 1, "\xDE\x00\x01\xA1H\x01", DeserializationError::NoMemory); - checkError(sizeofObject(1) + sizeofString(2), "\xDE\x00\x01\xA1H\x01", + checkError(sizeofObject(1) + sizeofString(2), 3, "\xDE\x00\x01\xA1H\x01", DeserializationError::Ok); } SECTION("{H:1,W:2}") { - checkError(sizeofObject(1) + sizeofString(2), + checkError(sizeofObject(1) + sizeofString(2), 3, "\xDE\x00\x02\xA1H\x01\xA1W\x02", DeserializationError::NoMemory); - checkError(sizeofObject(2) + 2 * sizeofObject(1), + checkError(sizeofObject(2) + 2 * sizeofObject(1), 5, "\xDE\x00\x02\xA1H\x01\xA1W\x02", DeserializationError::Ok); } } SECTION("map 32") { SECTION("{}") { - checkError(sizeofObject(0), "\xDF\x00\x00\x00\x00", + checkError(sizeofObject(0), 0, "\xDF\x00\x00\x00\x00", DeserializationError::Ok); } SECTION("{H:1}") { - checkError(sizeofObject(0), "\xDF\x00\x00\x00\x01\xA1H\x01", + checkError(sizeofObject(1) + sizeofString(2), 1, + "\xDF\x00\x00\x00\x01\xA1H\x01", DeserializationError::NoMemory); - checkError(sizeofObject(1) + sizeofString(2), + checkError(sizeofObject(1) + sizeofString(2), 3, "\xDF\x00\x00\x00\x01\xA1H\x01", DeserializationError::Ok); } SECTION("{H:1,W:2}") { - checkError(sizeofObject(1) + sizeofString(2), + checkError(sizeofObject(1) + 2 * sizeofString(2), 3, "\xDF\x00\x00\x00\x02\xA1H\x01\xA1W\x02", DeserializationError::NoMemory); - checkError(sizeofObject(2) + 2 * sizeofObject(1), + checkError(sizeofObject(2) + 2 * sizeofObject(1), 5, "\xDF\x00\x00\x00\x02\xA1H\x01\xA1W\x02", DeserializationError::Ok); } diff --git a/idf_component.yml b/idf_component.yml index 9ce67309..b7402ab6 100644 --- a/idf_component.yml +++ b/idf_component.yml @@ -1,7 +1,7 @@ version: "7.0.0-alpha" description: >- A simple and efficient JSON library for embedded C++. - ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ fixed allocation, ✔ zero-copy, ✔ streams, ✔ filtering, and more. + ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ zero-copy, ✔ streams, ✔ filtering, and more. It is the most popular Arduino library on GitHub ❤❤❤❤❤. Check out arduinojson.org for a comprehensive documentation. url: https://arduinojson.org/ diff --git a/library.json b/library.json index 901c6844..842ab845 100644 --- a/library.json +++ b/library.json @@ -1,7 +1,7 @@ { "name": "ArduinoJson", "keywords": "json, rest, http, web", - "description": "A simple and efficient JSON library for embedded C++. ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ fixed allocation, ✔ zero-copy, ✔ streams, ✔ filtering, and more. It is the most popular Arduino library on GitHub ❤❤❤❤❤. Check out arduinojson.org for a comprehensive documentation.", + "description": "A simple and efficient JSON library for embedded C++. ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ zero-copy, ✔ streams, ✔ filtering, and more. It is the most popular Arduino library on GitHub ❤❤❤❤❤. Check out arduinojson.org for a comprehensive documentation.", "homepage": "https://arduinojson.org/?utm_source=meta&utm_medium=library.json", "repository": { "type": "git", diff --git a/library.properties b/library.properties index 17c2d543..26278595 100644 --- a/library.properties +++ b/library.properties @@ -3,7 +3,7 @@ version=7.0.0-alpha author=Benoit Blanchon maintainer=Benoit Blanchon sentence=A simple and efficient JSON library for embedded C++. -paragraph=ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ fixed allocation, ✔ zero-copy, ✔ streams, ✔ filtering, and more. It is the most popular Arduino library on GitHub ❤❤❤❤❤. Check out arduinojson.org for a comprehensive documentation. +paragraph=ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ zero-copy, ✔ streams, ✔ filtering, and more. It is the most popular Arduino library on GitHub ❤❤❤❤❤. Check out arduinojson.org for a comprehensive documentation. category=Data Processing url=https://arduinojson.org/?utm_source=meta&utm_medium=library.properties architectures=* diff --git a/src/ArduinoJson/Collection/CollectionData.hpp b/src/ArduinoJson/Collection/CollectionData.hpp index aa807b5d..2c5c8f41 100644 --- a/src/ArduinoJson/Collection/CollectionData.hpp +++ b/src/ArduinoJson/Collection/CollectionData.hpp @@ -70,7 +70,7 @@ class CollectionData { return _head; } - void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance); + void movePointers(ptrdiff_t variantDistance); private: VariantSlot* getSlot(size_t index) const; diff --git a/src/ArduinoJson/Collection/CollectionImpl.hpp b/src/ArduinoJson/Collection/CollectionImpl.hpp index c99b524b..6d708529 100644 --- a/src/ArduinoJson/Collection/CollectionImpl.hpp +++ b/src/ArduinoJson/Collection/CollectionImpl.hpp @@ -168,7 +168,7 @@ inline size_t CollectionData::memoryUsage() const { for (VariantSlot* s = _head; s; s = s->next()) { total += sizeof(VariantSlot) + s->data()->memoryUsage(); if (s->ownsKey()) - total += strlen(s->key()) + 1; + total += sizeofString(strlen(s->key())); } return total; } @@ -186,12 +186,11 @@ inline void movePointer(T*& p, ptrdiff_t offset) { ARDUINOJSON_ASSERT(isAligned(p)); } -inline void CollectionData::movePointers(ptrdiff_t stringDistance, - ptrdiff_t variantDistance) { +inline void CollectionData::movePointers(ptrdiff_t variantDistance) { movePointer(_head, variantDistance); movePointer(_tail, variantDistance); for (VariantSlot* slot = _head; slot; slot = slot->next()) - slot->movePointers(stringDistance, variantDistance); + slot->movePointers(variantDistance); } ARDUINOJSON_END_PRIVATE_NAMESPACE diff --git a/src/ArduinoJson/Json/JsonDeserializer.hpp b/src/ArduinoJson/Json/JsonDeserializer.hpp index 0696f0ec..4b8e83be 100644 --- a/src/ArduinoJson/Json/JsonDeserializer.hpp +++ b/src/ArduinoJson/Json/JsonDeserializer.hpp @@ -277,7 +277,6 @@ class JsonDeserializer { VariantData* variant = object.getMember(adaptString(key.c_str())); if (!variant) { // Save key in memory pool. - // This MUST be done before adding the slot. key = _stringStorage.save(); // Allocate slot in object diff --git a/src/ArduinoJson/Memory/MemoryPool.hpp b/src/ArduinoJson/Memory/MemoryPool.hpp index 110f6e02..b632d9a6 100644 --- a/src/ArduinoJson/Memory/MemoryPool.hpp +++ b/src/ArduinoJson/Memory/MemoryPool.hpp @@ -25,9 +25,15 @@ constexpr size_t sizeofObject(size_t n) { return n * sizeof(VariantSlot); } +struct StringNode { + struct StringNode* next; + uint16_t length; + char data[1]; +}; + // Returns the size (in bytes) of an string with n characters. constexpr size_t sizeofString(size_t n) { - return n + 1; + return n + 1 + offsetof(StringNode, data); } // _begin _end @@ -46,6 +52,7 @@ class MemoryPool { } ~MemoryPool() { + deallocAllStrings(); deallocPool(); } @@ -53,6 +60,7 @@ class MemoryPool { MemoryPool& operator=(const MemoryPool& src) = delete; MemoryPool& operator=(MemoryPool&& src) { + deallocAllStrings(); deallocPool(); _allocator = src._allocator; _begin = src._begin; @@ -61,6 +69,8 @@ class MemoryPool { _right = src._right; _overflowed = src._overflowed; src._begin = src._end = src._left = src._right = nullptr; + _strings = src._strings; + src._strings = nullptr; return *this; } @@ -87,7 +97,10 @@ class MemoryPool { } size_t size() const { - return size_t(_left - _begin + _end - _right); + size_t total = size_t(_left - _begin + _end - _right); + for (auto node = _strings; node; node = node->next) + total += sizeofString(node->length); + return total; } bool overflowed() const { @@ -103,45 +116,71 @@ class MemoryPool { if (str.isNull()) return 0; - const char* existingCopy = findString(str); - if (existingCopy) - return existingCopy; + auto node = findString(str); + if (node) { + return node->data; + } size_t n = str.size(); - char* newCopy = allocString(n + 1); - if (newCopy) { - stringGetChars(str, newCopy, n); - newCopy[n] = 0; // force null-terminator + node = allocString(n); + if (!node) + return nullptr; + + stringGetChars(str, node->data, n); + node->data[n] = 0; // force NUL terminator + addStringToList(node); + return node->data; + } + + void addStringToList(StringNode* node) { + ARDUINOJSON_ASSERT(node != nullptr); + node->next = _strings; + _strings = node; + } + + template + StringNode* findString(const TAdaptedString& str) const { + for (auto node = _strings; node; node = node->next) { + if (stringEquals(str, adaptString(node->data, node->length))) + return node; } - return newCopy; + return nullptr; } - void getFreeZone(char** zoneStart, size_t* zoneSize) const { - *zoneStart = _left; - *zoneSize = size_t(_right - _left); + StringNode* allocString(size_t length) { + auto node = reinterpret_cast( + _allocator->allocate(sizeofString(length))); + if (node) { + node->length = uint16_t(length); + } else { + _overflowed = true; + } + return node; } - const char* saveStringFromFreeZone(size_t len) { - const char* dup = findString(adaptString(_left, len)); - if (dup) - return dup; - - const char* str = _left; - _left += len; - *_left++ = 0; - checkInvariants(); - return str; + StringNode* reallocString(StringNode* node, size_t length) { + ARDUINOJSON_ASSERT(node != nullptr); + auto newNode = reinterpret_cast( + _allocator->reallocate(node, sizeofString(length))); + if (newNode) { + newNode->length = uint16_t(length); + } else { + _overflowed = true; + _allocator->deallocate(node); + } + return newNode; } - void markAsOverflowed() { - _overflowed = true; + void deallocString(StringNode* node) { + _allocator->deallocate(node); } void clear() { _left = _begin; _right = _end; _overflowed = false; + deallocAllStrings(); } bool canAlloc(size_t bytes) const { @@ -169,8 +208,8 @@ class MemoryPool { static_cast(new_ptr) - static_cast(old_ptr); movePointers(ptr_offset); - reinterpret_cast(variant).movePointers( - ptr_offset, ptr_offset - bytes_reclaimed); + reinterpret_cast(variant).movePointers(ptr_offset - + bytes_reclaimed); } private: @@ -215,29 +254,12 @@ class MemoryPool { ARDUINOJSON_ASSERT(isAligned(_right)); } - template - const char* findString(const TAdaptedString& str) const { - size_t n = str.size(); - for (char* next = _begin; next + n < _left; ++next) { - if (next[n] == '\0' && stringEquals(str, adaptString(next, n))) - return next; - - // jump to next terminator - while (*next) - ++next; + void deallocAllStrings() { + while (_strings) { + auto node = _strings; + _strings = node->next; + deallocString(node); } - return 0; - } - - char* allocString(size_t n) { - if (!canAlloc(n)) { - _overflowed = true; - return 0; - } - char* s = _left; - _left += n; - checkInvariants(); - return s; } template @@ -271,6 +293,7 @@ class MemoryPool { Allocator* _allocator; char *_begin, *_left, *_right, *_end; bool _overflowed; + StringNode* _strings = nullptr; }; template diff --git a/src/ArduinoJson/MsgPack/MsgPackDeserializer.hpp b/src/ArduinoJson/MsgPack/MsgPackDeserializer.hpp index 9fb5a4ac..089bf496 100644 --- a/src/ArduinoJson/MsgPack/MsgPackDeserializer.hpp +++ b/src/ArduinoJson/MsgPack/MsgPackDeserializer.hpp @@ -494,7 +494,6 @@ class MsgPackDeserializer { ARDUINOJSON_ASSERT(object != 0); // Save key in memory pool. - // This MUST be done before adding the slot. key = _stringStorage.save(); VariantSlot* slot = object->addSlot(_pool); diff --git a/src/ArduinoJson/StringStorage/StringCopier.hpp b/src/ArduinoJson/StringStorage/StringCopier.hpp index 0834a4f6..79c2bd39 100644 --- a/src/ArduinoJson/StringStorage/StringCopier.hpp +++ b/src/ArduinoJson/StringStorage/StringCopier.hpp @@ -10,20 +10,31 @@ ARDUINOJSON_BEGIN_PRIVATE_NAMESPACE class StringCopier { public: + static const size_t initialCapacity = 31; + StringCopier(MemoryPool* pool) : _pool(pool) {} + ~StringCopier() { + if (_node) + _pool->deallocString(_node); + } + void startString() { - _pool->getFreeZone(&_ptr, &_capacity); _size = 0; - if (_capacity == 0) - _pool->markAsOverflowed(); + if (!_node) + _node = _pool->allocString(initialCapacity); } JsonString save() { - ARDUINOJSON_ASSERT(_ptr); - ARDUINOJSON_ASSERT(_size < _capacity); // needs room for the terminator - return JsonString(_pool->saveStringFromFreeZone(_size), _size, - JsonString::Copied); + ARDUINOJSON_ASSERT(_node != nullptr); + _node->data[_size] = 0; + StringNode* node = _pool->findString(adaptString(_node->data, _size)); + if (!node) { + node = _pool->reallocString(_node, _size); + _pool->addStringToList(node); + _node = nullptr; // next time we need a new string + } + return JsonString(node->data, node->length, JsonString::Copied); } void append(const char* s) { @@ -32,19 +43,19 @@ class StringCopier { } void append(const char* s, size_t n) { - while (n-- > 0) + while (n-- > 0) // TODO: memcpy append(*s++); } void append(char c) { - if (_size + 1 < _capacity) - _ptr[_size++] = c; - else - _pool->markAsOverflowed(); + if (_node && _size == _node->length) + _node = _pool->reallocString(_node, _size * 2U + 1); + if (_node) + _node->data[_size++] = c; } bool isValid() const { - return !_pool->overflowed(); + return _node != nullptr; } size_t size() const { @@ -52,21 +63,15 @@ class StringCopier { } JsonString str() const { - ARDUINOJSON_ASSERT(_ptr); - ARDUINOJSON_ASSERT(_size < _capacity); - _ptr[_size] = 0; - return JsonString(_ptr, _size, JsonString::Copied); + ARDUINOJSON_ASSERT(_node != nullptr); + _node->data[_size] = 0; + return JsonString(_node->data, _size, JsonString::Copied); } private: MemoryPool* _pool; - - // These fields aren't initialized by the constructor but startString() - // - // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.UninitializedObject) - char* _ptr; - // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.UninitializedObject) - size_t _size, _capacity; + StringNode* _node = nullptr; + size_t _size = 0; }; ARDUINOJSON_END_PRIVATE_NAMESPACE diff --git a/src/ArduinoJson/Variant/ConverterImpl.hpp b/src/ArduinoJson/Variant/ConverterImpl.hpp index f5e1d046..87c17417 100644 --- a/src/ArduinoJson/Variant/ConverterImpl.hpp +++ b/src/ArduinoJson/Variant/ConverterImpl.hpp @@ -5,6 +5,7 @@ #pragma once #include +#include #include #include @@ -205,43 +206,35 @@ struct Converter : private detail::VariantAttorney { namespace detail { class MemoryPoolPrint : public Print { public: - MemoryPoolPrint(MemoryPool* pool) : _pool(pool), _size(0) { - pool->getFreeZone(&_string, &_capacity); + MemoryPoolPrint(MemoryPool* pool) : _copier(pool) { + _copier.startString(); } JsonString str() { - ARDUINOJSON_ASSERT(_size < _capacity); - return JsonString(_pool->saveStringFromFreeZone(_size), _size, - JsonString::Copied); + ARDUINOJSON_ASSERT(!overflowed()); + return _copier.save(); } size_t write(uint8_t c) { - if (_size >= _capacity) - return 0; - - _string[_size++] = char(c); - return 1; + _copier.append(char(c)); + return _copier.isValid() ? 1 : 0; } size_t write(const uint8_t* buffer, size_t size) { - if (_size + size >= _capacity) { - _size = _capacity; // mark as overflowed - return 0; + for (size_t i = 0; i < size; i++) { + _copier.append(char(buffer[i])); + if (!_copier.isValid()) + return i; } - memcpy(&_string[_size], buffer, size); - _size += size; return size; } bool overflowed() const { - return _size >= _capacity; + return !_copier.isValid(); } private: - MemoryPool* _pool; - size_t _size; - char* _string; - size_t _capacity; + StringCopier _copier; }; } // namespace detail @@ -253,7 +246,6 @@ inline void convertToJson(const ::Printable& src, JsonVariant dst) { detail::MemoryPoolPrint print(pool); src.printTo(print); if (print.overflowed()) { - pool->markAsOverflowed(); data->setNull(); return; } diff --git a/src/ArduinoJson/Variant/VariantData.hpp b/src/ArduinoJson/Variant/VariantData.hpp index 92d6fc64..a5c3a0d1 100644 --- a/src/ArduinoJson/Variant/VariantData.hpp +++ b/src/ArduinoJson/Variant/VariantData.hpp @@ -226,9 +226,7 @@ class VariantData { switch (type()) { case VALUE_IS_OWNED_STRING: case VALUE_IS_OWNED_RAW: - // We always add a zero at the end: the deduplication function uses it - // to detect the beginning of the next string. - return _content.asString.size + 1; + return sizeofString(_content.asString.size); case VALUE_IS_OBJECT: case VALUE_IS_ARRAY: return _content.asCollection.memoryUsage(); @@ -277,11 +275,9 @@ class VariantData { return _content.asCollection.getOrAddMember(key, pool); } - void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance) { - if (_flags & OWNED_VALUE_BIT) - _content.asString.data += stringDistance; + void movePointers(ptrdiff_t variantDistance) { if (_flags & COLLECTION_MASK) - _content.asCollection.movePointers(stringDistance, variantDistance); + _content.asCollection.movePointers(variantDistance); } uint8_t type() const { diff --git a/src/ArduinoJson/Variant/VariantSlot.hpp b/src/ArduinoJson/Variant/VariantSlot.hpp index ab5d763c..35885c08 100644 --- a/src/ArduinoJson/Variant/VariantSlot.hpp +++ b/src/ArduinoJson/Variant/VariantSlot.hpp @@ -99,13 +99,9 @@ class VariantSlot { _key = 0; } - void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance) { - if (_flags & OWNED_KEY_BIT) - _key += stringDistance; - if (_flags & OWNED_VALUE_BIT) - _content.asString.data += stringDistance; + void movePointers(ptrdiff_t variantDistance) { if (_flags & COLLECTION_MASK) - _content.asCollection.movePointers(stringDistance, variantDistance); + _content.asCollection.movePointers(variantDistance); } };