diff --git a/CHANGELOG.md b/CHANGELOG.md index a6637e6c..264a3c8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ HEAD ---- * Added comparisons (`>`, `>=`, `==`, `!=`, `<`, and `<=`) between `JsonVariant`s +* Added string deduplication (issue #1303) v6.15.2 (2020-05-15) ------- diff --git a/README.md b/README.md index b73a550c..c7671f70 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ ArduinoJson is a C++ JSON library for Arduino and IoT (Internet Of Things). * [Consumes roughly 10% less RAM than the "official" Arduino_JSON library](https://arduinojson.org/2019/11/19/arduinojson-vs-arduino_json/?utm_source=github&utm_medium=readme) * [Fixed memory allocation, no heap fragmentation](https://arduinojson.org/v6/api/jsondocument/?utm_source=github&utm_medium=readme) * [Optionally works without heap memory (zero malloc)](https://arduinojson.org/v6/api/staticjsondocument/?utm_source=github&utm_medium=readme) + * Deduplicates strings * Versatile * [Supports custom allocators (to use external RAM chip, for example)](https://arduinojson.org/v6/how-to/use-external-ram-on-esp32/?utm_source=github&utm_medium=readme) * Supports [Arduino's `String`](https://arduinojson.org/v6/api/config/enable_arduino_string/) and [STL's `std::string`](https://arduinojson.org/v6/api/config/enable_std_string/?utm_source=github&utm_medium=readme) diff --git a/extras/tests/Helpers/WString.h b/extras/tests/Helpers/WString.h index 18122f7e..5999ebf4 100644 --- a/extras/tests/Helpers/WString.h +++ b/extras/tests/Helpers/WString.h @@ -9,6 +9,9 @@ // Reproduces Arduino's String class class String { public: + String() {} + explicit String(const char* s) : _str(s) {} + String& operator+=(const char* rhs) { _str += rhs; return *this; diff --git a/extras/tests/JsonDeserializer/filter.cpp b/extras/tests/JsonDeserializer/filter.cpp index d719a6c7..20b52c73 100644 --- a/extras/tests/JsonDeserializer/filter.cpp +++ b/extras/tests/JsonDeserializer/filter.cpp @@ -239,7 +239,7 @@ TEST_CASE("Filtering") { 10, DeserializationError::Ok, "[{\"example\":1},{\"example\":3}]", - JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 16 + JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 8 }, { "[',2,3]", diff --git a/extras/tests/JsonDeserializer/string.cpp b/extras/tests/JsonDeserializer/string.cpp index 7b5fdbfb..70a26e07 100644 --- a/extras/tests/JsonDeserializer/string.cpp +++ b/extras/tests/JsonDeserializer/string.cpp @@ -74,16 +74,16 @@ TEST_CASE("Invalid JSON string") { } } -TEST_CASE("Not enough room to duplicate the string") { - DynamicJsonDocument doc(JSON_OBJECT_SIZE(0)); +TEST_CASE("Not enough room to save the key") { + DynamicJsonDocument doc(JSON_OBJECT_SIZE(1) + 8); SECTION("Quoted string") { - REQUIRE(deserializeJson(doc, "{\"example\":1}") == + REQUIRE(deserializeJson(doc, "{\"accuracy\":1}") == DeserializationError::NoMemory); } SECTION("Non-quoted string") { - REQUIRE(deserializeJson(doc, "{example:1}") == + REQUIRE(deserializeJson(doc, "{accuracy:1}") == DeserializationError::NoMemory); } } diff --git a/extras/tests/JsonDocument/StaticJsonDocument.cpp b/extras/tests/JsonDocument/StaticJsonDocument.cpp index dc6a1d41..e60aaf70 100644 --- a/extras/tests/JsonDocument/StaticJsonDocument.cpp +++ b/extras/tests/JsonDocument/StaticJsonDocument.cpp @@ -212,7 +212,7 @@ TEST_CASE("StaticJsonDocument") { SECTION("garbageCollect()") { StaticJsonDocument<256> doc; - doc[std::string("example")] = std::string("example"); + doc[std::string("example")] = std::string("jukebox"); doc.remove("example"); REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1) + 16); diff --git a/extras/tests/MemoryPool/CMakeLists.txt b/extras/tests/MemoryPool/CMakeLists.txt index b7d7432b..ec1b989b 100644 --- a/extras/tests/MemoryPool/CMakeLists.txt +++ b/extras/tests/MemoryPool/CMakeLists.txt @@ -4,8 +4,8 @@ add_executable(MemoryPoolTests allocVariant.cpp - allocString.cpp clear.cpp + saveString.cpp size.cpp StringCopier.cpp ) diff --git a/extras/tests/MemoryPool/StringCopier.cpp b/extras/tests/MemoryPool/StringCopier.cpp index b2577b78..236c81ae 100644 --- a/extras/tests/MemoryPool/StringCopier.cpp +++ b/extras/tests/MemoryPool/StringCopier.cpp @@ -38,8 +38,47 @@ TEST_CASE("StringCopier") { str.startString(&pool); str.append('h'); - str.commit(&pool); + str.save(&pool); REQUIRE(1 == pool.size()); } } + +static const char* addStringToPool(MemoryPool* pool, const char* s) { + StringCopier str; + str.startString(pool); + str.append(s); + str.append('\0'); + return str.save(pool); +} + +TEST_CASE("StringCopier::save() deduplicates strings") { + char buffer[4096]; + MemoryPool pool(buffer, 4096); + + SECTION("Basic") { + const char* s1 = addStringToPool(&pool, "hello"); + const char* s2 = addStringToPool(&pool, "world"); + const char* s3 = addStringToPool(&pool, "hello"); + + REQUIRE(s1 == s3); + REQUIRE(s2 != s3); + REQUIRE(pool.size() == 12); + } + + SECTION("Requires terminator") { + const char* s1 = addStringToPool(&pool, "hello world"); + const char* s2 = addStringToPool(&pool, "hello"); + + REQUIRE(s2 != s1); + REQUIRE(pool.size() == 12 + 6); + } + + SECTION("Don't overrun") { + const char* s1 = addStringToPool(&pool, "hello world"); + const char* s2 = addStringToPool(&pool, "wor"); + + REQUIRE(s2 != s1); + REQUIRE(pool.size() == 12 + 4); + } +} diff --git a/extras/tests/MemoryPool/allocString.cpp b/extras/tests/MemoryPool/allocString.cpp deleted file mode 100644 index c93f4564..00000000 --- a/extras/tests/MemoryPool/allocString.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// ArduinoJson - arduinojson.org -// Copyright Benoit Blanchon 2014-2020 -// MIT License - -#include -#include - -using namespace ARDUINOJSON_NAMESPACE; - -TEST_CASE("MemoryPool::allocFrozenString()") { - const size_t poolCapacity = 64; - const size_t longestString = poolCapacity; - char buffer[poolCapacity]; - MemoryPool pool(buffer, poolCapacity); - - SECTION("Returns different addresses") { - char *a = pool.allocFrozenString(1); - char *b = pool.allocFrozenString(1); - REQUIRE(a != b); - } - - SECTION("Returns NULL when full") { - void *p1 = pool.allocFrozenString(longestString); - REQUIRE(p1 != 0); - - void *p2 = pool.allocFrozenString(1); - REQUIRE(p2 == 0); - } - - SECTION("Returns NULL when pool is too small") { - void *p = pool.allocFrozenString(longestString + 1); - REQUIRE(0 == p); - } - - SECTION("Returns NULL when buffer is NULL") { - MemoryPool pool2(0, poolCapacity); - REQUIRE(0 == pool2.allocFrozenString(2)); - } - - SECTION("Returns NULL when capacity is 0") { - MemoryPool pool2(buffer, 0); - REQUIRE(0 == pool2.allocFrozenString(2)); - } - - SECTION("Returns same address after clear()") { - void *a = pool.allocFrozenString(1); - pool.clear(); - void *b = pool.allocFrozenString(1); - - REQUIRE(a == b); - } - - SECTION("Can use full capacity when fresh") { - void *a = pool.allocFrozenString(longestString); - - REQUIRE(a != 0); - } - - SECTION("Can use full capacity after clear") { - pool.allocFrozenString(longestString); - pool.clear(); - - void *a = pool.allocFrozenString(longestString); - - REQUIRE(a != 0); - } -} diff --git a/extras/tests/MemoryPool/clear.cpp b/extras/tests/MemoryPool/clear.cpp index 2eb3b487..52666a6b 100644 --- a/extras/tests/MemoryPool/clear.cpp +++ b/extras/tests/MemoryPool/clear.cpp @@ -3,6 +3,7 @@ // MIT License #include +#include #include using namespace ARDUINOJSON_NAMESPACE; @@ -21,8 +22,8 @@ TEST_CASE("MemoryPool::clear()") { } SECTION("Discards allocated strings") { - pool.allocFrozenString(10); - REQUIRE(pool.size() > 0); + pool.saveString(adaptString(const_cast("123456789"))); + REQUIRE(pool.size() == 10); pool.clear(); diff --git a/extras/tests/MemoryPool/saveString.cpp b/extras/tests/MemoryPool/saveString.cpp new file mode 100644 index 00000000..56836fa2 --- /dev/null +++ b/extras/tests/MemoryPool/saveString.cpp @@ -0,0 +1,81 @@ +// ArduinoJson - arduinojson.org +// Copyright Benoit Blanchon 2014-2020 +// MIT License + +#include +#include +#include + +using namespace ARDUINOJSON_NAMESPACE; + +static const char *saveString(MemoryPool &pool, const char *s) { + return pool.saveString(adaptString(const_cast(s))); +} + +TEST_CASE("MemoryPool::saveString()") { + char buffer[32]; + MemoryPool pool(buffer, 32); + + SECTION("Duplicates different strings") { + const char *a = saveString(pool, "hello"); + const char *b = saveString(pool, "world"); + REQUIRE(a != b); + REQUIRE(pool.size() == 6 + 6); + } + + SECTION("Deduplicates identical strings") { + const char *a = saveString(pool, "hello"); + const char *b = saveString(pool, "hello"); + REQUIRE(a == b); + REQUIRE(pool.size() == 6); + } + + SECTION("Returns NULL when full") { + REQUIRE(pool.capacity() == 32); + + const void *p1 = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + REQUIRE(p1 != 0); + REQUIRE(pool.size() == 32); + + const void *p2 = saveString(pool, "b"); + REQUIRE(p2 == 0); + } + + SECTION("Returns NULL when pool is too small") { + const void *p = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + REQUIRE(0 == p); + } + + SECTION("Returns NULL when buffer is NULL") { + MemoryPool pool2(0, 32); + REQUIRE(0 == saveString(pool2, "a")); + } + + SECTION("Returns NULL when capacity is 0") { + MemoryPool pool2(buffer, 0); + REQUIRE(0 == saveString(pool2, "a")); + } + + SECTION("Returns same address after clear()") { + const void *a = saveString(pool, "hello"); + pool.clear(); + const void *b = saveString(pool, "world"); + + REQUIRE(a == b); + } + + SECTION("Can use full capacity when fresh") { + const void *a = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + + REQUIRE(a != 0); + } + + SECTION("Can use full capacity after clear") { + saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + pool.clear(); + + const void *a = saveString(pool, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + + REQUIRE(a != 0); + } +} diff --git a/extras/tests/MemoryPool/size.cpp b/extras/tests/MemoryPool/size.cpp index b137cc87..0e141df6 100644 --- a/extras/tests/MemoryPool/size.cpp +++ b/extras/tests/MemoryPool/size.cpp @@ -22,24 +22,6 @@ TEST_CASE("MemoryPool::size()") { REQUIRE(0 == pool.size()); } - SECTION("Decreases after freezeString()") { - StringSlot a = pool.allocExpandableString(); - pool.freezeString(a, 1); - REQUIRE(pool.size() == 1); - - StringSlot b = pool.allocExpandableString(); - pool.freezeString(b, 1); - REQUIRE(pool.size() == 2); - } - - SECTION("Increases after allocFrozenString()") { - pool.allocFrozenString(1); - REQUIRE(pool.size() == 1); - - pool.allocFrozenString(2); - REQUIRE(pool.size() == 3); - } - SECTION("Doesn't grow when memory pool is full") { const size_t variantCount = sizeof(buffer) / sizeof(VariantSlot); diff --git a/extras/tests/Misc/StringAdapters.cpp b/extras/tests/Misc/StringAdapters.cpp index c299bef9..4b19b9bb 100644 --- a/extras/tests/Misc/StringAdapters.cpp +++ b/extras/tests/Misc/StringAdapters.cpp @@ -6,6 +6,7 @@ #include "progmem_emulation.hpp" #include "weird_strcmp.hpp" +#include #include #include #include @@ -114,6 +115,21 @@ TEST_CASE("std::string") { CHECK(adapter.size() == 5); } +TEST_CASE("Arduino String") { + ::String str("bravo"); + ArduinoStringAdapter adapter = adaptString(str); + + CHECK(adapter.compare(NULL) > 0); + CHECK(adapter.compare("alpha") > 0); + CHECK(adapter.compare("bravo") == 0); + CHECK(adapter.compare("charlie") < 0); + + CHECK(adapter.equals("bravo")); + CHECK_FALSE(adapter.equals("charlie")); + + CHECK(adapter.size() == 5); +} + TEST_CASE("custom_string") { custom_string str("bravo"); StlStringAdapter adapter = adaptString(str); diff --git a/extras/tests/MixedConfiguration/CMakeLists.txt b/extras/tests/MixedConfiguration/CMakeLists.txt index 650b244c..81227637 100644 --- a/extras/tests/MixedConfiguration/CMakeLists.txt +++ b/extras/tests/MixedConfiguration/CMakeLists.txt @@ -18,6 +18,8 @@ add_executable(MixedConfigurationTests enable_nan_0.cpp enable_nan_1.cpp enable_progmem_1.cpp + enable_string_deduplication_0.cpp + enable_string_deduplication_1.cpp use_double_0.cpp use_double_1.cpp use_long_long_0.cpp diff --git a/extras/tests/MixedConfiguration/enable_string_deduplication_0.cpp b/extras/tests/MixedConfiguration/enable_string_deduplication_0.cpp new file mode 100644 index 00000000..d040dbab --- /dev/null +++ b/extras/tests/MixedConfiguration/enable_string_deduplication_0.cpp @@ -0,0 +1,125 @@ +// ArduinoJson - arduinojson.org +// Copyright Benoit Blanchon 2014-2020 +// MIT License + +#include "progmem_emulation.hpp" + +#define ARDUINOJSON_ENABLE_ARDUINO_STRING 1 +#define ARDUINOJSON_ENABLE_PROGMEM 1 +#define ARDUINOJSON_ENABLE_STRING_DEDUPLICATION 0 +#include + +#include + +TEST_CASE("ARDUINOJSON_ENABLE_STRING_DEDUPLICATION = 0") { + StaticJsonDocument<1024> doc; + + SECTION("deserializeJson()") { + SECTION("Deduplicate values") { + deserializeJson(doc, "[\"example\",\"example\"]"); + + CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 16); + CHECK(doc[0].as() != doc[1].as()); + } + + SECTION("Deduplicate keys") { + deserializeJson(doc, "[{\"example\":1},{\"example\":2}]"); + + CHECK(doc.memoryUsage() == + 2 * JSON_OBJECT_SIZE(1) + JSON_ARRAY_SIZE(2) + 16); + + const char* key1 = doc[0].as().begin()->key().c_str(); + const char* key2 = doc[1].as().begin()->key().c_str(); + + CHECK(key1 != key2); + } + } + + SECTION("JsonDocument") { + SECTION("values") { + SECTION("std::string") { + doc.add(std::string("example")); + doc.add(std::string("example")); + + CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 16); + CHECK(doc[0].as() != doc[1].as()); + } + + SECTION("char*") { + char value[] = "example"; + doc.add(value); + doc.add(value); + + CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 16); + CHECK(doc[0].as() != doc[1].as()); + } + + SECTION("Arduino String") { + doc.add(String("example")); + doc.add(String("example")); + + CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 16); + CHECK(doc[0].as() != doc[1].as()); + } + + SECTION("Flash string") { + doc.add(F("example")); + doc.add(F("example")); + + CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 16); + CHECK(doc[0].as() != doc[1].as()); + } + } + + SECTION("keys") { + SECTION("std::string") { + doc[0][std::string("example")] = 1; + doc[1][std::string("example")] = 2; + + CHECK(doc.memoryUsage() == + JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 16); + + const char* key1 = doc[0].as().begin()->key().c_str(); + const char* key2 = doc[1].as().begin()->key().c_str(); + CHECK(key1 != key2); + } + + SECTION("char*") { + char key[] = "example"; + doc[0][key] = 1; + doc[1][key] = 2; + + CHECK(doc.memoryUsage() == + JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 16); + + const char* key1 = doc[0].as().begin()->key().c_str(); + const char* key2 = doc[1].as().begin()->key().c_str(); + CHECK(key1 != key2); + } + + SECTION("Arduino String") { + doc[0][String("example")] = 1; + doc[1][String("example")] = 2; + + CHECK(doc.memoryUsage() == + JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 16); + + const char* key1 = doc[0].as().begin()->key().c_str(); + const char* key2 = doc[1].as().begin()->key().c_str(); + CHECK(key1 != key2); + } + + SECTION("Flash string") { + doc[0][F("example")] = 1; + doc[1][F("example")] = 2; + + CHECK(doc.memoryUsage() == + JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 16); + + const char* key1 = doc[0].as().begin()->key().c_str(); + const char* key2 = doc[1].as().begin()->key().c_str(); + CHECK(key1 != key2); + } + } + } +} diff --git a/extras/tests/MixedConfiguration/enable_string_deduplication_1.cpp b/extras/tests/MixedConfiguration/enable_string_deduplication_1.cpp new file mode 100644 index 00000000..6971cb6a --- /dev/null +++ b/extras/tests/MixedConfiguration/enable_string_deduplication_1.cpp @@ -0,0 +1,124 @@ +// ArduinoJson - arduinojson.org +// Copyright Benoit Blanchon 2014-2020 +// MIT License + +#include "progmem_emulation.hpp" + +#define ARDUINOJSON_ENABLE_ARDUINO_STRING 1 +#define ARDUINOJSON_ENABLE_PROGMEM 1 +#define ARDUINOJSON_ENABLE_STRING_DEDUPLICATION 1 +#include + +#include + +TEST_CASE("ARDUINOJSON_ENABLE_STRING_DEDUPLICATION = 1") { + StaticJsonDocument<1024> doc; + + SECTION("deserializeJson()") { + SECTION("Deduplicate values") { + deserializeJson(doc, "[\"example\",\"example\"]"); + + CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 8); + CHECK(doc[0].as() == doc[1].as()); + } + + SECTION("Deduplicate keys") { + deserializeJson(doc, "[{\"example\":1},{\"example\":2}]"); + + CHECK(doc.memoryUsage() == + 2 * JSON_OBJECT_SIZE(1) + JSON_ARRAY_SIZE(2) + 8); + + const char* key1 = doc[0].as().begin()->key().c_str(); + const char* key2 = doc[1].as().begin()->key().c_str(); + CHECK(key1 == key2); + } + } + + SECTION("JsonDocument") { + SECTION("values") { + SECTION("std::string") { + doc.add(std::string("example")); + doc.add(std::string("example")); + + CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 8); + CHECK(doc[0].as() == doc[1].as()); + } + + SECTION("char*") { + char value[] = "example"; + doc.add(value); + doc.add(value); + + CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 8); + CHECK(doc[0].as() == doc[1].as()); + } + + SECTION("Arduino String") { + doc.add(String("example")); + doc.add(String("example")); + + CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 8); + CHECK(doc[0].as() == doc[1].as()); + } + + SECTION("Flash string") { + doc.add(F("example")); + doc.add(F("example")); + + CHECK(doc.memoryUsage() == JSON_ARRAY_SIZE(2) + 8); + CHECK(doc[0].as() == doc[1].as()); + } + } + + SECTION("keys") { + SECTION("std::string") { + doc[0][std::string("example")] = 1; + doc[1][std::string("example")] = 2; + + CHECK(doc.memoryUsage() == + JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 8); + + const char* key1 = doc[0].as().begin()->key().c_str(); + const char* key2 = doc[1].as().begin()->key().c_str(); + CHECK(key1 == key2); + } + + SECTION("char*") { + char key[] = "example"; + doc[0][key] = 1; + doc[1][key] = 2; + + CHECK(doc.memoryUsage() == + JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 8); + + const char* key1 = doc[0].as().begin()->key().c_str(); + const char* key2 = doc[1].as().begin()->key().c_str(); + CHECK(key1 == key2); + } + + SECTION("Arduino String") { + doc[0][String("example")] = 1; + doc[1][String("example")] = 2; + + CHECK(doc.memoryUsage() == + JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 8); + + const char* key1 = doc[0].as().begin()->key().c_str(); + const char* key2 = doc[1].as().begin()->key().c_str(); + CHECK(key1 == key2); + } + + SECTION("Flash string") { + doc[0][F("example")] = 1; + doc[1][F("example")] = 2; + + CHECK(doc.memoryUsage() == + JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 8); + + const char* key1 = doc[0].as().begin()->key().c_str(); + const char* key2 = doc[1].as().begin()->key().c_str(); + CHECK(key1 == key2); + } + } + } +} diff --git a/src/ArduinoJson/Configuration.hpp b/src/ArduinoJson/Configuration.hpp index a73a2c5d..71358d92 100644 --- a/src/ArduinoJson/Configuration.hpp +++ b/src/ArduinoJson/Configuration.hpp @@ -215,6 +215,10 @@ #define ARDUINOJSON_TAB " " #endif +#ifndef ARDUINOJSON_ENABLE_STRING_DEDUPLICATION +#define ARDUINOJSON_ENABLE_STRING_DEDUPLICATION 1 +#endif + #ifndef ARDUINOJSON_STRING_BUFFER_SIZE #define ARDUINOJSON_STRING_BUFFER_SIZE 32 #endif diff --git a/src/ArduinoJson/Json/JsonDeserializer.hpp b/src/ArduinoJson/Json/JsonDeserializer.hpp index 3634fb0c..e6a1d2c5 100644 --- a/src/ArduinoJson/Json/JsonDeserializer.hpp +++ b/src/ArduinoJson/Json/JsonDeserializer.hpp @@ -212,8 +212,6 @@ class JsonDeserializer { // Read each key value pair for (;;) { - _stringStorage.startString(_pool); - // Parse key err = parseKey(); if (err) @@ -233,7 +231,9 @@ class JsonDeserializer { if (memberFilter.allow()) { VariantData *variant = object.getMember(adaptString(key)); if (!variant) { - _stringStorage.commit(_pool); + // Save key in memory pool. + // This MUST be done before adding the slot. + key = _stringStorage.save(_pool); // Allocate slot in object VariantSlot *slot = object.addSlot(_pool); @@ -325,6 +325,7 @@ class JsonDeserializer { } DeserializationError parseKey() { + _stringStorage.startString(_pool); if (isQuote(current())) { return parseQuotedString(); } else { @@ -337,8 +338,8 @@ class JsonDeserializer { DeserializationError err = parseQuotedString(); if (err) return err; - _stringStorage.commit(_pool); - variant.setOwnedString(make_not_null(_stringStorage.c_str())); + const char *value = _stringStorage.save(_pool); + variant.setOwnedString(make_not_null(value)); return DeserializationError::Ok; } diff --git a/src/ArduinoJson/Memory/MemoryPool.hpp b/src/ArduinoJson/Memory/MemoryPool.hpp index 089a9ca9..f97ddf1f 100644 --- a/src/ArduinoJson/Memory/MemoryPool.hpp +++ b/src/ArduinoJson/Memory/MemoryPool.hpp @@ -51,40 +51,43 @@ class MemoryPool { return allocRight(); } - char* allocFrozenString(size_t n) { - if (!canAlloc(n)) - return 0; - char* s = _left; - _left += n; - checkInvariants(); - return s; - } - template - char* saveString(const TAdaptedString& str) { + const char* saveString(const TAdaptedString& str) { if (str.isNull()) return 0; + +#if ARDUINOJSON_ENABLE_STRING_DEDUPLICATION + const char* existingCopy = findString(str.begin()); + if (existingCopy) + return existingCopy; +#endif + size_t n = str.size(); - char* dup = allocFrozenString(n + 1); - if (dup) { - str.copyTo(dup, n); - dup[n] = 0; // force null-terminator + + char* newCopy = allocString(n + 1); + if (newCopy) { + str.copyTo(newCopy, n); + newCopy[n] = 0; // force null-terminator } - return dup; + return newCopy; } - StringSlot allocExpandableString() { - StringSlot s; - s.value = _left; - s.size = size_t(_right - _left); - checkInvariants(); - return s; + void getFreeZone(char** zoneStart, size_t* zoneSize) const { + *zoneStart = _left; + *zoneSize = size_t(_right - _left); } - void freezeString(StringSlot& s, size_t newSize) { - _left = (s.value + newSize); - s.size = newSize; + const char* saveStringFromFreeZone(size_t len) { +#if ARDUINOJSON_ENABLE_STRING_DEDUPLICATION + const char* dup = findString(_left); + if (dup) + return dup; +#endif + + const char* str = _left; + _left += len; checkInvariants(); + return str; } void clear() { @@ -100,18 +103,6 @@ class MemoryPool { return _begin <= p && p < _end; } - template - T* allocRight() { - return reinterpret_cast(allocRight(sizeof(T))); - } - - void* allocRight(size_t bytes) { - if (!canAlloc(bytes)) - return 0; - _right -= bytes; - return _right; - } - // Workaround for missing placement new void* operator new(size_t, void* p) { return p; @@ -163,6 +154,46 @@ class MemoryPool { ARDUINOJSON_ASSERT(isAligned(_right)); } +#if ARDUINOJSON_ENABLE_STRING_DEDUPLICATION + template + const char* findString(TIterator str) { + for (char* next = _begin; next < _left; ++next) { + char* begin = next; + + // try to match + for (TIterator it = str; *it == *next; ++it) { + if (*next++ == 0) + return begin; + } + + // jump to next terminator + while (*next) ++next; + } + return 0; + } +#endif + + char* allocString(size_t n) { + if (!canAlloc(n)) + return 0; + char* s = _left; + _left += n; + checkInvariants(); + return s; + } + + template + T* allocRight() { + return reinterpret_cast(allocRight(sizeof(T))); + } + + void* allocRight(size_t bytes) { + if (!canAlloc(bytes)) + return 0; + _right -= bytes; + return _right; + } + char *_begin, *_left, *_right, *_end; }; diff --git a/src/ArduinoJson/MsgPack/MsgPackDeserializer.hpp b/src/ArduinoJson/MsgPack/MsgPackDeserializer.hpp index 34308166..c18bd17c 100644 --- a/src/ArduinoJson/MsgPack/MsgPackDeserializer.hpp +++ b/src/ArduinoJson/MsgPack/MsgPackDeserializer.hpp @@ -248,8 +248,8 @@ class MsgPackDeserializer { _stringStorage.append('\0'); if (!_stringStorage.isValid()) return DeserializationError::NoMemory; - _stringStorage.commit(_pool); - result = _stringStorage.c_str(); + + result = _stringStorage.save(_pool); return DeserializationError::Ok; } diff --git a/src/ArduinoJson/Namespace.hpp b/src/ArduinoJson/Namespace.hpp index 92cc5888..39a5103a 100644 --- a/src/ArduinoJson/Namespace.hpp +++ b/src/ArduinoJson/Namespace.hpp @@ -16,16 +16,17 @@ #define ARDUINOJSON_CONCAT8(A, B, C, D, E, F, G, H) \ ARDUINOJSON_CONCAT2(ARDUINOJSON_CONCAT4(A, B, C, D), \ ARDUINOJSON_CONCAT4(E, F, G, H)) -#define ARDUINOJSON_CONCAT12(A, B, C, D, E, F, G, H, I, J, K, L) \ - ARDUINOJSON_CONCAT8(A, B, C, D, E, F, G, \ - ARDUINOJSON_CONCAT4(H, I, J, ARDUINOJSON_CONCAT2(K, L))) +#define ARDUINOJSON_CONCAT13(A, B, C, D, E, F, G, H, I, J, K, L, M) \ + ARDUINOJSON_CONCAT8(A, B, C, D, E, ARDUINOJSON_CONCAT4(F, G, H, I), \ + ARDUINOJSON_CONCAT2(J, K), ARDUINOJSON_CONCAT2(L, M)) #define ARDUINOJSON_NAMESPACE \ - ARDUINOJSON_CONCAT12( \ + ARDUINOJSON_CONCAT13( \ ArduinoJson, ARDUINOJSON_VERSION_MAJOR, ARDUINOJSON_VERSION_MINOR, \ ARDUINOJSON_VERSION_REVISION, _, ARDUINOJSON_USE_LONG_LONG, \ ARDUINOJSON_USE_DOUBLE, ARDUINOJSON_DECODE_UNICODE, \ ARDUINOJSON_ENABLE_NAN, ARDUINOJSON_ENABLE_INFINITY, \ - ARDUINOJSON_ENABLE_PROGMEM, ARDUINOJSON_ENABLE_COMMENTS) + ARDUINOJSON_ENABLE_PROGMEM, ARDUINOJSON_ENABLE_COMMENTS, \ + ARDUINOJSON_ENABLE_STRING_DEDUPLICATION) #endif diff --git a/src/ArduinoJson/StringStorage/StringCopier.hpp b/src/ArduinoJson/StringStorage/StringCopier.hpp index 6cde7880..65828de9 100644 --- a/src/ArduinoJson/StringStorage/StringCopier.hpp +++ b/src/ArduinoJson/StringStorage/StringCopier.hpp @@ -11,13 +11,13 @@ namespace ARDUINOJSON_NAMESPACE { class StringCopier { public: void startString(MemoryPool* pool) { - _slot = pool->allocExpandableString(); + pool->getFreeZone(&_ptr, &_capacity); _size = 0; } - void commit(MemoryPool* pool) { - ARDUINOJSON_ASSERT(_slot.value); - pool->freezeString(_slot, _size); + const char* save(MemoryPool* pool) { + ARDUINOJSON_ASSERT(_ptr); + return pool->saveStringFromFreeZone(_size); } void append(const char* s) { @@ -29,27 +29,28 @@ class StringCopier { } void append(char c) { - if (!_slot.value) + if (!_ptr) return; - if (_size >= _slot.size) { - _slot.value = 0; + if (_size >= _capacity) { + _ptr = 0; return; } - _slot.value[_size++] = c; + _ptr[_size++] = c; } bool isValid() { - return _slot.value != 0; + return _ptr != 0; } const char* c_str() { - return _slot.value; + return _ptr; } private: + char* _ptr; size_t _size; - StringSlot _slot; + size_t _capacity; }; } // namespace ARDUINOJSON_NAMESPACE diff --git a/src/ArduinoJson/StringStorage/StringMover.hpp b/src/ArduinoJson/StringStorage/StringMover.hpp index 539357f7..5fc4a595 100644 --- a/src/ArduinoJson/StringStorage/StringMover.hpp +++ b/src/ArduinoJson/StringStorage/StringMover.hpp @@ -16,7 +16,9 @@ class StringMover { _startPtr = _writePtr; } - void commit(MemoryPool*) const {} + const char* save(MemoryPool*) const { + return _startPtr; + } void append(char c) { *_writePtr++ = c; diff --git a/src/ArduinoJson/Strings/ArduinoStringAdapter.hpp b/src/ArduinoJson/Strings/ArduinoStringAdapter.hpp index ce228ca7..bf3202c1 100644 --- a/src/ArduinoJson/Strings/ArduinoStringAdapter.hpp +++ b/src/ArduinoJson/Strings/ArduinoStringAdapter.hpp @@ -39,6 +39,10 @@ class ArduinoStringAdapter { return _str->length(); } + const char* begin() const { + return _str->c_str(); + } + typedef storage_policies::store_by_copy storage_policy; private: diff --git a/src/ArduinoJson/Strings/ConstRamStringAdapter.hpp b/src/ArduinoJson/Strings/ConstRamStringAdapter.hpp index 1edba1f5..2d27e864 100644 --- a/src/ArduinoJson/Strings/ConstRamStringAdapter.hpp +++ b/src/ArduinoJson/Strings/ConstRamStringAdapter.hpp @@ -39,6 +39,10 @@ class ConstRamStringAdapter { return _str; } + const char* begin() const { + return _str; + } + typedef storage_policies::store_by_address storage_policy; protected: diff --git a/src/ArduinoJson/Strings/FlashStringAdapter.hpp b/src/ArduinoJson/Strings/FlashStringAdapter.hpp index 1c280b95..2d6f95fc 100644 --- a/src/ArduinoJson/Strings/FlashStringAdapter.hpp +++ b/src/ArduinoJson/Strings/FlashStringAdapter.hpp @@ -5,6 +5,7 @@ #pragma once #include +#include #include #include @@ -42,6 +43,10 @@ class FlashStringAdapter { return strlen_P(reinterpret_cast(_str)); } + FlashStringIterator begin() const { + return FlashStringIterator(_str); + } + typedef storage_policies::store_by_copy storage_policy; private: diff --git a/src/ArduinoJson/Strings/FlashStringIterator.hpp b/src/ArduinoJson/Strings/FlashStringIterator.hpp new file mode 100644 index 00000000..299405fb --- /dev/null +++ b/src/ArduinoJson/Strings/FlashStringIterator.hpp @@ -0,0 +1,44 @@ +// ArduinoJson - arduinojson.org +// Copyright Benoit Blanchon 2014-2020 +// MIT License + +#pragma once + +namespace ARDUINOJSON_NAMESPACE { + +class FlashStringIterator { + public: + explicit FlashStringIterator(const __FlashStringHelper* ptr) + : _ptr(reinterpret_cast(ptr)) {} + + explicit FlashStringIterator(const char* ptr) : _ptr(ptr) {} + + FlashStringIterator operator+(ptrdiff_t d) const { + return FlashStringIterator(_ptr + d); + } + + ptrdiff_t operator-(FlashStringIterator other) const { + return _ptr - other._ptr; + } + + FlashStringIterator operator++(int) { + return FlashStringIterator(_ptr++); + } + + FlashStringIterator operator++() { + return FlashStringIterator(++_ptr); + } + + bool operator!=(FlashStringIterator other) const { + return _ptr != other._ptr; + } + + char operator*() const { + return char(pgm_read_byte(_ptr)); + } + + private: + const char* _ptr; +}; + +} // namespace ARDUINOJSON_NAMESPACE diff --git a/src/ArduinoJson/Strings/SizedFlashStringAdapter.hpp b/src/ArduinoJson/Strings/SizedFlashStringAdapter.hpp index b6809a80..da3cc726 100644 --- a/src/ArduinoJson/Strings/SizedFlashStringAdapter.hpp +++ b/src/ArduinoJson/Strings/SizedFlashStringAdapter.hpp @@ -5,6 +5,7 @@ #pragma once #include +#include #include #include @@ -41,6 +42,10 @@ class SizedFlashStringAdapter { return _size; } + FlashStringIterator begin() const { + return FlashStringIterator(_str); + } + typedef storage_policies::store_by_copy storage_policy; private: diff --git a/src/ArduinoJson/Strings/SizedRamStringAdapter.hpp b/src/ArduinoJson/Strings/SizedRamStringAdapter.hpp index 2c1e87f6..54eda232 100644 --- a/src/ArduinoJson/Strings/SizedRamStringAdapter.hpp +++ b/src/ArduinoJson/Strings/SizedRamStringAdapter.hpp @@ -36,6 +36,10 @@ class SizedRamStringAdapter { return _size; } + const char* begin() const { + return _str; + } + typedef storage_policies::store_by_copy storage_policy; private: diff --git a/src/ArduinoJson/Strings/StlStringAdapter.hpp b/src/ArduinoJson/Strings/StlStringAdapter.hpp index fd6bf6d0..a7d2c915 100644 --- a/src/ArduinoJson/Strings/StlStringAdapter.hpp +++ b/src/ArduinoJson/Strings/StlStringAdapter.hpp @@ -41,6 +41,10 @@ class StlStringAdapter { return _str->size(); } + const char* begin() const { + return _str->c_str(); + } + typedef storage_policies::store_by_copy storage_policy; private: diff --git a/src/ArduinoJson/Variant/VariantData.hpp b/src/ArduinoJson/Variant/VariantData.hpp index 87caa15f..d8b44ef2 100644 --- a/src/ArduinoJson/Variant/VariantData.hpp +++ b/src/ArduinoJson/Variant/VariantData.hpp @@ -192,7 +192,7 @@ class VariantData { template bool setOwnedRaw(SerializedValue value, MemoryPool *pool) { - char *dup = pool->saveString(adaptString(value.data(), value.size())); + const char *dup = pool->saveString(adaptString(value.data(), value.size())); if (dup) { setType(VALUE_IS_OWNED_RAW); _content.asRaw.data = dup;