diff --git a/CHANGELOG.md b/CHANGELOG.md index fd0ffe19..a1c2cec4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ ArduinoJson: change log ======================= +HEAD +---- + +* Added `BasicJsonDocument::shrinkToFit()` + v6.13.0 (2019-11-01) ------- diff --git a/extras/tests/JsonDocument/CMakeLists.txt b/extras/tests/JsonDocument/CMakeLists.txt index 1e2d7b8a..319da1a8 100644 --- a/extras/tests/JsonDocument/CMakeLists.txt +++ b/extras/tests/JsonDocument/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable(JsonDocumentTests isNull.cpp nesting.cpp remove.cpp + shrinkToFit.cpp size.cpp StaticJsonDocument.cpp subscript.cpp diff --git a/extras/tests/JsonDocument/shrinkToFit.cpp b/extras/tests/JsonDocument/shrinkToFit.cpp new file mode 100644 index 00000000..a927a4c4 --- /dev/null +++ b/extras/tests/JsonDocument/shrinkToFit.cpp @@ -0,0 +1,151 @@ +// ArduinoJson - arduinojson.org +// Copyright Benoit Blanchon 2014-2019 +// MIT License + +#include +#include + +#include // malloc, free +#include + +using ARDUINOJSON_NAMESPACE::addPadding; + +class ArmoredAllocator { + public: + ArmoredAllocator() : _ptr(0), _size(0) {} + + void* allocate(size_t size) { + _ptr = malloc(size); + _size = size; + return _ptr; + } + + void deallocate(void* ptr) { + REQUIRE(ptr == _ptr); + free(ptr); + _ptr = 0; + _size = 0; + } + + void* reallocate(void* ptr, size_t new_size) { + REQUIRE(ptr == _ptr); + // don't call realloc, instead alloc a new buffer and erase the old one + // this way we make sure we support relocation + void* new_ptr = malloc(new_size); + memcpy(new_ptr, _ptr, std::min(new_size, _size)); + memset(_ptr, '#', _size); // erase + free(_ptr); + _ptr = new_ptr; + return new_ptr; + } + + private: + void* _ptr; + size_t _size; +}; + +typedef BasicJsonDocument ShrinkToFitTestDocument; + +void testShrinkToFit(ShrinkToFitTestDocument& doc, std::string expected_json, + size_t expected_size) { + doc.shrinkToFit(); + + REQUIRE(doc.capacity() == expected_size); + REQUIRE(doc.memoryUsage() == expected_size); + + std::string json; + serializeJson(doc, json); + REQUIRE(json == expected_json); +} + +TEST_CASE("BasicJsonDocument::shrinkToFit()") { + ShrinkToFitTestDocument doc(4096); + + SECTION("null") { + testShrinkToFit(doc, "null", 0); + } + + SECTION("empty object") { + deserializeJson(doc, "{}"); + testShrinkToFit(doc, "{}", JSON_OBJECT_SIZE(0)); + } + + SECTION("empty array") { + deserializeJson(doc, "[]"); + testShrinkToFit(doc, "[]", JSON_ARRAY_SIZE(0)); + } + + SECTION("linked string") { + doc.set("hello"); + testShrinkToFit(doc, "\"hello\"", 0); + } + + SECTION("owned string") { + doc.set(std::string("abcdefg")); + testShrinkToFit(doc, "\"abcdefg\"", 8); + } + + SECTION("linked raw") { + doc.set(serialized("[{},123]")); + testShrinkToFit(doc, "[{},123]", 0); + } + + SECTION("owned raw") { + doc.set(serialized(std::string("[{},123]"))); + testShrinkToFit(doc, "[{},123]", 8); + } + + SECTION("linked key") { + doc["key"] = 42; + testShrinkToFit(doc, "{\"key\":42}", JSON_OBJECT_SIZE(1)); + } + + SECTION("owned key") { + doc[std::string("abcdefg")] = 42; + testShrinkToFit(doc, "{\"abcdefg\":42}", JSON_OBJECT_SIZE(1) + 8); + } + + SECTION("linked string in array") { + doc.add("hello"); + testShrinkToFit(doc, "[\"hello\"]", JSON_ARRAY_SIZE(1)); + } + + SECTION("owned string in array") { + doc.add(std::string("abcdefg")); + testShrinkToFit(doc, "[\"abcdefg\"]", JSON_ARRAY_SIZE(1) + 8); + } + + SECTION("linked string in object") { + doc["key"] = "hello"; + testShrinkToFit(doc, "{\"key\":\"hello\"}", JSON_OBJECT_SIZE(1)); + } + + SECTION("owned string in object") { + doc["key"] = std::string("abcdefg"); + testShrinkToFit(doc, "{\"key\":\"abcdefg\"}", JSON_ARRAY_SIZE(1) + 8); + } + + SECTION("unaligned") { + doc.add(std::string("?")); // two bytes in the string pool + REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1) + 2); + + doc.shrinkToFit(); + + // the new capacity should be padded to align the pointers + REQUIRE(doc.capacity() == JSON_OBJECT_SIZE(1) + sizeof(void*)); + REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1) + 2); + REQUIRE(doc[0] == "?"); + } +} + +TEST_CASE("DynamicJsonDocument::shrinkToFit()") { + DynamicJsonDocument doc(4096); + + deserializeJson(doc, "{\"hello\":[\"world\"]"); + + doc.shrinkToFit(); + + std::string json; + serializeJson(doc, json); + REQUIRE(json == "{\"hello\":[\"world\"]}"); +} diff --git a/src/ArduinoJson/Collection/CollectionData.hpp b/src/ArduinoJson/Collection/CollectionData.hpp index 9a8c5bfe..ba1b008f 100644 --- a/src/ArduinoJson/Collection/CollectionData.hpp +++ b/src/ArduinoJson/Collection/CollectionData.hpp @@ -5,6 +5,7 @@ #pragma once #include +#include #include // size_t @@ -63,6 +64,8 @@ class CollectionData { size_t nesting() const; size_t size() const; + void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance); + private: VariantSlot *getSlot(size_t index) const; diff --git a/src/ArduinoJson/Collection/CollectionImpl.hpp b/src/ArduinoJson/Collection/CollectionImpl.hpp index b70fb253..f7860e97 100644 --- a/src/ArduinoJson/Collection/CollectionImpl.hpp +++ b/src/ArduinoJson/Collection/CollectionImpl.hpp @@ -160,4 +160,20 @@ inline size_t CollectionData::size() const { return slotSize(_head); } +template +inline void movePointer(T*& p, ptrdiff_t offset) { + if (!p) return; + p = reinterpret_cast( + reinterpret_cast(reinterpret_cast(p) + offset)); + ARDUINOJSON_ASSERT(isAligned(p)); +} + +inline void CollectionData::movePointers(ptrdiff_t stringDistance, + ptrdiff_t variantDistance) { + movePointer(_head, variantDistance); + movePointer(_tail, variantDistance); + for (VariantSlot* slot = _head; slot; slot = slot->next()) + slot->movePointers(stringDistance, variantDistance); +} + } // namespace ARDUINOJSON_NAMESPACE diff --git a/src/ArduinoJson/Document/BasicJsonDocument.hpp b/src/ArduinoJson/Document/BasicJsonDocument.hpp index df6f83d5..6539d8d4 100644 --- a/src/ArduinoJson/Document/BasicJsonDocument.hpp +++ b/src/ArduinoJson/Document/BasicJsonDocument.hpp @@ -8,6 +8,8 @@ namespace ARDUINOJSON_NAMESPACE { +// Helper to implement the "base-from-member" idiom +// (we need to store the allocator before constructing JsonDocument) template class AllocatorOwner { protected: @@ -15,12 +17,16 @@ class AllocatorOwner { AllocatorOwner(const AllocatorOwner& src) : _allocator(src._allocator) {} AllocatorOwner(TAllocator allocator) : _allocator(allocator) {} - void* allocate(size_t n) { - return _allocator.allocate(n); + void* allocate(size_t size) { + return _allocator.allocate(size); } - void deallocate(void* p) { - _allocator.deallocate(p); + void deallocate(void* ptr) { + _allocator.deallocate(ptr); + } + + void* reallocate(void* ptr, size_t new_size) { + return _allocator.reallocate(ptr, new_size); } private: @@ -69,6 +75,20 @@ class BasicJsonDocument : AllocatorOwner, public JsonDocument { return *this; } + void shrinkToFit() { + ptrdiff_t bytes_reclaimed = _pool.squash(); + if (bytes_reclaimed == 0) return; + + void* old_ptr = _pool.buffer(); + void* new_ptr = this->reallocate(old_ptr, _pool.capacity()); + + ptrdiff_t ptr_offset = + static_cast(new_ptr) - static_cast(old_ptr); + + _pool.movePointers(ptr_offset); + _data.movePointers(ptr_offset, ptr_offset - bytes_reclaimed); + } + private: MemoryPool allocPool(size_t requiredSize) { size_t capa = addPadding(requiredSize); diff --git a/src/ArduinoJson/Document/DynamicJsonDocument.hpp b/src/ArduinoJson/Document/DynamicJsonDocument.hpp index 3e2b54b2..fd3befa1 100644 --- a/src/ArduinoJson/Document/DynamicJsonDocument.hpp +++ b/src/ArduinoJson/Document/DynamicJsonDocument.hpp @@ -11,12 +11,16 @@ namespace ARDUINOJSON_NAMESPACE { struct DefaultAllocator { - void* allocate(size_t n) { - return malloc(n); + void* allocate(size_t size) { + return malloc(size); } - void deallocate(void* p) { - free(p); + void deallocate(void* ptr) { + free(ptr); + } + + void* reallocate(void* ptr, size_t new_size) { + return realloc(ptr, new_size); } }; diff --git a/src/ArduinoJson/Document/JsonDocument.hpp b/src/ArduinoJson/Document/JsonDocument.hpp index 499c6502..e9db32c1 100644 --- a/src/ArduinoJson/Document/JsonDocument.hpp +++ b/src/ArduinoJson/Document/JsonDocument.hpp @@ -301,7 +301,6 @@ class JsonDocument : public Visitable { _pool = pool; } - private: VariantRef getVariant() { return VariantRef(&_pool, &_data); } diff --git a/src/ArduinoJson/Memory/Alignment.hpp b/src/ArduinoJson/Memory/Alignment.hpp index 96acb4d3..e25741f4 100644 --- a/src/ArduinoJson/Memory/Alignment.hpp +++ b/src/ArduinoJson/Memory/Alignment.hpp @@ -21,6 +21,12 @@ inline size_t addPadding(size_t bytes) { return (bytes + mask) & ~mask; } +template +inline T *addPadding(T *p) { + size_t address = addPadding(reinterpret_cast(p)); + return reinterpret_cast(address); +} + template struct AddPadding { static const size_t mask = sizeof(void *) - 1; diff --git a/src/ArduinoJson/Memory/MemoryPool.hpp b/src/ArduinoJson/Memory/MemoryPool.hpp index 4c4e8581..d1b6480d 100644 --- a/src/ArduinoJson/Memory/MemoryPool.hpp +++ b/src/ArduinoJson/Memory/MemoryPool.hpp @@ -10,13 +10,15 @@ #include #include +#include // memmove + namespace ARDUINOJSON_NAMESPACE { -// _begin _end -// v v -// +-------------+--------------+-----------+ -// | strings... | (free) | ...slots | -// +-------------+--------------+-----------+ +// _begin _end +// v v +// +-------------+--------------+--------------+ +// | strings... | (free) | ...variants | +// +-------------+--------------+--------------+ // ^ ^ // _left _right @@ -101,6 +103,39 @@ class MemoryPool { return p; } + // Squash the free space between strings and variants + // + // _begin _end + // v v + // +-------------+--------------+ + // | strings... | ...variants | + // +-------------+--------------+ + // ^ + // _left _right + // + // This funcion is called before a realloc. + ptrdiff_t squash() { + char* new_right = addPadding(_left); + if (new_right >= _right) return 0; + + size_t right_size = static_cast(_end - _right); + memmove(new_right, _right, right_size); + + ptrdiff_t bytes_reclaimed = _right - new_right; + _right = new_right; + _end = new_right + right_size; + return bytes_reclaimed; + } + + // Move all pointers together + // This funcion is called after a realloc. + void movePointers(ptrdiff_t offset) { + _begin += offset; + _left += offset; + _right += offset; + _end += offset; + } + private: StringSlot* allocStringSlot() { return allocRight(); diff --git a/src/ArduinoJson/Variant/VariantContent.hpp b/src/ArduinoJson/Variant/VariantContent.hpp index c1becfe0..573b5898 100644 --- a/src/ArduinoJson/Variant/VariantContent.hpp +++ b/src/ArduinoJson/Variant/VariantContent.hpp @@ -16,14 +16,14 @@ namespace ARDUINOJSON_NAMESPACE { enum { VALUE_MASK = 0x7F, - OWNERSHIP_BIT = 0x01, + VALUE_IS_OWNED = 0x01, VALUE_IS_NULL = 0, VALUE_IS_LINKED_RAW = 0x02, VALUE_IS_OWNED_RAW = 0x03, VALUE_IS_LINKED_STRING = 0x04, VALUE_IS_OWNED_STRING = 0x05, - // CAUTION: no OWNERSHIP_BIT below + // CAUTION: no VALUE_IS_OWNED below VALUE_IS_BOOLEAN = 0x06, VALUE_IS_POSITIVE_INTEGER = 0x08, VALUE_IS_NEGATIVE_INTEGER = 0x0A, diff --git a/src/ArduinoJson/Variant/VariantData.hpp b/src/ArduinoJson/Variant/VariantData.hpp index 1094b89d..931c0141 100644 --- a/src/ArduinoJson/Variant/VariantData.hpp +++ b/src/ArduinoJson/Variant/VariantData.hpp @@ -103,7 +103,7 @@ class VariantData { bool equals(const VariantData &other) const { // Check that variant have the same type, but ignore string ownership - if ((type() | OWNERSHIP_BIT) != (other.type() | OWNERSHIP_BIT)) + if ((type() | VALUE_IS_OWNED) != (other.type() | VALUE_IS_OWNED)) return false; switch (type()) { @@ -352,6 +352,12 @@ class VariantData { return _content.asCollection.add(key, pool); } + void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance) { + if (_flags & VALUE_IS_OWNED) _content.asString += stringDistance; + if (_flags & COLLECTION_MASK) + _content.asCollection.movePointers(stringDistance, variantDistance); + } + private: uint8_t type() const { return _flags & VALUE_MASK; diff --git a/src/ArduinoJson/Variant/VariantSlot.hpp b/src/ArduinoJson/Variant/VariantSlot.hpp index a17c890f..0599c1eb 100644 --- a/src/ArduinoJson/Variant/VariantSlot.hpp +++ b/src/ArduinoJson/Variant/VariantSlot.hpp @@ -14,8 +14,6 @@ namespace ARDUINOJSON_NAMESPACE { typedef conditional::type VariantSlotDiff; -class VairantData; - class VariantSlot { // CAUTION: same layout as VariantData // we cannot use composition because it adds padding @@ -93,6 +91,13 @@ class VariantSlot { _flags = 0; _key = 0; } + + void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance) { + if (_flags & KEY_IS_OWNED) _key += stringDistance; + if (_flags & VALUE_IS_OWNED) _content.asString += stringDistance; + if (_flags & COLLECTION_MASK) + _content.asCollection.movePointers(stringDistance, variantDistance); + } }; } // namespace ARDUINOJSON_NAMESPACE