Store the strings in the heap

src/ArduinoJson/Collection/CollectionData.hpp

@@ -70,7 +70,7 @@ class CollectionData {
     return _head;
   }
 
-  void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance);
+  void movePointers(ptrdiff_t variantDistance);
 
  private:
   VariantSlot* getSlot(size_t index) const;

src/ArduinoJson/Collection/CollectionImpl.hpp

@@ -168,7 +168,7 @@ inline size_t CollectionData::memoryUsage() const {
   for (VariantSlot* s = _head; s; s = s->next()) {
     total += sizeof(VariantSlot) + s->data()->memoryUsage();
     if (s->ownsKey())
-      total += strlen(s->key()) + 1;
+      total += sizeofString(strlen(s->key()));
   }
   return total;
 }
@@ -186,12 +186,11 @@ inline void movePointer(T*& p, ptrdiff_t offset) {
   ARDUINOJSON_ASSERT(isAligned(p));
 }
 
-inline void CollectionData::movePointers(ptrdiff_t stringDistance,
-                                         ptrdiff_t variantDistance) {
+inline void CollectionData::movePointers(ptrdiff_t variantDistance) {
   movePointer(_head, variantDistance);
   movePointer(_tail, variantDistance);
   for (VariantSlot* slot = _head; slot; slot = slot->next())
-    slot->movePointers(stringDistance, variantDistance);
+    slot->movePointers(variantDistance);
 }
 
 ARDUINOJSON_END_PRIVATE_NAMESPACE

src/ArduinoJson/Json/JsonDeserializer.hpp

@@ -277,7 +277,6 @@ class JsonDeserializer {
         VariantData* variant = object.getMember(adaptString(key.c_str()));
         if (!variant) {
           // Save key in memory pool.
-          // This MUST be done before adding the slot.
           key = _stringStorage.save();
 
           // Allocate slot in object

src/ArduinoJson/Memory/MemoryPool.hpp

@@ -25,9 +25,15 @@ constexpr size_t sizeofObject(size_t n) {
   return n * sizeof(VariantSlot);
 }
 
+struct StringNode {
+  struct StringNode* next;
+  uint16_t length;
+  char data[1];
+};
+
 // Returns the size (in bytes) of an string with n characters.
 constexpr size_t sizeofString(size_t n) {
-  return n + 1;
+  return n + 1 + offsetof(StringNode, data);
 }
 
 // _begin                                   _end
@@ -46,6 +52,7 @@ class MemoryPool {
   }
 
   ~MemoryPool() {
+    deallocAllStrings();
     deallocPool();
   }
 
@@ -53,6 +60,7 @@ class MemoryPool {
   MemoryPool& operator=(const MemoryPool& src) = delete;
 
   MemoryPool& operator=(MemoryPool&& src) {
+    deallocAllStrings();
     deallocPool();
     _allocator = src._allocator;
     _begin = src._begin;
@@ -61,6 +69,8 @@ class MemoryPool {
     _right = src._right;
     _overflowed = src._overflowed;
     src._begin = src._end = src._left = src._right = nullptr;
+    _strings = src._strings;
+    src._strings = nullptr;
     return *this;
   }
 
@@ -87,7 +97,10 @@ class MemoryPool {
   }
 
   size_t size() const {
-    return size_t(_left - _begin + _end - _right);
+    size_t total = size_t(_left - _begin + _end - _right);
+    for (auto node = _strings; node; node = node->next)
+      total += sizeofString(node->length);
+    return total;
   }
 
   bool overflowed() const {
@@ -103,45 +116,71 @@ class MemoryPool {
     if (str.isNull())
       return 0;
 
-    const char* existingCopy = findString(str);
-    if (existingCopy)
-      return existingCopy;
+    auto node = findString(str);
+    if (node) {
+      return node->data;
+    }
 
     size_t n = str.size();
 
-    char* newCopy = allocString(n + 1);
-    if (newCopy) {
-      stringGetChars(str, newCopy, n);
-      newCopy[n] = 0;  // force null-terminator
+    node = allocString(n);
+    if (!node)
+      return nullptr;
+
+    stringGetChars(str, node->data, n);
+    node->data[n] = 0;  // force NUL terminator
+    addStringToList(node);
+    return node->data;
+  }
+
+  void addStringToList(StringNode* node) {
+    ARDUINOJSON_ASSERT(node != nullptr);
+    node->next = _strings;
+    _strings = node;
+  }
+
+  template <typename TAdaptedString>
+  StringNode* findString(const TAdaptedString& str) const {
+    for (auto node = _strings; node; node = node->next) {
+      if (stringEquals(str, adaptString(node->data, node->length)))
+        return node;
     }
-    return newCopy;
+    return nullptr;
   }
 
-  void getFreeZone(char** zoneStart, size_t* zoneSize) const {
-    *zoneStart = _left;
-    *zoneSize = size_t(_right - _left);
+  StringNode* allocString(size_t length) {
+    auto node = reinterpret_cast<StringNode*>(
+        _allocator->allocate(sizeofString(length)));
+    if (node) {
+      node->length = uint16_t(length);
+    } else {
+      _overflowed = true;
+    }
+    return node;
   }
 
-  const char* saveStringFromFreeZone(size_t len) {
-    const char* dup = findString(adaptString(_left, len));
-    if (dup)
-      return dup;
-
-    const char* str = _left;
-    _left += len;
-    *_left++ = 0;
-    checkInvariants();
-    return str;
+  StringNode* reallocString(StringNode* node, size_t length) {
+    ARDUINOJSON_ASSERT(node != nullptr);
+    auto newNode = reinterpret_cast<StringNode*>(
+        _allocator->reallocate(node, sizeofString(length)));
+    if (newNode) {
+      newNode->length = uint16_t(length);
+    } else {
+      _overflowed = true;
+      _allocator->deallocate(node);
+    }
+    return newNode;
   }
 
-  void markAsOverflowed() {
-    _overflowed = true;
+  void deallocString(StringNode* node) {
+    _allocator->deallocate(node);
   }
 
   void clear() {
     _left = _begin;
     _right = _end;
     _overflowed = false;
+    deallocAllStrings();
   }
 
   bool canAlloc(size_t bytes) const {
@@ -169,8 +208,8 @@ class MemoryPool {
         static_cast<char*>(new_ptr) - static_cast<char*>(old_ptr);
 
     movePointers(ptr_offset);
-    reinterpret_cast<VariantSlot&>(variant).movePointers(
-        ptr_offset, ptr_offset - bytes_reclaimed);
+    reinterpret_cast<VariantSlot&>(variant).movePointers(ptr_offset -
+                                                         bytes_reclaimed);
   }
 
  private:
@@ -215,29 +254,12 @@ class MemoryPool {
     ARDUINOJSON_ASSERT(isAligned(_right));
   }
 
-  template <typename TAdaptedString>
-  const char* findString(const TAdaptedString& str) const {
-    size_t n = str.size();
-    for (char* next = _begin; next + n < _left; ++next) {
-      if (next[n] == '\0' && stringEquals(str, adaptString(next, n)))
-        return next;
-
-      // jump to next terminator
-      while (*next)
-        ++next;
+  void deallocAllStrings() {
+    while (_strings) {
+      auto node = _strings;
+      _strings = node->next;
+      deallocString(node);
     }
-    return 0;
-  }
-
-  char* allocString(size_t n) {
-    if (!canAlloc(n)) {
-      _overflowed = true;
-      return 0;
-    }
-    char* s = _left;
-    _left += n;
-    checkInvariants();
-    return s;
   }
 
   template <typename T>
@@ -271,6 +293,7 @@ class MemoryPool {
   Allocator* _allocator;
   char *_begin, *_left, *_right, *_end;
   bool _overflowed;
+  StringNode* _strings = nullptr;
 };
 
 template <typename TAdaptedString, typename TCallback>

src/ArduinoJson/MsgPack/MsgPackDeserializer.hpp

@@ -494,7 +494,6 @@ class MsgPackDeserializer {
         ARDUINOJSON_ASSERT(object != 0);
 
         // Save key in memory pool.
-        // This MUST be done before adding the slot.
         key = _stringStorage.save();
 
         VariantSlot* slot = object->addSlot(_pool);

src/ArduinoJson/StringStorage/StringCopier.hpp

@@ -10,20 +10,31 @@ ARDUINOJSON_BEGIN_PRIVATE_NAMESPACE
 
 class StringCopier {
  public:
+  static const size_t initialCapacity = 31;
+
   StringCopier(MemoryPool* pool) : _pool(pool) {}
 
+  ~StringCopier() {
+    if (_node)
+      _pool->deallocString(_node);
+  }
+
   void startString() {
-    _pool->getFreeZone(&_ptr, &_capacity);
     _size = 0;
-    if (_capacity == 0)
-      _pool->markAsOverflowed();
+    if (!_node)
+      _node = _pool->allocString(initialCapacity);
   }
 
   JsonString save() {
-    ARDUINOJSON_ASSERT(_ptr);
-    ARDUINOJSON_ASSERT(_size < _capacity);  // needs room for the terminator
-    return JsonString(_pool->saveStringFromFreeZone(_size), _size,
-                      JsonString::Copied);
+    ARDUINOJSON_ASSERT(_node != nullptr);
+    _node->data[_size] = 0;
+    StringNode* node = _pool->findString(adaptString(_node->data, _size));
+    if (!node) {
+      node = _pool->reallocString(_node, _size);
+      _pool->addStringToList(node);
+      _node = nullptr;  // next time we need a new string
+    }
+    return JsonString(node->data, node->length, JsonString::Copied);
   }
 
   void append(const char* s) {
@@ -32,19 +43,19 @@ class StringCopier {
   }
 
   void append(const char* s, size_t n) {
-    while (n-- > 0)
+    while (n-- > 0)  // TODO: memcpy
       append(*s++);
   }
 
   void append(char c) {
-    if (_size + 1 < _capacity)
-      _ptr[_size++] = c;
-    else
-      _pool->markAsOverflowed();
+    if (_node && _size == _node->length)
+      _node = _pool->reallocString(_node, _size * 2U + 1);
+    if (_node)
+      _node->data[_size++] = c;
   }
 
   bool isValid() const {
-    return !_pool->overflowed();
+    return _node != nullptr;
   }
 
   size_t size() const {
@@ -52,21 +63,15 @@ class StringCopier {
   }
 
   JsonString str() const {
-    ARDUINOJSON_ASSERT(_ptr);
-    ARDUINOJSON_ASSERT(_size < _capacity);
-    _ptr[_size] = 0;
-    return JsonString(_ptr, _size, JsonString::Copied);
+    ARDUINOJSON_ASSERT(_node != nullptr);
+    _node->data[_size] = 0;
+    return JsonString(_node->data, _size, JsonString::Copied);
   }
 
  private:
   MemoryPool* _pool;
-
-  // These fields aren't initialized by the constructor but startString()
-  //
-  // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.UninitializedObject)
-  char* _ptr;
-  // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.UninitializedObject)
-  size_t _size, _capacity;
+  StringNode* _node = nullptr;
+  size_t _size = 0;
 };
 
 ARDUINOJSON_END_PRIVATE_NAMESPACE

src/ArduinoJson/Variant/ConverterImpl.hpp

@@ -5,6 +5,7 @@
 #pragma once
 
 #include <ArduinoJson/Json/JsonSerializer.hpp>
+#include <ArduinoJson/StringStorage/StringCopier.hpp>
 #include <ArduinoJson/Variant/JsonVariantConst.hpp>
 #include <ArduinoJson/Variant/VariantFunctions.hpp>
 
@@ -205,43 +206,35 @@ struct Converter<decltype(nullptr)> : private detail::VariantAttorney {
 namespace detail {
 class MemoryPoolPrint : public Print {
  public:
-  MemoryPoolPrint(MemoryPool* pool) : _pool(pool), _size(0) {
-    pool->getFreeZone(&_string, &_capacity);
+  MemoryPoolPrint(MemoryPool* pool) : _copier(pool) {
+    _copier.startString();
   }
 
   JsonString str() {
-    ARDUINOJSON_ASSERT(_size < _capacity);
-    return JsonString(_pool->saveStringFromFreeZone(_size), _size,
-                      JsonString::Copied);
+    ARDUINOJSON_ASSERT(!overflowed());
+    return _copier.save();
   }
 
   size_t write(uint8_t c) {
-    if (_size >= _capacity)
-      return 0;
-
-    _string[_size++] = char(c);
-    return 1;
+    _copier.append(char(c));
+    return _copier.isValid() ? 1 : 0;
   }
 
   size_t write(const uint8_t* buffer, size_t size) {
-    if (_size + size >= _capacity) {
-      _size = _capacity;  // mark as overflowed
-      return 0;
+    for (size_t i = 0; i < size; i++) {
+      _copier.append(char(buffer[i]));
+      if (!_copier.isValid())
+        return i;
     }
-    memcpy(&_string[_size], buffer, size);
-    _size += size;
     return size;
   }
 
   bool overflowed() const {
-    return _size >= _capacity;
+    return !_copier.isValid();
   }
 
  private:
-  MemoryPool* _pool;
-  size_t _size;
-  char* _string;
-  size_t _capacity;
+  StringCopier _copier;
 };
 }  // namespace detail
 
@@ -253,7 +246,6 @@ inline void convertToJson(const ::Printable& src, JsonVariant dst) {
   detail::MemoryPoolPrint print(pool);
   src.printTo(print);
   if (print.overflowed()) {
-    pool->markAsOverflowed();
     data->setNull();
     return;
   }

src/ArduinoJson/Variant/VariantData.hpp

@@ -226,9 +226,7 @@ class VariantData {
     switch (type()) {
       case VALUE_IS_OWNED_STRING:
       case VALUE_IS_OWNED_RAW:
-        // We always add a zero at the end: the deduplication function uses it
-        // to detect the beginning of the next string.
-        return _content.asString.size + 1;
+        return sizeofString(_content.asString.size);
       case VALUE_IS_OBJECT:
       case VALUE_IS_ARRAY:
         return _content.asCollection.memoryUsage();
@@ -277,11 +275,9 @@ class VariantData {
     return _content.asCollection.getOrAddMember(key, pool);
   }
 
-  void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance) {
-    if (_flags & OWNED_VALUE_BIT)
-      _content.asString.data += stringDistance;
+  void movePointers(ptrdiff_t variantDistance) {
     if (_flags & COLLECTION_MASK)
-      _content.asCollection.movePointers(stringDistance, variantDistance);
+      _content.asCollection.movePointers(variantDistance);
   }
 
   uint8_t type() const {

src/ArduinoJson/Variant/VariantSlot.hpp

@@ -99,13 +99,9 @@ class VariantSlot {
     _key = 0;
   }
 
-  void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance) {
-    if (_flags & OWNED_KEY_BIT)
-      _key += stringDistance;
-    if (_flags & OWNED_VALUE_BIT)
-      _content.asString.data += stringDistance;
+  void movePointers(ptrdiff_t variantDistance) {
     if (_flags & COLLECTION_MASK)
-      _content.asCollection.movePointers(stringDistance, variantDistance);
+      _content.asCollection.movePointers(variantDistance);
   }
 };