Replacing a value now releases the memory

This commit is contained in:
Benoit Blanchon
2018-11-12 18:28:34 +01:00
parent f375459d53
commit 720e6548c7
18 changed files with 335 additions and 132 deletions

View File

@ -8,8 +8,8 @@ HEAD
* Removed `JsonObject::is<T>(k)` and `JsonObject::set(k,v)`
* Replaced `T JsonArray::get<T>(i)` with `JsonVariant JsonArray::get(i)`
* Replaced `T JsonObject::get<T>(k)` with `JsonVariant JsonObject::get(k)`
* `JsonArray::remove()` and `JsonObject::remove()` now release the memory
* Added `JSON_STRING_SIZE()`
* Replacing or removing a value now releases the memory.
v6.5.0-beta (2018-10-13)
-----------

View File

@ -7,11 +7,14 @@ project(ArduinoJson)
enable_testing()
if(${COVERAGE})
set(CMAKE_CXX_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage")
if(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Clang)")
add_definitions(-DARDUINOJSON_DEBUG)
add_compile_options(-g -O0)
endif()
add_definitions(-DARDUINOJSON_DEBUG)
if(${COVERAGE})
set(CMAKE_CXX_FLAGS "-fprofile-arcs -ftest-coverage")
endif()
include_directories(${CMAKE_CURRENT_LIST_DIR}/src)
add_subdirectory(third-party/catch)

View File

@ -6,3 +6,8 @@ add_executable(msgpack_fuzzer
msgpack_fuzzer.cpp
fuzzer_main.cpp
)
add_executable(json_fuzzer
json_fuzzer.cpp
fuzzer_main.cpp
)

View File

@ -16,7 +16,7 @@ fuzz() {
export ASAN_OPTIONS="detect_leaks=0"
export LLVM_PROFILE_FILE="${FUZZER}.profraw"
./${FUZZER} "$CORPUS_DIR" "$SEED_CORPUS_DIR" -max_total_time=30
./${FUZZER} "$CORPUS_DIR" "$SEED_CORPUS_DIR" -max_total_time=30 -timeout=1
llvm-profdata-${CLANG} merge -sparse ${LLVM_PROFILE_FILE} -o ${FUZZER}.profdata
llvm-cov-${CLANG} report ./${FUZZER} -instr-profile=${FUZZER}.profdata

View File

@ -100,4 +100,15 @@ inline size_t arraySize(const JsonArrayData* arr) {
if (!arr) return 0;
return slotSize(arr->head);
}
inline void arrayFree(JsonArrayData* arr, MemoryPool* pool) {
ARDUINOJSON_ASSERT(arr);
VariantSlot* cur = arr->head;
while (cur) {
VariantSlot* next = cur->next;
slotFree(cur, pool);
cur = next;
}
}
} // namespace ARDUINOJSON_NAMESPACE

View File

@ -5,6 +5,7 @@
#pragma once
#include "../Memory/MemoryPool.hpp"
#include "../Polyfills/assert.hpp"
#include "../Strings/StringTypes.hpp"
#include "JsonVariantData.hpp"
@ -61,25 +62,15 @@ inline size_t slotSize(const VariantSlot* var) {
return n;
}
void variantFree(JsonVariantData* var, MemoryPool* pool);
inline void slotFree(VariantSlot* var, MemoryPool* pool) {
const JsonVariantData& v = var->value;
ARDUINOJSON_ASSERT(var != 0);
ARDUINOJSON_ASSERT(pool != 0);
switch (v.type) {
case JSON_ARRAY:
case JSON_OBJECT:
for (VariantSlot* s = v.content.asObject.head; s; s = s->next) {
slotFree(s, pool);
}
break;
case JSON_OWNED_STRING:
case JSON_OWNED_RAW:
pool->freeString(v.content.asOwnedString);
break;
default:
break;
}
variantFree(&var->value, pool);
if (v.keyIsOwned) pool->freeString(var->ownedKey);
if (var->value.keyIsOwned) pool->freeString(var->ownedKey);
pool->freeVariant(var);
}

View File

@ -13,6 +13,24 @@
namespace ARDUINOJSON_NAMESPACE {
inline void variantFree(JsonVariantData* var, MemoryPool* pool) {
ARDUINOJSON_ASSERT(var != 0);
ARDUINOJSON_ASSERT(pool != 0);
switch (var->type) {
case JSON_ARRAY:
case JSON_OBJECT:
arrayFree(&var->content.asArray, pool);
break;
case JSON_OWNED_STRING:
case JSON_OWNED_RAW:
pool->freeString(var->content.asOwnedString);
break;
default:
break;
}
}
template <typename T>
inline T variantAsIntegral(const JsonVariantData* var) {
if (!var) return 0;
@ -98,23 +116,30 @@ inline const JsonObjectData* variantAsObject(const JsonVariantData* var) {
return 0;
}
inline bool variantSetBoolean(JsonVariantData* var, bool value) {
inline bool variantSetBoolean(JsonVariantData* var, bool value,
MemoryPool* pool) {
if (!var) return false;
variantFree(var, pool);
var->type = JSON_BOOLEAN;
var->content.asInteger = static_cast<JsonUInt>(value);
return true;
}
inline bool variantSetFloat(JsonVariantData* var, JsonFloat value) {
inline bool variantSetFloat(JsonVariantData* var, JsonFloat value,
MemoryPool* pool) {
if (!var) return false;
variantFree(var, pool);
var->type = JSON_FLOAT;
var->content.asFloat = value;
return true;
}
template <typename T>
inline bool variantSetSignedInteger(JsonVariantData* var, T value) {
inline bool variantSetSignedInteger(JsonVariantData* var, T value,
MemoryPool* pool) {
if (!var) return false;
variantFree(var, pool);
if (value >= 0) {
var->type = JSON_POSITIVE_INTEGER;
var->content.asInteger = static_cast<JsonUInt>(value);
@ -125,16 +150,20 @@ inline bool variantSetSignedInteger(JsonVariantData* var, T value) {
return true;
}
inline bool variantSetSignedInteger(JsonVariantData* var, JsonUInt value) {
inline bool variantSetUnsignedInteger(JsonVariantData* var, JsonUInt value,
MemoryPool* pool) {
if (!var) return false;
variantFree(var, pool);
var->type = JSON_POSITIVE_INTEGER;
var->content.asInteger = static_cast<JsonUInt>(value);
return true;
}
inline bool variantSetLinkedRaw(JsonVariantData* var,
SerializedValue<const char*> value) {
SerializedValue<const char*> value,
MemoryPool* pool) {
if (!var) return false;
variantFree(var, pool);
var->type = JSON_LINKED_RAW;
var->content.asRaw.data = value.data();
var->content.asRaw.size = value.size();
@ -145,6 +174,7 @@ template <typename T>
inline bool variantSetOwnedRaw(JsonVariantData* var, SerializedValue<T> value,
MemoryPool* pool) {
if (!var) return false;
variantFree(var, pool);
StringSlot* slot = makeString(value.data(), value.size()).save(pool);
if (slot) {
var->type = JSON_OWNED_RAW;
@ -159,6 +189,7 @@ inline bool variantSetOwnedRaw(JsonVariantData* var, SerializedValue<T> value,
template <typename T>
inline bool variantSetString(JsonVariantData* var, T value, MemoryPool* pool) {
if (!var) return false;
variantFree(var, pool);
StringSlot* slot = value.save(pool);
if (slot) {
var->type = JSON_OWNED_STRING;
@ -170,34 +201,42 @@ inline bool variantSetString(JsonVariantData* var, T value, MemoryPool* pool) {
}
}
inline bool variantSetOwnedString(JsonVariantData* var, StringSlot* slot) {
inline bool variantSetOwnedString(JsonVariantData* var, StringSlot* slot,
MemoryPool* pool) {
if (!var) return false;
variantFree(var, pool);
var->type = JSON_OWNED_STRING;
var->content.asOwnedString = slot;
return true;
}
inline bool variantSetString(JsonVariantData* var, const char* value) {
inline bool variantSetString(JsonVariantData* var, const char* value,
MemoryPool* pool) {
if (!var) return false;
variantFree(var, pool);
var->type = JSON_LINKED_STRING;
var->content.asString = value;
return true;
}
inline void variantSetNull(JsonVariantData* var) {
if (var) var->type = JSON_NULL;
inline void variantSetNull(JsonVariantData* var, MemoryPool* pool) {
if (!var) return;
variantFree(var, pool);
var->type = JSON_NULL;
}
inline JsonArrayData* variantToArray(JsonVariantData* var) {
inline JsonArrayData* variantToArray(JsonVariantData* var, MemoryPool* pool) {
if (!var) return 0;
variantFree(var, pool);
var->type = JSON_ARRAY;
var->content.asArray.head = 0;
var->content.asArray.tail = 0;
return &var->content.asArray;
}
inline JsonObjectData* variantToObject(JsonVariantData* var) {
inline JsonObjectData* variantToObject(JsonVariantData* var, MemoryPool* pool) {
if (!var) return 0;
variantFree(var, pool);
var->type = JSON_OBJECT;
var->content.asObject.head = 0;
var->content.asObject.tail = 0;
@ -208,14 +247,16 @@ inline bool variantCopy(JsonVariantData* dst, const JsonVariantData* src,
MemoryPool* pool) {
if (!dst) return false;
if (!src) {
variantFree(dst, pool);
dst->type = JSON_NULL;
return true;
}
switch (src->type) {
case JSON_ARRAY:
return arrayCopy(variantToArray(dst), &src->content.asArray, pool);
return arrayCopy(variantToArray(dst, pool), &src->content.asArray, pool);
case JSON_OBJECT:
return objectCopy(variantToObject(dst), &src->content.asObject, pool);
return objectCopy(variantToObject(dst, pool), &src->content.asObject,
pool);
case JSON_OWNED_STRING:
return variantSetString(
dst, makeString(src->content.asOwnedString->value), pool);
@ -225,6 +266,7 @@ inline bool variantCopy(JsonVariantData* dst, const JsonVariantData* src,
src->content.asOwnedRaw->size),
pool);
default:
variantFree(dst, pool);
// caution: don't override keyIsOwned
dst->type = src->type;
dst->content = src->content;

View File

@ -53,7 +53,7 @@ class JsonDocument : public Visitable {
template <typename T>
typename JsonVariantTo<T>::type to() {
_memoryPool.clear();
clear();
return getVariant().template to<T>();
}

View File

@ -133,7 +133,7 @@ class JsonVariant : public JsonVariantProxy<JsonVariantData>,
// set(bool value)
FORCE_INLINE bool set(bool value) const {
return variantSetBoolean(_data, value);
return variantSetBoolean(_data, value, _memoryPool);
}
// set(double value);
@ -142,7 +142,7 @@ class JsonVariant : public JsonVariantProxy<JsonVariantData>,
FORCE_INLINE bool set(
T value,
typename enable_if<is_floating_point<T>::value>::type * = 0) const {
return variantSetFloat(_data, static_cast<JsonFloat>(value));
return variantSetFloat(_data, static_cast<JsonFloat>(value), _memoryPool);
}
// set(char)
@ -155,7 +155,7 @@ class JsonVariant : public JsonVariantProxy<JsonVariantData>,
T value,
typename enable_if<is_integral<T>::value && is_signed<T>::value>::type * =
0) const {
return variantSetSignedInteger(_data, value);
return variantSetSignedInteger(_data, value, _memoryPool);
}
// set(unsigned short)
@ -165,12 +165,13 @@ class JsonVariant : public JsonVariantProxy<JsonVariantData>,
FORCE_INLINE bool set(
T value, typename enable_if<is_integral<T>::value &&
is_unsigned<T>::value>::type * = 0) const {
return variantSetSignedInteger(_data, static_cast<JsonUInt>(value));
return variantSetUnsignedInteger(_data, static_cast<JsonUInt>(value),
_memoryPool);
}
// set(SerializedValue<const char *>)
FORCE_INLINE bool set(SerializedValue<const char *> value) const {
return variantSetLinkedRaw(_data, value);
return variantSetLinkedRaw(_data, value, _memoryPool);
}
// set(SerializedValue<std::string>)
@ -201,15 +202,15 @@ class JsonVariant : public JsonVariantProxy<JsonVariantData>,
// set(const char*);
FORCE_INLINE bool set(const char *value) const {
return variantSetString(_data, value);
return variantSetString(_data, value, _memoryPool);
}
// for internal use only
FORCE_INLINE bool set(StringInMemoryPool value) const {
return variantSetOwnedString(_data, value.slot());
return variantSetOwnedString(_data, value.slot(), _memoryPool);
}
FORCE_INLINE bool set(ZeroTerminatedRamStringConst value) const {
return variantSetString(_data, value.c_str());
return variantSetString(_data, value.c_str(), _memoryPool);
}
bool set(JsonVariantConst value) const;

View File

@ -53,19 +53,19 @@ JsonVariant::as() const {
template <typename T>
inline typename enable_if<is_same<T, JsonArray>::value, JsonArray>::type
JsonVariant::to() const {
return JsonArray(_memoryPool, variantToArray(_data));
return JsonArray(_memoryPool, variantToArray(_data, _memoryPool));
}
template <typename T>
typename enable_if<is_same<T, JsonObject>::value, JsonObject>::type
JsonVariant::to() const {
return JsonObject(_memoryPool, variantToObject(_data));
return JsonObject(_memoryPool, variantToObject(_data, _memoryPool));
}
template <typename T>
typename enable_if<is_same<T, JsonVariant>::value, JsonVariant>::type
JsonVariant::to() const {
variantSetNull(_data);
variantSetNull(_data, _memoryPool);
return *this;
}

View File

@ -10,7 +10,7 @@ using namespace Catch::Matchers;
namespace my {
using ARDUINOJSON_NAMESPACE::isinf;
using ARDUINOJSON_NAMESPACE::isnan;
}
} // namespace my
TEST_CASE("deserializeJson(DynamicJsonDocument&)") {
DynamicJsonDocument doc;
@ -223,4 +223,10 @@ TEST_CASE("deserializeJson(DynamicJsonDocument&)") {
REQUIRE(err == DeserializationError::IncompleteInput);
}
}
SECTION("Repeated object key") {
DeserializationError err = deserializeJson(doc, "{a:{b:{c:1}},a:2}");
REQUIRE(err == DeserializationError::Ok);
}
}

View File

@ -22,4 +22,17 @@ TEST_CASE("JsonObject::createNestedObject()") {
obj.createNestedObject(vla);
}
#endif
SECTION("releases memory from nested object") {
obj.createNestedObject(std::string("a"))
.createNestedObject(std::string("b"))
.set(std::string("c"))
.set(1);
// {"a":{"b":{"c":1}}}
REQUIRE(doc.memoryUsage() ==
3 * JSON_OBJECT_SIZE(1) + 3 * JSON_STRING_SIZE(2));
obj.createNestedObject(std::string("a"));
REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1) + JSON_STRING_SIZE(2));
}
}

View File

@ -141,6 +141,22 @@ TEST_CASE("JsonObject::operator[]") {
REQUIRE(expectedSize <= doc.memoryUsage());
}
SECTION("should release string memory when overiding value") {
obj["hello"] = std::string("world");
REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1) + JSON_STRING_SIZE(6));
obj["hello"] = 42;
REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1));
}
SECTION("should release nested array memory when overiding value") {
obj.createNestedArray("hello").add("world");
REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1) + JSON_ARRAY_SIZE(1));
obj["hello"] = 42;
REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1));
}
SECTION("should ignore null key") {
// object must have a value to make a call to strcmp()
obj["dummy"] = 42;

View File

@ -6,12 +6,14 @@ add_executable(JsonVariantTests
as.cpp
compare.cpp
copy.cpp
get.cpp
is.cpp
isnull.cpp
misc.cpp
or.cpp
set_get.cpp
set.cpp
subscript.cpp
to.cpp
undefined.cpp
)

View File

@ -83,4 +83,23 @@ TEST_CASE("JsonVariant::set(JsonVariant)") {
REQUIRE(doc1.memoryUsage() == JSON_STRING_SIZE(8));
REQUIRE(doc2.memoryUsage() == JSON_STRING_SIZE(8));
}
SECTION("releases string memory when replacing with null") {
var1.set(std::string("hello"));
REQUIRE(doc1.memoryUsage() == JSON_STRING_SIZE(6));
var1.set(JsonVariant());
REQUIRE(doc1.memoryUsage() == 0);
}
SECTION("releases string memory when replacing with iteger") {
var1.set(std::string("hello"));
REQUIRE(doc1.memoryUsage() == JSON_STRING_SIZE(6));
var2.set(42);
var1.set(var2);
REQUIRE(doc1.memoryUsage() == 0);
}
}

View File

@ -138,87 +138,3 @@ TEST_CASE("JsonVariant set()/get()") {
checkValue<JsonObject>(object);
}
}
TEST_CASE("JsonVariant and strings") {
DynamicJsonDocument doc;
JsonVariant variant = doc.to<JsonVariant>();
SECTION("stores const char* by reference") {
char str[16];
strcpy(str, "hello");
variant.set(static_cast<const char *>(str));
strcpy(str, "world");
REQUIRE(variant == "world");
}
SECTION("stores char* by copy") {
char str[16];
strcpy(str, "hello");
variant.set(str);
strcpy(str, "world");
REQUIRE(variant == "hello");
}
SECTION("stores unsigned char* by copy") {
char str[16];
strcpy(str, "hello");
variant.set(reinterpret_cast<unsigned char *>(str));
strcpy(str, "world");
REQUIRE(variant == "hello");
}
SECTION("stores signed char* by copy") {
char str[16];
strcpy(str, "hello");
variant.set(reinterpret_cast<signed char *>(str));
strcpy(str, "world");
REQUIRE(variant == "hello");
}
#ifdef HAS_VARIABLE_LENGTH_ARRAY
SECTION("stores VLA by copy") {
int n = 16;
char str[n];
strcpy(str, "hello");
variant.set(str);
strcpy(str, "world");
REQUIRE(variant == "hello");
}
#endif
SECTION("stores std::string by copy") {
std::string str;
str = "hello";
variant.set(str);
str.replace(0, 5, "world");
REQUIRE(variant == "hello");
}
}
TEST_CASE("JsonVariant with not enough memory") {
StaticJsonDocument<1> doc;
JsonVariant v = doc.to<JsonVariant>();
SECTION("std::string") {
v.set(std::string("hello"));
REQUIRE(v.isNull());
}
SECTION("Serialized<std::string>") {
v.set(serialized(std::string("hello")));
REQUIRE(v.isNull());
}
}

146
test/JsonVariant/set.cpp Normal file
View File

@ -0,0 +1,146 @@
// ArduinoJson - arduinojson.org
// Copyright Benoit Blanchon 2014-2018
// MIT License
#include <ArduinoJson.h>
#include <catch.hpp>
TEST_CASE("JsonVariant and strings") {
DynamicJsonDocument doc;
JsonVariant variant = doc.to<JsonVariant>();
SECTION("stores const char* by reference") {
char str[16];
strcpy(str, "hello");
variant.set(static_cast<const char *>(str));
strcpy(str, "world");
REQUIRE(variant == "world");
}
SECTION("stores char* by copy") {
char str[16];
strcpy(str, "hello");
variant.set(str);
strcpy(str, "world");
REQUIRE(variant == "hello");
}
SECTION("stores unsigned char* by copy") {
char str[16];
strcpy(str, "hello");
variant.set(reinterpret_cast<unsigned char *>(str));
strcpy(str, "world");
REQUIRE(variant == "hello");
}
SECTION("stores signed char* by copy") {
char str[16];
strcpy(str, "hello");
variant.set(reinterpret_cast<signed char *>(str));
strcpy(str, "world");
REQUIRE(variant == "hello");
}
#ifdef HAS_VARIABLE_LENGTH_ARRAY
SECTION("stores VLA by copy") {
int n = 16;
char str[n];
strcpy(str, "hello");
variant.set(str);
strcpy(str, "world");
REQUIRE(variant == "hello");
}
#endif
SECTION("stores std::string by copy") {
std::string str;
str = "hello";
variant.set(str);
str.replace(0, 5, "world");
REQUIRE(variant == "hello");
}
}
TEST_CASE("JsonVariant::set() should release string memory") {
DynamicJsonDocument doc;
JsonVariant variant = doc.to<JsonVariant>();
variant.set(std::string("hello"));
REQUIRE(doc.memoryUsage() == JSON_STRING_SIZE(6));
SECTION("int") {
variant.set(-42);
REQUIRE(doc.memoryUsage() == 0);
}
SECTION("unsigned int") {
variant.set(42U);
REQUIRE(doc.memoryUsage() == 0);
}
SECTION("bool") {
variant.set(true);
REQUIRE(doc.memoryUsage() == 0);
}
SECTION("float") {
variant.set(3.14);
REQUIRE(doc.memoryUsage() == 0);
}
SECTION("const char*") {
variant.set("hello");
REQUIRE(doc.memoryUsage() == 0);
}
SECTION("std::string") {
variant.set(std::string("X"));
REQUIRE(doc.memoryUsage() == JSON_STRING_SIZE(2));
}
SECTION("SerializedValue<const char*>") {
variant.set(serialized("[42]"));
REQUIRE(doc.memoryUsage() == 0);
}
SECTION("SerializedValue<std::string>") {
variant.set(serialized(std::string("42")));
REQUIRE(doc.memoryUsage() == JSON_STRING_SIZE(2));
}
SECTION("StringInMemoryPool") {
DeserializationError err =
deserializeJson(doc, std::string("{\"A\":\"hello\",\"A\":\"B\"}"));
REQUIRE(err == DeserializationError::Ok);
// it stores the key twice, but should release "hello"
REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1) + 3 * JSON_STRING_SIZE(2));
}
}
TEST_CASE("JsonVariant with not enough memory") {
StaticJsonDocument<1> doc;
JsonVariant v = doc.to<JsonVariant>();
SECTION("std::string") {
v.set(std::string("hello"));
REQUIRE(v.isNull());
}
SECTION("Serialized<std::string>") {
v.set(serialized(std::string("hello")));
REQUIRE(v.isNull());
}
}

32
test/JsonVariant/to.cpp Normal file
View File

@ -0,0 +1,32 @@
// ArduinoJson - arduinojson.org
// Copyright Benoit Blanchon 2014-2018
// MIT License
#include <ArduinoJson.h>
#include <stdint.h>
#include <catch.hpp>
static const char* null = 0;
TEST_CASE("JsonVariant::to<T>() releases string memory") {
DynamicJsonDocument doc;
JsonVariant variant = doc.to<JsonVariant>();
variant.set(std::string("hello"));
REQUIRE(doc.memoryUsage() == JSON_STRING_SIZE(6));
SECTION("JsonVariant") {
variant.to<JsonVariant>();
REQUIRE(doc.memoryUsage() == 0);
}
SECTION("JsonArray") {
variant.to<JsonArray>();
REQUIRE(doc.memoryUsage() == 0);
}
SECTION("JsonObject") {
variant.to<JsonObject>();
REQUIRE(doc.memoryUsage() == 0);
}
}