Improved deserializeMsgPack() speed by reading several bytes at once

CHANGELOG.md

@@ -7,6 +7,7 @@ HEAD
 * Added support for custom writer classes (issue #1088)
 * Added conversion from `JsonArray` and `JsonObject` to `bool`, to be consistent with `JsonVariant`
 * Fixed `deserializeJson()` when input contains duplicate keys (issue #1095)
+* Improved `deserializeMsgPack()` speed by reading several bytes at once
 
 v6.12.0 (2019-09-05)
 -------

extras/tests/Misc/CMakeLists.txt

@@ -5,7 +5,7 @@
 add_executable(MiscTests
 	conflicts.cpp
 	FloatParts.cpp
-	StreamReader.cpp
+	Readers.cpp
 	StringAdapters.cpp
 	StringWriter.cpp
 	TypeTraits.cpp

extras/tests/Misc/Readers.cpp

@@ -0,0 +1,166 @@
+// ArduinoJson - arduinojson.org
+// Copyright Benoit Blanchon 2014-2019
+// MIT License
+
+#include <ArduinoJson.h>
+#include <catch.hpp>
+
+using namespace ARDUINOJSON_NAMESPACE;
+
+TEST_CASE("StdStreamReader") {
+  SECTION("read()") {
+    std::istringstream src("\x01\xFF");
+    StdStreamReader reader(src);
+
+    REQUIRE(reader.read() == 0x01);
+    REQUIRE(reader.read() == 0xFF);
+    REQUIRE(reader.read() == -1);
+  }
+
+  SECTION("readBytes() all at once") {
+    std::istringstream src("ABC");
+    StdStreamReader reader(src);
+
+    char buffer[8] = "abcd";
+    REQUIRE(reader.readBytes(buffer, 4) == 3);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'd');
+  }
+
+  SECTION("readBytes() in two parts") {
+    std::istringstream src("ABCDEF");
+    StdStreamReader reader(src);
+
+    char buffer[12] = "abcdefg";
+    REQUIRE(reader.readBytes(buffer, 4) == 4);
+    REQUIRE(reader.readBytes(buffer + 4, 4) == 2);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'D');
+    REQUIRE(buffer[4] == 'E');
+    REQUIRE(buffer[5] == 'F');
+    REQUIRE(buffer[6] == 'g');
+  }
+}
+
+TEST_CASE("SafeCharPointerReader") {
+  SECTION("read") {
+    SafeCharPointerReader reader("\x01\xFF", 2);
+    REQUIRE(reader.read() == 0x01);
+    REQUIRE(reader.read() == 0xFF);
+    REQUIRE(reader.read() == -1);
+    REQUIRE(reader.read() == -1);
+  }
+
+  SECTION("readBytes() all at once") {
+    SafeCharPointerReader reader("ABCD", 3);
+
+    char buffer[8] = "abcd";
+    REQUIRE(reader.readBytes(buffer, 4) == 3);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'd');
+  }
+
+  SECTION("readBytes() in two parts") {
+    SafeCharPointerReader reader("ABCDEF", 6);
+
+    char buffer[8] = "abcdefg";
+    REQUIRE(reader.readBytes(buffer, 4) == 4);
+    REQUIRE(reader.readBytes(buffer + 4, 4) == 2);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'D');
+    REQUIRE(buffer[4] == 'E');
+    REQUIRE(buffer[5] == 'F');
+    REQUIRE(buffer[6] == 'g');
+  }
+}
+
+TEST_CASE("UnsafeCharPointerReader") {
+  SECTION("read()") {
+    UnsafeCharPointerReader reader("\x01\xFF\x00\x12");
+    REQUIRE(reader.read() == 0x01);
+    REQUIRE(reader.read() == 0xFF);
+    REQUIRE(reader.read() == 0);
+    REQUIRE(reader.read() == 0x12);
+  }
+
+  SECTION("readBytes() all at once") {
+    UnsafeCharPointerReader reader("ABCD");
+
+    char buffer[8] = "abcd";
+    REQUIRE(reader.readBytes(buffer, 3) == 3);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'd');
+  }
+
+  SECTION("readBytes() in two parts") {
+    UnsafeCharPointerReader reader("ABCDEF");
+
+    char buffer[8] = "abcdefg";
+    REQUIRE(reader.readBytes(buffer, 4) == 4);
+    REQUIRE(reader.readBytes(buffer + 4, 2) == 2);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'D');
+    REQUIRE(buffer[4] == 'E');
+    REQUIRE(buffer[5] == 'F');
+    REQUIRE(buffer[6] == 'g');
+  }
+}
+
+TEST_CASE("IteratorReader") {
+  SECTION("read()") {
+    std::string src("\x01\xFF");
+    IteratorReader<std::string::const_iterator> reader(src.begin(), src.end());
+
+    REQUIRE(reader.read() == 0x01);
+    REQUIRE(reader.read() == 0xFF);
+    REQUIRE(reader.read() == -1);
+  }
+
+  SECTION("readBytes() all at once") {
+    std::string src("ABC");
+    IteratorReader<std::string::const_iterator> reader(src.begin(), src.end());
+
+    char buffer[8] = "abcd";
+    REQUIRE(reader.readBytes(buffer, 4) == 3);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'd');
+  }
+
+  SECTION("readBytes() in two parts") {
+    std::string src("ABCDEF");
+    IteratorReader<std::string::const_iterator> reader(src.begin(), src.end());
+
+    char buffer[12] = "abcdefg";
+    REQUIRE(reader.readBytes(buffer, 4) == 4);
+    REQUIRE(reader.readBytes(buffer + 4, 4) == 2);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'D');
+    REQUIRE(buffer[4] == 'E');
+    REQUIRE(buffer[5] == 'F');
+    REQUIRE(buffer[6] == 'g');
+  }
+}

extras/tests/Misc/StreamReader.cpp

@@ -1,33 +0,0 @@
-// ArduinoJson - arduinojson.org
-// Copyright Benoit Blanchon 2014-2019
-// MIT License
-
-#include <ArduinoJson.h>
-#include <catch.hpp>
-
-using namespace ARDUINOJSON_NAMESPACE;
-
-TEST_CASE("StdStreamReader") {
-  std::istringstream src("\x01\xFF");
-  StdStreamReader reader(src);
-
-  REQUIRE(reader.read() == 0x01);
-  REQUIRE(reader.read() == 0xFF);
-  REQUIRE(reader.read() == -1);
-}
-
-TEST_CASE("SafeCharPointerReader") {
-  SafeCharPointerReader reader("\x01\xFF", 2);
-
-  REQUIRE(reader.read() == 0x01);
-  REQUIRE(reader.read() == 0xFF);
-  REQUIRE(reader.read() == -1);
-}
-
-TEST_CASE("UnsafeCharPointerReader") {
-  UnsafeCharPointerReader reader("\x01\xFF");
-
-  REQUIRE(reader.read() == 0x01);
-  REQUIRE(reader.read() == 0xFF);
-  REQUIRE(reader.read() == 0);
-}

extras/tests/MixedConfiguration/enable_progmem_1.cpp

@@ -85,3 +85,83 @@ TEST_CASE("memcpy_P") {
   CHECK(dst[2] == 'C');
   CHECK(dst[3] == 0);
 }
+
+TEST_CASE("SafeCharPointerReader") {
+  using ARDUINOJSON_NAMESPACE::SafeFlashStringReader;
+
+  SECTION("read") {
+    SafeFlashStringReader reader(F("\x01\xFF"), 2);
+    REQUIRE(reader.read() == 0x01);
+    REQUIRE(reader.read() == 0xFF);
+    REQUIRE(reader.read() == -1);
+    REQUIRE(reader.read() == -1);
+  }
+
+  SECTION("readBytes() all at once") {
+    SafeFlashStringReader reader(F("ABCD"), 3);
+
+    char buffer[8] = "abcd";
+    REQUIRE(reader.readBytes(buffer, 4) == 3);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'd');
+  }
+
+  SECTION("readBytes() in two parts") {
+    SafeFlashStringReader reader(F("ABCDEF"), 6);
+
+    char buffer[8] = "abcdefg";
+    REQUIRE(reader.readBytes(buffer, 4) == 4);
+    REQUIRE(reader.readBytes(buffer + 4, 4) == 2);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'D');
+    REQUIRE(buffer[4] == 'E');
+    REQUIRE(buffer[5] == 'F');
+    REQUIRE(buffer[6] == 'g');
+  }
+}
+
+TEST_CASE("UnsafeFlashStringReader") {
+  using ARDUINOJSON_NAMESPACE::UnsafeFlashStringReader;
+
+  SECTION("read()") {
+    UnsafeFlashStringReader reader(F("\x01\xFF\x00\x12"));
+    REQUIRE(reader.read() == 0x01);
+    REQUIRE(reader.read() == 0xFF);
+    REQUIRE(reader.read() == 0);
+    REQUIRE(reader.read() == 0x12);
+  }
+
+  SECTION("readBytes() all at once") {
+    UnsafeFlashStringReader reader(F("ABCD"));
+
+    char buffer[8] = "abcd";
+    REQUIRE(reader.readBytes(buffer, 3) == 3);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'd');
+  }
+
+  SECTION("readBytes() in two parts") {
+    UnsafeFlashStringReader reader(F("ABCDEF"));
+
+    char buffer[8] = "abcdefg";
+    REQUIRE(reader.readBytes(buffer, 4) == 4);
+    REQUIRE(reader.readBytes(buffer + 4, 2) == 2);
+
+    REQUIRE(buffer[0] == 'A');
+    REQUIRE(buffer[1] == 'B');
+    REQUIRE(buffer[2] == 'C');
+    REQUIRE(buffer[3] == 'D');
+    REQUIRE(buffer[4] == 'E');
+    REQUIRE(buffer[5] == 'F');
+    REQUIRE(buffer[6] == 'g');
+  }
+}

src/ArduinoJson/Deserialization/ArduinoStreamReader.hpp

@@ -23,6 +23,10 @@ struct ArduinoStreamReader {
     char c;
     return _stream.readBytes(&c, 1) ? c : -1;
   }
+
+  size_t readBytes(char* buffer, size_t length) {
+    return _stream.readBytes(buffer, length);
+  }
 };
 
 inline ArduinoStreamReader makeReader(Stream& input) {

src/ArduinoJson/Deserialization/CharPointerReader.hpp

@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include <ArduinoJson/Deserialization/IteratorReader.hpp>
 #include <ArduinoJson/Namespace.hpp>
 
 namespace ARDUINOJSON_NAMESPACE {
@@ -28,22 +29,17 @@ class UnsafeCharPointerReader {
   int read() {
     return static_cast<unsigned char>(*_ptr++);
   }
+
+  size_t readBytes(char* buffer, size_t length) {
+    for (size_t i = 0; i < length; i++) buffer[i] = *_ptr++;
+    return length;
+  }
 };
 
-class SafeCharPointerReader {
+class SafeCharPointerReader : public IteratorReader<const char*> {
-  const char* _ptr;
-  const char* _end;
-
  public:
   explicit SafeCharPointerReader(const char* ptr, size_t len)
-      : _ptr(ptr ? ptr : reinterpret_cast<const char*>("")), _end(_ptr + len) {}
+      : IteratorReader<const char*>(ptr, ptr + len) {}
-
-  int read() {
-    if (_ptr < _end)
-      return static_cast<unsigned char>(*_ptr++);
-    else
-      return -1;
-  }
 };
 
 template <typename TChar>

src/ArduinoJson/Deserialization/FlashStringReader.hpp

@@ -19,6 +19,12 @@ class UnsafeFlashStringReader {
   int read() {
     return pgm_read_byte(_ptr++);
   }
+
+  size_t readBytes(char* buffer, size_t length) {
+    memcpy_P(buffer, _ptr, length);
+    _ptr += length;
+    return length;
+  }
 };
 
 class SafeFlashStringReader {
@@ -35,6 +41,14 @@ class SafeFlashStringReader {
     else
       return -1;
   }
+
+  size_t readBytes(char* buffer, size_t length) {
+    size_t available = static_cast<size_t>(_end - _ptr);
+    if (available < length) length = available;
+    memcpy_P(buffer, _ptr, length);
+    _ptr += length;
+    return length;
+  }
 };
 
 inline UnsafeFlashStringReader makeReader(const __FlashStringHelper* input) {

src/ArduinoJson/Deserialization/IteratorReader.hpp

@@ -22,6 +22,12 @@ class IteratorReader {
     else
       return -1;
   }
+
+  size_t readBytes(char* buffer, size_t length) {
+    size_t i = 0;
+    while (i < length && _ptr < _end) buffer[i++] = *_ptr++;
+    return i;
+  }
 };
 
 template <typename TInput>

src/ArduinoJson/Deserialization/StdStreamReader.hpp

@@ -24,6 +24,11 @@ class StdStreamReader {
     return _stream.get();
   }
 
+  size_t readBytes(char* buffer, size_t length) {
+    _stream.read(buffer, static_cast<std::streamsize>(length));
+    return static_cast<size_t>(_stream.gcount());
+  }
+
  private:
   StdStreamReader& operator=(const StdStreamReader&);  // Visual Studio C4512
 };

src/ArduinoJson/MsgPack/MsgPackDeserializer.hpp

@@ -142,10 +142,7 @@ class MsgPackDeserializer {
   }
 
   bool readBytes(uint8_t *p, size_t n) {
-    for (size_t i = 0; i < n; i++) {
+    return _reader.readBytes(reinterpret_cast<char *>(p), n) == n;
-      if (!readByte(p[i])) return false;
-    }
-    return true;
   }
 
   template <typename T>